From 3eb15d2eabb0b170c96de7241aa2eed7d5d27879 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 16 Mar 2018 22:33:33 +0100 Subject: Remove DocumentList with children... Conflicts: storage/src/tests/persistence/filestorage/filestormanagertest.cpp Resolve merge conflict. --- vdslib/src/tests/container/CMakeLists.txt | 1 - vdslib/src/tests/container/documentlisttest.cpp | 534 --------------------- vdslib/src/vespa/vdslib/container/CMakeLists.txt | 4 - vdslib/src/vespa/vdslib/container/documentlist.cpp | 448 ----------------- vdslib/src/vespa/vdslib/container/documentlist.h | 213 -------- .../vespa/vdslib/container/mutabledocumentlist.cpp | 217 --------- .../vespa/vdslib/container/mutabledocumentlist.h | 46 -- .../src/vespa/vdslib/container/operationlist.cpp | 48 -- vdslib/src/vespa/vdslib/container/operationlist.h | 84 ---- .../vdslib/container/writabledocumentlist.cpp | 97 ---- .../vespa/vdslib/container/writabledocumentlist.h | 46 -- 11 files changed, 1738 deletions(-) delete mode 100644 vdslib/src/tests/container/documentlisttest.cpp delete mode 100644 vdslib/src/vespa/vdslib/container/documentlist.cpp delete mode 100644 vdslib/src/vespa/vdslib/container/documentlist.h delete mode 100644 vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp delete mode 100644 vdslib/src/vespa/vdslib/container/mutabledocumentlist.h delete mode 100644 vdslib/src/vespa/vdslib/container/operationlist.cpp delete mode 100644 vdslib/src/vespa/vdslib/container/operationlist.h delete mode 100644 vdslib/src/vespa/vdslib/container/writabledocumentlist.cpp delete mode 100644 vdslib/src/vespa/vdslib/container/writabledocumentlist.h (limited to 'vdslib') diff --git a/vdslib/src/tests/container/CMakeLists.txt b/vdslib/src/tests/container/CMakeLists.txt index c78e0276bf6..8f82bf17a28 100644 --- a/vdslib/src/tests/container/CMakeLists.txt +++ b/vdslib/src/tests/container/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vdslib_containertest SOURCES - documentlisttest.cpp parameterstest.cpp searchresulttest.cpp documentsummarytest.cpp diff --git a/vdslib/src/tests/container/documentlisttest.cpp b/vdslib/src/tests/container/documentlisttest.cpp deleted file mode 100644 index 884241cc483..00000000000 --- a/vdslib/src/tests/container/documentlisttest.cpp +++ /dev/null @@ -1,534 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using document::DocumentTypeRepo; -using document::readDocumenttypesConfig; -using vespalib::nbostream; - -namespace vdslib { - -struct WritableDocumentListTest : public CppUnit::TestFixture { - - void testSimple(); - void testAlignedWriting(); - void testSizeOf(); - void testReadJavaFile(); - void testGetSerializedSize(); - void testCopyEntry(); - void testOperationList(); - void testSetTimestamp(); - void testDifferentBuckets(); - - CPPUNIT_TEST_SUITE(WritableDocumentListTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST(testAlignedWriting); - CPPUNIT_TEST(testSizeOf); - CPPUNIT_TEST(testReadJavaFile); - CPPUNIT_TEST(testGetSerializedSize); - CPPUNIT_TEST(testCopyEntry); - CPPUNIT_TEST(testOperationList); - CPPUNIT_TEST(testSetTimestamp); - CPPUNIT_TEST(testDifferentBuckets); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(WritableDocumentListTest); - -void WritableDocumentListTest::testDifferentBuckets() -{ - document::TestDocMan docman; - std::vector buffer(1024); - WritableDocumentList block(docman.getTypeRepoSP(), &buffer[0], buffer.size()); - - std::unique_ptr doc1 = docman.createDocument("This is a test", "userdoc:test:1234:1"); - block.addPut(*doc1); - - std::unique_ptr doc2 = docman.createDocument("This is a test", "userdoc:test:4567:1"); - try { - block.addPut(*doc2); - CPPUNIT_ASSERT(false); - } catch (...) { - } - - block.addRemove(document::DocumentId("userdoc:test:1234:2")); - - try { - block.addRemove(document::DocumentId("userdoc:test:4567:2")); - CPPUNIT_ASSERT(false); - } catch (...) { - } -} - -void WritableDocumentListTest::testSimple() -{ - document::TestDocMan docman; - std::vector buffer(1024); - std::vector docs; - for (uint32_t i=1; i<10; ++i) { - docs.push_back(document::Document::SP( - docman.createDocument("This is a test", - vespalib::make_string("userdoc:test:123456789:%d", i) - ).release())); - } - - WritableDocumentList block(docman.getTypeRepoSP(), &buffer[0], buffer.size()); - // begin() should now equal start() - CPPUNIT_ASSERT(block.begin() == block.end()); - // Add docs in the easy way. - block.addPut(*docs[0]); - block.addRemove(docs[1]->getId()); - block.addPut(*docs[2], 0xfee1deadbabeb00bull); - - // Add the way slotfile will. - WritableDocumentList::MetaEntry entry; - entry.timestamp = 1234; - entry.headerPos = 0; - nbostream stream; - docs[3]->serializeHeader(stream); - entry.headerLen = stream.size(); - entry.bodyPos = entry.headerLen; - stream.clear(); - docs[3]->serializeBody(stream); - entry.bodyLen = stream.size(); - entry.flags = 0; - - CPPUNIT_ASSERT(block.countFree() > entry.headerLen + entry.bodyLen - + sizeof(WritableDocumentList::MetaEntry)); - char *pos = block.prepareMultiput(1, entry.headerLen + entry.bodyLen); - CPPUNIT_ASSERT(pos != 0); - document::ByteBuffer bb(pos, entry.headerLen + entry.bodyLen); - docs[3]->serializeHeader(bb); - docs[3]->serializeBody(bb); - std::vector entries; - entries.push_back(entry); - block.commitMultiput(entries, pos); - - // Copy buffer someplace else to simulate serialize/deserialize - std::vector copy(buffer); - - // Get documents out again. Verify correctness.. - { - WritableDocumentList::const_iterator it = block.begin(); - // First document putted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - - std::unique_ptr doc(it->getDocument()); - CPPUNIT_ASSERT(doc.get()); - - CPPUNIT_ASSERT_EQUAL(*docs[0], *doc); - - CPPUNIT_ASSERT(++it != block.end()); - // First document deleted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - - CPPUNIT_ASSERT_EQUAL(docs[1]->getId(), doc->getId()); - CPPUNIT_ASSERT(++it != block.end()); - // Second document putted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0xfee1deadbabeb00bull, - it->getTimestamp()); - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - - CPPUNIT_ASSERT_EQUAL(*docs[2], *doc); - CPPUNIT_ASSERT(++it != block.end()); - // Third document putted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)1234, it->getTimestamp()); - - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - - CPPUNIT_ASSERT_EQUAL(*docs[3], *doc); - CPPUNIT_ASSERT(++it == block.end()); - } - - // Test downsizing - CPPUNIT_ASSERT_EQUAL(621u, block.countFree()); - uint32_t requiredSize = block.getBufferSize() - block.countFree(); - std::vector otherBuffer(requiredSize); - DocumentList otherBlock(block, &otherBuffer[0], otherBuffer.size()); - CPPUNIT_ASSERT_EQUAL(403u, otherBlock.getBufferSize()); - // Get documents out again of other block. Verify correctness.. - { - DocumentList::const_iterator it = otherBlock.begin(); - // First document putted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - - std::unique_ptr doc(it->getDocument()); - CPPUNIT_ASSERT(doc.get()); - - CPPUNIT_ASSERT_EQUAL(*docs[0], *doc); - - CPPUNIT_ASSERT(++it != block.end()); - // First document deleted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - - CPPUNIT_ASSERT_EQUAL(docs[1]->getId(), doc->getId()); - CPPUNIT_ASSERT(++it != block.end()); - // Second document putted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0xfee1deadbabeb00bull, it->getTimestamp()); - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - CPPUNIT_ASSERT_EQUAL(*docs[2], *doc); - CPPUNIT_ASSERT(++it != block.end()); - // Third document putted - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)1234, it->getTimestamp()); - - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - CPPUNIT_ASSERT_EQUAL(*docs[3], *doc); - CPPUNIT_ASSERT(++it == block.end()); - } - - // begin() should equal start() after clear - block.clear(); - CPPUNIT_ASSERT(block.begin() == block.end()); -} - -void WritableDocumentListTest::testSetTimestamp() -{ - document::TestDocMan docman; - std::vector buffer(1024); - - WritableDocumentList block(docman.getTypeRepoSP(), &buffer[0], buffer.size()); - // begin() should now equal start() - CPPUNIT_ASSERT(block.begin() == block.end()); - // Add docs in the easy way. - std::unique_ptr doc( - docman.createDocument("This is a test", - vespalib::make_string("userdoc:test:123456789:t"))); - - block.addPut(*doc); - - CPPUNIT_ASSERT(block.begin() != block.end()); - - block.begin()->setTimestamp(1234); - - CPPUNIT_ASSERT_EQUAL((Timestamp)1234, block.begin()->getTimestamp()); -} - -void WritableDocumentListTest::testAlignedWriting() -{ - document::TestDocMan docman; - vespalib::RandomGen randomizer(5123); - std::vector buffer(1024*1024); - - std::vector docs; - for (uint32_t i=1; i<10; ++i) { - docs.push_back(document::Document::SP( - docman.createDocument("Aligned writing test blaaaah", - vespalib::make_string("userdoc:test:123456789:%d", i) - ).release())); - } - - WritableDocumentList block(docman.getTypeRepoSP(), &buffer[0], buffer.size()); - - // Add documents aligned the way slotfile will. - for (uint32_t i=1; i<50; ++i) { - std::vector entries; - uint32_t currentPos = 0; - for (uint32_t j=0, n=randomizer.nextUint32(1,10); j currentPos + entries.size() - * sizeof(WritableDocumentList::MetaEntry)); - char *pos = block.prepareMultiput(entries.size(), currentPos); - CPPUNIT_ASSERT(pos != 0); - CPPUNIT_ASSERT((pos - &buffer[0]) % 512 == 0); - block.commitMultiput(entries, pos); - } -} - -void WritableDocumentListTest::testSizeOf() -{ - CPPUNIT_ASSERT_EQUAL(size_t(32), sizeof(WritableDocumentList::MetaEntry)); - - std::string buffercont("This is a buffer of data we will create meta " - " entry from to verify binary compability"); - std::vector buffer(buffercont.begin(), buffercont.end()); - CPPUNIT_ASSERT(buffer.size() > sizeof(WritableDocumentList::MetaEntry)); - - WritableDocumentList::MetaEntry* e(reinterpret_cast(&buffer[0])); - - CPPUNIT_ASSERT_EQUAL(Timestamp(2338328219631577172ull), e->timestamp); - CPPUNIT_ASSERT_EQUAL(uint32_t(1969365089), e->headerPos); - CPPUNIT_ASSERT_EQUAL(uint32_t(1919247974), e->headerLen); - CPPUNIT_ASSERT_EQUAL(uint32_t(543584032), e->bodyPos); - CPPUNIT_ASSERT_EQUAL(uint32_t(1635017060), e->bodyLen); - CPPUNIT_ASSERT_EQUAL(uint32_t(32), uint32_t(e->flags)); -} - -void WritableDocumentListTest::testReadJavaFile() -{ - DocumentTypeRepo::SP repo(new DocumentTypeRepo(readDocumenttypesConfig(TEST_PATH("../test/files/documenttypes.cfg")))); - - //read file - int file = open(TEST_PATH("../test/files/documentlist-java.dat").c_str(), O_RDONLY); - if (file == -1) { - CPPUNIT_ASSERT(0); - } - - uint32_t len = lseek(file, 0, SEEK_END); - lseek(file, 0, SEEK_SET); - - vespalib::MallocPtr data(len); - CPPUNIT_ASSERT_EQUAL((ssize_t) len, read(file, data, len)); - close(file); - - - //create documentlist - DocumentList block(repo, data.str(), len, true); - - CPPUNIT_ASSERT_EQUAL((uint32_t) 4, block.size()); - - DocumentList::const_iterator it = block.begin(); - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT(!it->isBodyStripped()); - CPPUNIT_ASSERT(!it->isUpdateEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - std::unique_ptr doc(it->getDocument()); - CPPUNIT_ASSERT(doc.get()); - CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:foo:99999999:1"), - doc->getId()); - vespalib::string foo = "foo"; - CPPUNIT_ASSERT_EQUAL(foo, doc->getValue("headerstring")->getAsString()); - - CPPUNIT_ASSERT(++it != block.end()); - - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(it->isRemoveEntry()); - CPPUNIT_ASSERT(!it->isBodyStripped()); - CPPUNIT_ASSERT(!it->isUpdateEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:foo:99999999:2"), - doc->getId()); - - CPPUNIT_ASSERT(++it != block.end()); - - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT(!it->isBodyStripped()); - CPPUNIT_ASSERT(!it->isUpdateEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:foo:99999999:3"), - doc->getId()); - CPPUNIT_ASSERT_EQUAL(5.5f, doc->getValue("bodyfloat")->getAsFloat()); - - CPPUNIT_ASSERT(++it != block.end()); - - CPPUNIT_ASSERT(it->valid()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - CPPUNIT_ASSERT(!it->isBodyStripped()); - CPPUNIT_ASSERT(it->isUpdateEntry()); - CPPUNIT_ASSERT_EQUAL((uint64_t)0, it->getTimestamp()); - document::DocumentUpdate::UP docUp = it->getUpdate(); - CPPUNIT_ASSERT(docUp.get()); - const document::AssignValueUpdate* valUp = dynamic_cast(docUp->getUpdates().front().getUpdates().front().get()); - vespalib::string ballooooo = "ballooooo"; - CPPUNIT_ASSERT_EQUAL(ballooooo, valUp->getValue().getAsString()); - - CPPUNIT_ASSERT(++it == block.end()); -} - -void WritableDocumentListTest::testGetSerializedSize() { - document::TestDocMan docman; - std::vector buffer(1024); - std::vector docs; - for (uint32_t i=1; i<3; ++i) { - docs.push_back(document::Document::SP( - docman.createDocument("This is a test, blah bloh bluh blih", - vespalib::make_string("userdoc:test:1298798789:%d", i) - ).release())); - } - WritableDocumentList block(docman.getTypeRepoSP(), &buffer[0], buffer.size()); - // begin() should now equal start() - CPPUNIT_ASSERT(block.begin() == block.end()); - // Add docs in the easy way. - block.addPut(*docs[0]); - block.addRemove(docs[1]->getId()); - - WritableDocumentList::const_iterator it = block.begin(); - CPPUNIT_ASSERT_EQUAL((const uint32_t)(docs[0]->serialize()->getLength() - + sizeof(DocumentList::MetaEntry)), - it->getSerializedSize()); -} - -void WritableDocumentListTest::testCopyEntry() { - DocumentTypeRepo::SP repo(new DocumentTypeRepo(readDocumenttypesConfig( - TEST_PATH("../test/files/documenttypes.cfg")))); - - //read file - int file = open(TEST_PATH("../test/files/documentlist-java.dat").c_str(), O_RDONLY); - if (file == -1) { - CPPUNIT_ASSERT(0); - } - - uint32_t len = lseek(file, 0, SEEK_END); - lseek(file, 0, SEEK_SET); - - vespalib::MallocPtr data(len); - CPPUNIT_ASSERT_EQUAL((ssize_t) len, read(file, data, len)); - close(file); - - - //create documentlist - DocumentList block(repo, data.str(), len, true); - - CPPUNIT_ASSERT_EQUAL((uint32_t) 4, block.size()); - - //create a writabledocumentlist - std::vector buffer(1024); - WritableDocumentList wrBlock(repo, &buffer[0], buffer.size()); - - DocumentList::const_iterator it = block.begin(); - wrBlock.addEntry(*it); - CPPUNIT_ASSERT_EQUAL((uint32_t) 1, wrBlock.size()); - - ++it; - wrBlock.addEntry(*it); - CPPUNIT_ASSERT_EQUAL((uint32_t) 2, wrBlock.size()); - - ++it; - wrBlock.addEntry(*it); - CPPUNIT_ASSERT_EQUAL((uint32_t) 3, wrBlock.size()); - - ++it; - wrBlock.addEntry(*it); - CPPUNIT_ASSERT_EQUAL((uint32_t) 4, wrBlock.size()); - - - it = block.begin(); - DocumentList::const_iterator wrIt = wrBlock.begin(); - - //test equality of first entry - CPPUNIT_ASSERT_EQUAL(it->getFlags(), wrIt->getFlags()); - std::unique_ptr doc(it->getDocument()); - CPPUNIT_ASSERT(doc.get()); - - std::unique_ptr wrDoc(wrIt->getDocument()); - CPPUNIT_ASSERT(wrDoc.get()); - CPPUNIT_ASSERT_EQUAL(*doc, *wrDoc); - - ++it; - ++wrIt; - - //test equality of second entry - CPPUNIT_ASSERT_EQUAL(it->getFlags(), wrIt->getFlags()); - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - - wrDoc = wrIt->getDocument(); - CPPUNIT_ASSERT(wrDoc.get()); - CPPUNIT_ASSERT_EQUAL(*doc, *wrDoc); - - ++it; - ++wrIt; - - //test equality of third entry - CPPUNIT_ASSERT_EQUAL(it->getFlags(), wrIt->getFlags()); - doc = it->getDocument(); - CPPUNIT_ASSERT(doc.get()); - - wrDoc = wrIt->getDocument(); - CPPUNIT_ASSERT(wrDoc.get()); - CPPUNIT_ASSERT_EQUAL(*doc, *wrDoc); - - ++it; - ++wrIt; - - //test equality of fourth entry - CPPUNIT_ASSERT_EQUAL(it->getFlags(), wrIt->getFlags()); - document::DocumentUpdate::UP docUp = it->getUpdate(); - CPPUNIT_ASSERT(docUp.get()); - - document::DocumentUpdate::UP wrDocUp = wrIt->getUpdate(); - CPPUNIT_ASSERT(wrDocUp.get()); - CPPUNIT_ASSERT_EQUAL(docUp->getId(), wrDocUp->getId()); -} - -void WritableDocumentListTest::testOperationList() -{ - document::TestDocMan docman; - OperationList ol; - for (uint32_t i=0; i<3000; ++i) { - ol.addPut(docman.createDocument( - "This is a test, blah bloh bluh blih", - vespalib::make_string("userdoc:test:1298798789:%d", i))); - } - - for (uint32_t i=5000; i<5900; ++i) { - ol.addRemove(document::DocumentId( - vespalib::make_string("userdoc:test:1298798789:%d", i))); - } - - std::vector buf(ol.getRequiredBufferSize()); - - MutableDocumentList mdl(docman.getTypeRepoSP(), &(buf[0]), buf.size()); - mdl.addOperationList(ol); - - DocumentList::const_iterator it = mdl.begin(); - - for (uint32_t i=0; i<3000; ++i, it++) { - CPPUNIT_ASSERT_EQUAL( - vespalib::make_string("userdoc:test:1298798789:%d", i), - it->getDocument()->getId().toString()); - CPPUNIT_ASSERT(!it->isRemoveEntry()); - } - - for (uint32_t i=5000; i<5900; ++i, it++) { - CPPUNIT_ASSERT_EQUAL( - vespalib::make_string("userdoc:test:1298798789:%d", i), - it->getDocument()->getId().toString()); - CPPUNIT_ASSERT(it->isRemoveEntry()); - } -} - -} diff --git a/vdslib/src/vespa/vdslib/container/CMakeLists.txt b/vdslib/src/vespa/vdslib/container/CMakeLists.txt index e9a73cf65e5..0526606196b 100644 --- a/vdslib/src/vespa/vdslib/container/CMakeLists.txt +++ b/vdslib/src/vespa/vdslib/container/CMakeLists.txt @@ -2,13 +2,9 @@ vespa_add_library(vdslib_container OBJECT SOURCES dummycppfile.cpp - documentlist.cpp - mutabledocumentlist.cpp - writabledocumentlist.cpp parameters.cpp searchresult.cpp documentsummary.cpp - operationlist.cpp visitorstatistics.cpp visitorordering.cpp DEPENDS diff --git a/vdslib/src/vespa/vdslib/container/documentlist.cpp b/vdslib/src/vespa/vdslib/container/documentlist.cpp deleted file mode 100644 index 18f7e138260..00000000000 --- a/vdslib/src/vespa/vdslib/container/documentlist.cpp +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentlist.h" -#include -#include -#include -#include -#include -#include -#include - -#include -LOG_SETUP(".vdslib.container.documentlist"); - -using document::ByteBuffer; - -namespace vdslib { - -DocumentList::MetaEntry::MetaEntry() -{ - memset(this, 0, sizeof(MetaEntry)); -} - -void -DocumentList::MetaEntry::print(std::ostream& out, const std::string& indent) const -{ - (void) indent; - out << "MetaEntry(Ts 0x" << std::hex << timestamp << ", h " - << headerPos << "/" << headerLen << ", b " << bodyPos << "/" << bodyLen; - if (flags & REMOVE_ENTRY) out << ", remove"; - if (flags & BODY_STRIPPED) out << ", body stripped"; - if (flags & BODY_IN_HEADER) out << ", body in header"; - out << ")"; -} - -DocumentList::Entry::Entry(char* start, uint32_t entry, - const document::DocumentTypeRepo &repo) - : _metaEntry(reinterpret_cast( - start + sizeof(uint32_t) + entry * sizeof(MetaEntry))), - _start(start), - _entry(entry), - _repo(&repo) -{ -} - -DocumentList::Entry -DocumentList::Entry::next() const -{ - if (_entry + 1 >= *reinterpret_cast(_start)) { - return Entry(); - } - return Entry(_start, _entry + 1, *_repo); -} - -document::DocumentId -DocumentList::Entry::getDocumentId() const -{ - ByteBuffer buf(_start + _metaEntry->headerPos, _metaEntry->headerLen); - try { - if (isUpdateEntry()) { - document::DocumentUpdate::UP update( - document::DocumentUpdate::create42(*_repo, buf)); - return update->getId(); - } else { - return document::Document::getIdFromSerialized(buf); - } - } catch (const document::DeserializeException& e) { - std::ostringstream ss; - ss << "Failed to deserialize document ID from " << *this; - throw document::DeserializeException(ss.str(), e, VESPA_STRLOC); - } -} - -std::unique_ptr -DocumentList::Entry::getDocument( - const document::DocumentType *anticipatedType) const -{ - if (isUpdateEntry()) { - throw vespalib::IllegalStateException("Entry contains an update. " - "Call getUpdate(), not getDocument()", VESPA_STRLOC); - } - ByteBuffer hbuf(_start + _metaEntry->headerPos, _metaEntry->headerLen); - ByteBuffer bbuf(_start + _metaEntry->bodyPos, _metaEntry->bodyLen); - std::unique_ptr doc; - try { - if (_metaEntry->bodyLen == 0) { - doc.reset(new document::Document(*_repo, hbuf, anticipatedType)); - } else { - doc.reset(new document::Document(*_repo, hbuf, bbuf, anticipatedType)); - } - } catch (const document::DeserializeException& e) { - std::ostringstream ss; - ss << "Failed to deserialize document from " << *this; - throw document::DeserializeException(ss.str(), e, VESPA_STRLOC); - } - if (hbuf.getRemaining() != 0 || bbuf.getRemaining() != 0) { - assert(hbuf.getPos() + hbuf.getRemaining() == hbuf.getLength()); - assert(bbuf.getPos() + bbuf.getRemaining() == bbuf.getLength()); - throw document::DeserializeException(vespalib::make_string( - "Deserializing document %s, only %lu of %lu header bytes and " - "%lu of %lu body bytes were consumed.", - doc->getId().toString().c_str(), - hbuf.getPos(), hbuf.getLength(), - bbuf.getPos(), bbuf.getLength()), VESPA_STRLOC); - } - doc->setLastModified(_metaEntry->timestamp); - return doc; -} - -std::unique_ptr -DocumentList::Entry::getUpdate() const -{ - if (!isUpdateEntry()) { - throw vespalib::IllegalStateException("Entry contains a document. " - "Call getDocument(), not getUpdate()", VESPA_STRLOC); - } - assert(_metaEntry->bodyLen == 0); - ByteBuffer buf(_start + _metaEntry->headerPos, _metaEntry->headerLen); - document::DocumentUpdate::UP update( - document::DocumentUpdate::create42(*_repo, buf)); - if (buf.getRemaining() != 0) { - assert(buf.getPos() + buf.getRemaining() == buf.getLength()); - throw document::DeserializeException(vespalib::make_string( - "Deserializing document update %s, only %lu of %lu bytes " - "were consumed.", - update->getId().toString().c_str(), - buf.getPos(), buf.getLength()), VESPA_STRLOC); - } - return update; -} - -bool -DocumentList::Entry::getUpdate(document::DocumentUpdate& update) const -{ - if (!isUpdateEntry()) { - throw vespalib::IllegalStateException("Entry contains a document. " - "Call getDocument(), not getUpdate()", VESPA_STRLOC); - } - ByteBuffer buf(_start + _metaEntry->headerPos, _metaEntry->headerLen); - assert(_metaEntry->bodyLen == 0); - update.deserialize42(*_repo, buf); - return (buf.getRemaining() == 0); -} - - -const DocumentList::Entry::BufferPosition -DocumentList::Entry::getRawHeader() const -{ - return BufferPosition(_start + _metaEntry->headerPos, - _metaEntry->headerLen); -} - -const DocumentList::Entry::BufferPosition -DocumentList::Entry::getRawBody() const -{ - return BufferPosition(_start + _metaEntry->bodyPos, _metaEntry->bodyLen); -} - -void -DocumentList::Entry::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocEntry(Timestamp: " << getTimestamp(); - if (isRemoveEntry()) out << ", removed"; - out << ", (h p/s " << _metaEntry->headerPos << "/" << _metaEntry->headerLen - << ", b " << _metaEntry->bodyPos << "/" << _metaEntry->bodyLen << ")"; - if (verbose) { - vespalib::string escaped; - if (_metaEntry->headerLen > 0 && _metaEntry->headerLen < 256) { - vespalib::string header(_start+_metaEntry->headerPos, _metaEntry->headerLen); - out << "\n" << indent << " " - << document::StringUtil::escape(header, escaped); - } - if (_metaEntry->bodyLen > 0 && _metaEntry->bodyLen < 256) { - vespalib::string body(_start + _metaEntry->bodyPos, _metaEntry->bodyLen); - out << "\n" << indent << " " - << document::StringUtil::escape(body, escaped); - } - } - out << ")"; -} - -DocumentList::const_iterator& DocumentList::const_iterator::operator++() -{ - assert(_entry.valid()); - _entry = _entry.next(); - return *this; -} - -DocumentList::const_iterator DocumentList::const_iterator::operator++(int) -{ - const_iterator it(_entry); - operator++(); - return it; -} - -bool DocumentList::const_iterator::operator==(const const_iterator& it) const -{ - return (_entry == it._entry); -} - -bool DocumentList::const_iterator::operator!=(const const_iterator& it) const -{ - return !(_entry == it._entry); -} - - -DocumentList::DocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, uint32_t bufferSize, bool keepexisting) - : _buffer(buffer), - _bufferSize(bufferSize), - _wasted(0), - _freePtr(buffer), - _repo(repo) -{ - init(keepexisting); -} - -void DocumentList::init(bool keepexisting) -{ - if (_buffer == 0) { - assert(_bufferSize == 0); - return; - } - assert(_bufferSize > sizeof(uint32_t)); - if (keepexisting) { - uint32_t min = 0xFFFFFFFF; - for (uint32_t i=0, n=docCount(); i 0 && entry.headerPos < min) { - min = entry.headerPos; - } - if (entry.bodyLen > 0 && entry.bodyPos < min) { - min = entry.bodyPos; - } - } - if (docCount() > 0) { - assert(min < _bufferSize); - _freePtr += min; - } else { - _freePtr += _bufferSize; - } - } else { - docCount() = 0; - _freePtr += _bufferSize; - } - checkConsistency(); -} - -DocumentList::DocumentList(const DocumentList& source, - char* buffer, uint32_t bufferSize) - : _buffer(buffer), - _bufferSize(bufferSize), - _wasted(0), - _freePtr(buffer), - _repo(source._repo) -{ - assert(buffer == 0 ? bufferSize == 0 : bufferSize >= sizeof(uint32_t)); - if (source.size() == 0) { - if (buffer != 0) { - docCount() = 0; - _freePtr += bufferSize; - } - return; - } - - // If we get here we know that source contains documents - uint32_t n = source.docCount(); - uint64_t need = source.spaceNeeded(); - - if (need > bufferSize) { - std::ostringstream ost; - ost << "Cannot create a documentlist of size " << bufferSize - << " bytes containing the data of documentlist of size " - << source.getBufferSize() << ", needing " << need - << " bytes minimum."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } - // If we get here we know that this object has enough space to fit all - uint32_t pos = bufferSize; - for (uint32_t i = 0; i < n; ++i) { - MetaEntry& entry(getMeta(i) = source.getMeta(i)); - - pos -= entry.bodyLen; - memcpy(_buffer + pos, source._buffer + entry.bodyPos, entry.bodyLen); - entry.bodyPos = pos; - - pos -= entry.headerLen; - memcpy(_buffer + pos, source._buffer + entry.headerPos, entry.headerLen); - entry.headerPos = pos; - } - _freePtr = _buffer + pos; - docCount() = n; - checkConsistency(); -} - -DocumentList::~DocumentList() -{ -} - -DocumentList::DocumentList(const DocumentList &rhs) - : document::Printable(rhs), - _buffer(rhs._buffer), - _bufferSize(rhs._bufferSize), - _wasted(rhs._wasted), - _freePtr(rhs._freePtr), - _repo(rhs._repo) -{ - checkConsistency(); -} - -DocumentList& -DocumentList::operator=(const DocumentList &rhs) -{ - document::Printable::operator=(rhs); - _buffer = rhs._buffer; - _bufferSize = rhs._bufferSize; - _wasted = rhs._wasted; - _freePtr = rhs._freePtr; - _repo = _repo; - checkConsistency(); - return *this; -} - -namespace { -struct PosLen { - uint32_t pos; - uint32_t len; - PosLen() : pos(0), len(0) {}; - bool operator< (const PosLen& other) const { return pos < other.pos; } -}; -} // namespace - -void -DocumentList::checkConsistency(bool do_memset) -{ - unsigned long need = spaceNeeded(); - unsigned long free = countFree(); - unsigned long bsiz = getBufferSize(); - if (do_memset || need + free + _wasted != bsiz) { - std::vector blocks; - uint32_t n = docCount(); - for (uint32_t i = 0; i < n; ++i) { - MetaEntry& entry(getMeta(i)); - if (entry.headerLen > 0) { - PosLen pl; - pl.pos = entry.headerPos; - pl.len = entry.headerLen; - blocks.push_back(pl); - } - if (entry.bodyLen > 0) { - PosLen pl; - pl.pos = entry.bodyPos; - pl.len = entry.bodyLen; - blocks.push_back(pl); - } - } - std::sort(blocks.begin(), blocks.end()); - _wasted = 0; - uint32_t prevStart = bsiz; - uint32_t prevLength = 0; - for (uint32_t i = blocks.size(); i-- > 0; ) { - uint32_t curEnd = blocks[i].pos + blocks[i].len; - if (curEnd > prevStart - && (blocks[i].pos != prevStart - || blocks[i].len != prevLength)) - { - LOG(error, "DocumentList has overlapping blocks (block %u: curEnd(%u) > prevStart(%u))", - i, curEnd, prevStart); - std::ostringstream oss; - print(oss, true, ""); - fprintf(stderr, "%s\n", oss.str().c_str()); - assert(!"DocumentList has overlapping blocks!"); - } - if (curEnd < prevStart) { - uint32_t len = prevStart - curEnd; - if (do_memset) { - LOG(debug, "waste %u bytes filled with 0xFF", len); - memset(_buffer + curEnd, 0xff, len); - } - _wasted += len; - } - prevStart = blocks[i].pos; - prevLength = blocks[i].len; - } - if (_freePtr > _buffer + prevStart) { - assert(!"_freePtr inside data block"); - } - if (_freePtr < _buffer + prevStart) { - // may be needed for alignment - uint32_t len = _buffer + prevStart - _freePtr; - if (do_memset) { - LOG(debug, "waste %u bytes before start, filled with 0xFF", len); - memset(_freePtr, 0xFF, len); - } - _wasted += len; - } - if (_freePtr < _buffer + sizeof(uint32_t) + sizeof(MetaEntry)*size()) { - assert(!"_freePtr inside meta block"); - } - } -} - -void -DocumentList::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << "DocumentList(buffer: " << (void*) _buffer << ", size: " - << std::dec << _bufferSize << ", freeptr: " << (void*) _freePtr; - if (_buffer != 0) { - out << ", doccount: " << size(); - if (_bufferSize >= sizeof(MetaEntry) * size() + sizeof(uint32_t)) { - for (uint32_t i=0, n=size(); i _bufferSize || - entry.bodyPos + entry.bodyLen > _bufferSize) { - fprintf(stderr, " Invalid entry. Aborting print.\n"); - return; - } - } - } else { - out << "\n" << indent << " Too small to contain these entries."; - } - } - uint32_t counter = 0; - for (DocumentList::const_iterator it = begin(); it != end(); ++it) { - out << "\n" << indent << " "; - if (++counter > 16) { - out << "..."; - break; - } - it->print(out, verbose, indent + " "); - } - if (verbose && _bufferSize < 256) { - vespalib::string escaped; - vespalib::string tmp(_buffer, _bufferSize); - out << "\n" << indent << " content: " - << document::StringUtil::escape(tmp, escaped); - } - out << ")"; -} - -std::ostream& operator<<(std::ostream& out, const DocumentList::MetaEntry& e) { - e.print(out); - return out; -} - -} // namespace vdslib diff --git a/vdslib/src/vespa/vdslib/container/documentlist.h b/vdslib/src/vespa/vdslib/container/documentlist.h deleted file mode 100644 index ed44d933282..00000000000 --- a/vdslib/src/vespa/vdslib/container/documentlist.h +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class vdslib::DocumentList - * @ingroup messageapi - * - * @brief A utility class for a buffer containing a list of documents. - * - * During visiting and subscriptions, one or more documents need to be sent - * to clients. Both documents added and removed. For performance reasons, we - * might need to bundle multiple entries together in a buffer, and for some - * extreme performance requirements, we might need to use shared memory to send - * this data to a process running on the same computer. - * - * The format is as follows. The first 4 bytes contain the number of meta - * entries in the block. After this, the list of meta entries is. Each entry - * is the memory representation of a MetaEntry object. After this list, a - * generic block with header and body blocks exist. The meta entry points to - * the data they use. Meta entry pointers are indexes starting from the start - * of the docblock. - * - */ - -#pragma once - -#include -#include -#include - -namespace document { - class DocumentUpdate; -} -namespace vdslib { - -class DocumentList : public document::Printable { -public: - - struct MetaEntry { - enum Flag { - REMOVE_ENTRY = 1, - BODY_STRIPPED = 2, - BODY_IN_HEADER = 4, - UPDATE_ENTRY = 8, - COMPRESSED = 16 - }; - Timestamp timestamp; - uint32_t headerPos; - uint32_t headerLen; - uint32_t bodyPos; - uint32_t bodyLen; - uint8_t flags; - uint8_t padding[7]; // In order to align equally in 32bit and 64bit - - MetaEntry(); - void print(std::ostream& out, const std::string& indent = "") const; - }; - - class Entry : public document::Printable { - private: - MetaEntry* _metaEntry; - char* _start; - uint32_t _entry; - const document::DocumentTypeRepo *_repo; - - public: - Entry() : _metaEntry(0), _start(0), _entry(0), _repo(0) {} - Entry(char* start, uint32_t entry, - const document::DocumentTypeRepo &repo); - - Entry(const Entry &rhs) - : document::Printable(rhs), - _metaEntry(rhs._metaEntry), - _start(rhs._start), - _entry(rhs._entry), - _repo(rhs._repo) - { } - - Entry& operator=(const Entry &rhs) - { - document::Printable::operator=(rhs); - _metaEntry = rhs._metaEntry; - _start = rhs._start; - _entry = rhs._entry; - _repo = rhs._repo; - return *this; - } - - Entry next() const; - - /** Entries in iterators gotten from DocumentList::end() are invalid. */ - bool valid() const { return (_start != 0); } - bool isRemoveEntry() const { return _metaEntry->flags & MetaEntry::REMOVE_ENTRY; } - bool isBodyStripped() const { return _metaEntry->flags & MetaEntry::BODY_STRIPPED; } - bool isUpdateEntry() const { return _metaEntry->flags & MetaEntry::UPDATE_ENTRY; } - - uint8_t getFlags() const { return _metaEntry->flags; } - Timestamp getTimestamp() const { return _metaEntry->timestamp; } - void setTimestamp(Timestamp t) const { _metaEntry->timestamp = t; } - - document::DocumentId getDocumentId() const; - document::Document::UP getDocument(const document::DocumentType *anticipatedType = 0) const; - std::unique_ptr getUpdate() const; - public: - bool getUpdate(document::DocumentUpdate&) const; - - typedef std::pair BufferPosition; - /** - * Get the raw header of the document; note that in case - * BODY_IN_HEADER is set, this also includes the body. - */ - const BufferPosition getRawHeader() const; - /** - * Get the raw body of the document; note that in case BODY_IN_HEADER - * is set, this should not be used. - */ - const BufferPosition getRawBody() const; - uint32_t getSerializedSize() const { - return _metaEntry->headerLen + _metaEntry->bodyLen - + sizeof(MetaEntry); - } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - bool operator==(const Entry& e) const { return (_start == e._start && _entry == e._entry); } - }; - - class const_iterator { - Entry _entry; - - public: - typedef std::input_iterator_tag iterator_category; - typedef Entry value_type; - typedef uint32_t difference_type; - typedef const Entry* pointer; - typedef const Entry& reference; - - const_iterator(const Entry& e) : _entry(e) {} - - const Entry& operator*() { return _entry; } - const Entry* operator->() { return &_entry; } - const_iterator& operator++(); // Prefix - const_iterator operator++(int); // Postfix - bool operator==(const const_iterator& it) const; - bool operator!=(const const_iterator& it) const; - }; - - /** - * Create a new documentlist, using the given buffer. - * @param keepexisting If set to true, assume buffer is already filled. - */ - DocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, - uint32_t bufferSize, bool keepexisting = false); - - DocumentList(const DocumentList& source, char* buffer, uint32_t bufferSize); - - virtual ~DocumentList(); - DocumentList& operator=(const DocumentList &rhs); - DocumentList(const DocumentList &rhs); - - /** return number of bytes free space (in the middle of the buffer) */ - uint32_t countFree() const { - return (_freePtr - _buffer) - sizeof(uint32_t) - sizeof(MetaEntry) * size(); - } - - void clear() { docCount() = 0; _freePtr = _buffer + _bufferSize; } - - uint32_t getBufferSize() const { return _bufferSize; } - char* getBuffer() { return _buffer; } - const char* getBuffer() const { return _buffer; } - - const_iterator begin() const { - return const_iterator(Entry((_freePtr < _buffer + _bufferSize ? _buffer : 0), 0, *_repo)); - } - const_iterator end() const { return const_iterator(Entry()); } - uint32_t size() const { return (_buffer == 0 ? 0 : docCount()); } - - /** compute minimum number of bytes needed to hold the current documentlist */ - uint64_t spaceNeeded() const { - uint32_t n = docCount(); - uint64_t need = sizeof(uint32_t); - for (uint32_t i = 0; i < n; ++i) { - const MetaEntry& entry(getMeta(i)); - need += sizeof(MetaEntry) + entry.headerLen + entry.bodyLen; - } - return need; - } - - void checkConsistency(bool do_memset = false); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - const document::DocumentTypeRepo::SP & getTypeRepo() const { return _repo; } - -protected: - char *_buffer; - uint32_t _bufferSize; - uint32_t _wasted; - char *_freePtr; - - uint32_t docCount() const { return *reinterpret_cast(_buffer); } - - uint32_t& docCount() { return *reinterpret_cast(_buffer); } - const MetaEntry& getMeta(uint32_t index) const { - return *reinterpret_cast(_buffer + sizeof(uint32_t) + index * sizeof(MetaEntry)); - } - MetaEntry& getMeta(uint32_t index) { - return *reinterpret_cast(_buffer + sizeof(uint32_t) + index * sizeof(MetaEntry)); - } -private: - void init(bool keepExisting); - document::DocumentTypeRepo::SP _repo; -}; - -std::ostream& operator<<(std::ostream& out, const DocumentList::MetaEntry& e); - -} - diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp deleted file mode 100644 index 940d142db2f..00000000000 --- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.cpp +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "mutabledocumentlist.h" -#include -#include -#include - -using vespalib::compression::CompressionConfig; -using vespalib::nbostream; - -namespace vdslib { -MutableDocumentList::MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, - uint32_t bufferSize, bool keepexisting) - : DocumentList(repo, buffer, bufferSize, keepexisting) -{ -} - -MutableDocumentList::MutableDocumentList(const DocumentList& source, char* buffer, uint32_t bufferSize) - : DocumentList(source, buffer, bufferSize) -{ -} - -bool -MutableDocumentList::addOperationList(const OperationList& opl) -{ - for(uint32_t i=0; i < opl.getOperationList().size(); i++){ - switch(opl.getOperationList()[i].opt) { - case OperationList::Operation::PUT: - if (!addPut(*opl.getOperationList()[i].document, 0)) return false; - break; - case OperationList::Operation::UPDATE: - if (!addUpdate(*opl.getOperationList()[i].documentUpdate, 0)) return false; - break; - case OperationList::Operation::REMOVE: - if (!addRemove(opl.getOperationList()[i].docId,0)) return false; - break; - } - } - checkConsistency(); - return true; -} - -bool -MutableDocumentList::addPut(const document::Document& doc, Timestamp ts, bool addBody) -{ - uint32_t freePos = _freePtr - _buffer; - - nbostream stream; - doc.serializeHeader(stream); - uint32_t headerSize = stream.size(); - - if (addBody) { - doc.serializeBody(stream); - } - - uint32_t totalSize = stream.size(); - uint32_t bodySize = totalSize - headerSize; - - if (countFree() < totalSize + sizeof(MetaEntry)) { - return false; - } - - MetaEntry& entry(getMeta(docCount())); - entry.timestamp = ts; - entry.headerPos = freePos - totalSize; - entry.headerLen = headerSize; - entry.bodyPos = bodySize ? (freePos - bodySize) : 0; - entry.bodyLen = bodySize; - entry.flags = 0; - - if (doc.getType().getFieldsType().getCompressionConfig().type != CompressionConfig::NONE) { - entry.flags |= MetaEntry::COMPRESSED; - } - - document::ByteBuffer buffer(_freePtr - totalSize, totalSize); - - buffer.putBytes(stream.peek(), stream.size()); - if (!addBody) { - entry.flags |= MetaEntry::BODY_STRIPPED; - } - - // Here we're sure we've completed writing doc, and can update internal - // info to commit the write. - _freePtr -= totalSize; - ++docCount(); - checkConsistency(); - return true; -} - -bool -MutableDocumentList::addUpdate(const document::DocumentUpdate& update, Timestamp ts) -{ - vespalib::nbostream os; - update.serialize42(os); - uint32_t updsize = os.size(); - - if (countFree() - sizeof(MetaEntry) < updsize) { - return false; - } - - MetaEntry& entry(getMeta(docCount())); - entry.timestamp = ts; - entry.headerPos = (_freePtr - _buffer - updsize); - entry.headerLen = updsize; - entry.bodyPos = 0; - entry.bodyLen = 0; - entry.flags = 0; - - document::ByteBuffer buffer(_freePtr - updsize, updsize); - - buffer.putBytes(os.c_str(), os.size()); - entry.flags |= MetaEntry::UPDATE_ENTRY; - - // Here we're sure we've completed writing update, and can update internal - // info to commit the write. - _freePtr -= updsize; - ++docCount(); - checkConsistency(); - return true; -} - - -bool -MutableDocumentList::addRemove(const document::DocumentId& docId, Timestamp ts) -{ - // Creating a document by fetching the first document type declared - - const document::DataType *type(document::DataType::DOCUMENT); - if (docId.hasDocType()) { - type = getTypeRepo()->getDocumentType(docId.getDocType()); - } - document::Document doc(*type, docId); - nbostream stream; - doc.serializeHeader(stream); - uint32_t ssize = stream.size(); - if (countFree() < ssize + sizeof(MetaEntry)) { - return false; - } - - MetaEntry& entry(getMeta(docCount())); - entry.timestamp = ts; - entry.headerPos = (_freePtr - _buffer - ssize); - entry.headerLen = ssize; - entry.bodyPos = 0; - entry.bodyLen = 0; - entry.flags = MetaEntry::REMOVE_ENTRY; - document::ByteBuffer buffer(_freePtr - ssize, ssize); - doc.serializeHeader(buffer); - // Here we're sure we've completed writing doc, and can update internal - // info to commit the remove. - _freePtr -= ssize; - ++docCount(); - //printState("Post removing"); - checkConsistency(); - return true; -} - - -bool -MutableDocumentList::addEntry(const DocumentList::Entry& inEntry) -{ - return addEntry(inEntry, inEntry.getTimestamp()); -} - -bool -MutableDocumentList::addEntry(const DocumentList::Entry& inEntry, Timestamp ts) -{ - if (countFree() < inEntry.getSerializedSize()) { - return false; - } - - MetaEntry& entry(getMeta(docCount())); - entry.timestamp = ts; - entry.headerPos = 0; - entry.headerLen = 0; - entry.bodyPos = 0; - entry.bodyLen = 0; - entry.flags = inEntry.getFlags(); - - if ((inEntry.getFlags() & DocumentList::MetaEntry::BODY_IN_HEADER) || - (inEntry.getFlags() & DocumentList::MetaEntry::BODY_STRIPPED)) - { - DocumentList::Entry::BufferPosition header = inEntry.getRawHeader(); - document::ByteBuffer buffer(_freePtr - header.second, header.second); - - entry.headerPos = (_freePtr - _buffer - header.second); - entry.headerLen = header.second; - entry.bodyPos = 0; - entry.bodyLen = 0; - buffer.putBytes(header.first, header.second); - - // Here we're sure we've completed writing doc, and can update internal - // info to commit the write. - _freePtr -= header.second; - } else { - DocumentList::Entry::BufferPosition header = inEntry.getRawHeader(); - DocumentList::Entry::BufferPosition body = inEntry.getRawBody(); - document::ByteBuffer buffer(_freePtr - (header.second + body.second), (header.second + body.second)); - - entry.headerPos = (_freePtr - _buffer - header.second); - entry.headerLen = header.second; - entry.bodyPos = (_freePtr - _buffer - header.second - body.second); - entry.bodyLen = body.second; - buffer.putBytes(body.first, body.second); - buffer.putBytes(header.first, header.second); - - // Here we're sure we've completed writing doc, and can update internal - // info to commit the write. - _freePtr -= (header.second + body.second); - } - - ++docCount(); - checkConsistency(); - return true; -} - -} // vdslib diff --git a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.h b/vdslib/src/vespa/vdslib/container/mutabledocumentlist.h deleted file mode 100644 index 67d50c7dfeb..00000000000 --- a/vdslib/src/vespa/vdslib/container/mutabledocumentlist.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::MutableDocumentList - * @ingroup messageapi - * - * @brief A utility class for a buffer containing a list of documents. - * - * When writing to the docblock, it will typically be filled up from the end - * and forwards, until the free gap between the meta entry list and the data - * it uses, is so small that no more entry fits. - * - * @version $Id$ - */ - -#pragma once - -#include "documentlist.h" -#include "operationlist.h" - -namespace vdslib { - -class MutableDocumentList : public DocumentList { -public: - /** - * Create a new docblock, using the given buffer. - * @param keepexisting If set to true, assume buffer is already filled. - */ - MutableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, uint32_t bufferSize, bool keepexisting = false); - - MutableDocumentList(const DocumentList& source, char* buffer, uint32_t bufferSize); - - // Want to take const pointers to docs here, but can't since we need to - // call serializeHeader/Body.. Grmpf.. - - /** Returns false if no more space in docblock. (Entry not added) */ - bool addPut(const document::Document&, Timestamp = 0, bool addBody = true); - /** Returns false if no more space in docblock. (Entry not added) */ - bool addRemove(const document::DocumentId& docId, Timestamp = 0); - bool addEntry(const DocumentList::Entry& inEntry); - bool addEntry(const DocumentList::Entry& inEntry, Timestamp ts); - bool addOperationList(const OperationList& list); - bool addUpdate(const document::DocumentUpdate&, Timestamp = 0); -}; - -} // vdslib - diff --git a/vdslib/src/vespa/vdslib/container/operationlist.cpp b/vdslib/src/vespa/vdslib/container/operationlist.cpp deleted file mode 100644 index 6d147036c24..00000000000 --- a/vdslib/src/vespa/vdslib/container/operationlist.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "operationlist.h" -#include "documentlist.h" -#include -#include -#include - -namespace vdslib { - -OperationList::Operation::~Operation() {} -OperationList::OperationList() {} -OperationList::~OperationList() {} - -int OperationList::getRequiredBufferSize() const { - int bufferSize = 4; - - vespalib::nbostream stream; - - // Creating a document by fetching the first document type declared - for(uint32_t i=0; i < _operations.size();i++){ - stream.clear(); - switch(_operations[i].opt) { - case OperationList::Operation::REMOVE: - { - document::Document doc(*document::DataType::DOCUMENT, _operations[i].docId); - doc.serializeHeader(stream); - break; - } - case OperationList::Operation::PUT: - { - _operations[i].document->serializeHeader(stream); - _operations[i].document->serializeBody(stream); - break; - } - case OperationList::Operation::UPDATE: - { - _operations[i].documentUpdate->serialize42(stream); - break; - } - } - bufferSize += stream.size(); - } - - return bufferSize + (_operations.size() * sizeof(DocumentList::MetaEntry)); -} - -} // vdslib diff --git a/vdslib/src/vespa/vdslib/container/operationlist.h b/vdslib/src/vespa/vdslib/container/operationlist.h deleted file mode 100644 index 67b948cc105..00000000000 --- a/vdslib/src/vespa/vdslib/container/operationlist.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include - -namespace document { class DocumentUpdate; } - -namespace vdslib { - -class OperationList{ -public: - - struct Operation{ - enum OpType{ - PUT=0, - UPDATE=1, - REMOVE=2 - }; - - Operation(document::DocumentId dId) - : docId(dId), - opt(REMOVE) - { - } - - Operation(std::shared_ptr doc) - : document(doc), - opt(PUT) - { - } - - Operation(std::shared_ptr doc) - : documentUpdate(doc), - opt(UPDATE) - { - } - ~Operation(); - - document::DocumentId docId; - std::shared_ptr document; - std::shared_ptr documentUpdate; - OpType opt; - }; - - void addPut(std::shared_ptr doc) { - _operations.push_back(Operation(doc)); - } - - void addUpdate(std::shared_ptr docUpdate) { - _operations.push_back(Operation(docUpdate)); - } - - void addRemove(document::DocumentId docId) { - _operations.push_back(Operation(docId)); - } - - int getRequiredBufferSize() const; - - const std::vector& getOperationList() const{ - return _operations; - } - - // Deprecated functions. This list used to const cast and ruin source - // objects in copy constructor. Keeping deprecated functions to avoid - // breaking factory now, as they still work fine. Just code bloat. - - void addPut(std::unique_ptr doc) { - _operations.push_back( - Operation(std::shared_ptr(std::move(doc)))); - } - - void addUpdate(std::unique_ptr docUpdate) { - _operations.push_back( - Operation(std::shared_ptr(std::move(docUpdate)))); - } - - OperationList(); - ~OperationList(); -private: - std::vector _operations; - -}; -} - diff --git a/vdslib/src/vespa/vdslib/container/writabledocumentlist.cpp b/vdslib/src/vespa/vdslib/container/writabledocumentlist.cpp deleted file mode 100644 index dd0b582a9d8..00000000000 --- a/vdslib/src/vespa/vdslib/container/writabledocumentlist.cpp +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "writabledocumentlist.h" -#include -#include - -#include - -LOG_SETUP(".vdslib.container.writabledocumentlist"); - -namespace vdslib { -WritableDocumentList::WritableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, uint32_t bufferSize, bool keepexisting) - : MutableDocumentList(repo, buffer, bufferSize, keepexisting) -{ -} - -WritableDocumentList::WritableDocumentList(const DocumentList& source, - char* buffer, - uint32_t bufferSize) - : MutableDocumentList(source, buffer, bufferSize) -{ -} - -char* -WritableDocumentList::prepareMultiput(uint32_t count, uint32_t contentSize) -{ - //printState("Prepare multiput"); - uint32_t freeSpace = countFree(); - uint32_t metaSpace = count * sizeof(MetaEntry); - if (freeSpace < metaSpace || freeSpace - metaSpace < contentSize) { - return 0; - } - return _freePtr - contentSize; -} - -bool -WritableDocumentList::commitMultiput(const std::vector& meta, char* contentPtr) -{ - //printState("Pre commit multiput"); - uint32_t diff = contentPtr - _buffer; - uint32_t oldDocCount = docCount(); - uint32_t highPos = 0; - uint32_t nlowPos = (_freePtr - _buffer); - for (uint32_t i=0; i highPos) { - LOG(debug, "filling %u bytes with 0xFF", (freePos - highPos)); - memset(_buffer + highPos, 0xff, (freePos - highPos)); - _wasted += (freePos - highPos); - } - - // Here we should have written all. Commit alterations. - _freePtr = contentPtr; - - // check for waste before written blocks - freePos = _freePtr - _buffer; - if (freePos < nlowPos) { - LOG(debug, "filling %u bytes with 0xFF", (nlowPos - freePos)); - memset(_buffer + freePos, 0xff, (nlowPos - freePos)); - _wasted += (nlowPos - freePos); - } - if (freePos > nlowPos) { - vespalib::string msg = vespalib::make_string( - "bad multiput, wrote at %p (before allocated %p)", - _buffer + nlowPos, contentPtr); - throw vespalib::IllegalArgumentException(msg, VESPA_STRLOC); - } - - docCount() += meta.size(); - //printState("Post commit multiput"); - - checkConsistency(); - - return true; -} - -} // vdslib diff --git a/vdslib/src/vespa/vdslib/container/writabledocumentlist.h b/vdslib/src/vespa/vdslib/container/writabledocumentlist.h deleted file mode 100644 index 94e9ccb9a7b..00000000000 --- a/vdslib/src/vespa/vdslib/container/writabledocumentlist.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::api::WritableDocumentList - * @ingroup messageapi - * - * @brief A utility class for a buffer containing a list of documents. - * - * When writing to the docblock, it will typically be filled up from the end - * and forwards, until the free gap between the meta entry list and the data - * it uses, is so small that no more entry fits. - * - * @version $Id$ - */ - -#pragma once - -#include - -namespace vdslib { - -class WritableDocumentList : public MutableDocumentList { -public: - /** - * Create a new docblock, using the given buffer. - * @param keepexisting If set to true, assume buffer is already filled. - */ - WritableDocumentList(const document::DocumentTypeRepo::SP & repo, char* buffer, uint32_t bufferSize, bool keepexisting = false); - - WritableDocumentList(const DocumentList& source, char* buffer, uint32_t bufferSize); - /** - * Prepare a multiput/remove. - * Returns a char* to the part of the buffer you can write contentSize - * data to. (Both header and body data), 0 if not enough space. - */ - char* prepareMultiput(uint32_t docCount, uint32_t contentSize); - /** - * Commit a multiput/remove. Call this after you've written all content to - * contentPos gotten from prepareMultiput(). Give relative positions from - * contentPos in meta entries. - */ - bool commitMultiput(const std::vector& meta, char* contentPos); - -}; - -} // vdslib - -- cgit v1.2.3