diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-01-21 20:57:49 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-01-21 20:57:49 +0100 |
| commit | c9db06db7b01a2da9228c38783fd9f95c4fc7bba (patch) | |
| tree | 715ee5ac9cc3f925e1a72d6ba7db54ba2510cc79 /document | |
| parent | 85b84fc8e4db0976a6e3e9b97b2e4822e8764ff4 (diff) | |
| parent | aa0bfb11d05381a555c197a2ab26ee117c0758e6 (diff) | |
Merge pull request #11822 from vespa-engine/balder/reduce-bytebuffer-exposure
Balder/reduce bytebuffer exposure
Diffstat (limited to 'document')
30 files changed, 392 insertions, 2279 deletions
diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp index 014884e44cd..0bdf51194fb 100644 --- a/document/src/tests/arrayfieldvaluetest.cpp +++ b/document/src/tests/arrayfieldvaluetest.cpp @@ -15,9 +15,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -53,40 +52,39 @@ TEST(ArrayFieldValueTest, testArray) EXPECT_EQ(IntFieldValue(3), (IntFieldValue&) value[2]); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream stream(value.serialize()); ArrayFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(stream, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By index - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.remove(1); EXPECT_TRUE(!value2.contains(IntFieldValue(2))); EXPECT_EQ(size_t(2), value2.size()); // By value - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // By value with multiple present - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.add(IntFieldValue(1)); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index d75a16ea1a6..7735ccdbb1d 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocman.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/document/datatype/annotationreferencedatatype.h> #include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/repo/configbuilder.h> @@ -9,6 +8,9 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> @@ -28,10 +30,14 @@ using namespace fieldvalue; TEST(DocumentTest, testSizeOf) { + EXPECT_EQ(24u, sizeof(std::vector<char>)); + EXPECT_EQ(24u, sizeof(vespalib::alloc::Alloc)); + EXPECT_EQ(40u, sizeof(ByteBuffer)); + EXPECT_EQ(32u, sizeof(vespalib::GrowableByteBuffer)); EXPECT_EQ(88ul, sizeof(IdString)); EXPECT_EQ(104ul, sizeof(DocumentId)); - EXPECT_EQ(208ul, sizeof(Document)); - EXPECT_EQ(72ul, sizeof(StructFieldValue)); + EXPECT_EQ(200ul, sizeof(Document)); + EXPECT_EQ(64ul, sizeof(StructFieldValue)); EXPECT_EQ(24ul, sizeof(StructuredFieldValue)); EXPECT_EQ(64ul, sizeof(SerializableArray)); } @@ -63,7 +69,7 @@ TEST(DocumentTest, testFieldPath) class Handler : public fieldvalue::IteratorHandler { public: Handler(); - ~Handler(); + ~Handler() override; const std::string & getResult() const { return _result; } private: void onPrimitive(uint32_t, const Content&) override { @@ -386,13 +392,13 @@ TEST(DocumentTest, testSimpleUsage) EXPECT_EQ(1, value.getValue(intF)->getAsInt()); EXPECT_EQ(2, value.getValue(longF)->getAsInt()); - // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + // Serialize & equality + nbostream buffer; + value.serialize(buffer); Document value2(*repo.getDocumentType("test"), DocumentId("id::test:n=3:foo")); EXPECT_TRUE(value != value2); - value2.deserialize(repo, *buffer); + value2.deserialize(repo, buffer); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); EXPECT_EQ(DocumentId("id:ns:test::1"), value2.getId()); @@ -400,15 +406,15 @@ TEST(DocumentTest, testSimpleUsage) // Various ways of removing { // By value - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); @@ -561,29 +567,31 @@ TEST(DocumentTest, testReadSerializedFile) int fd = open(TEST_PATH("data/serializejava.dat").c_str(), O_RDONLY); size_t len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != (ssize_t)len) { + if (read(fd, buf.get(), len) != (ssize_t)len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); verifyJavaDocument(doc2); - EXPECT_EQ(len, buf2->getPos()); - EXPECT_TRUE(memcmp(buf2->getBuffer(), buf.getBuffer(), buf2->getPos()) == 0); + EXPECT_TRUE(buf2.empty()); + buf2.rp(0); + EXPECT_EQ(len, buf2.size()); + EXPECT_TRUE(memcmp(buf2.peek(), buf.get(), buf2.size()) == 0); doc2.setValue("stringfield", StringFieldValue("hei")); - std::unique_ptr<ByteBuffer> buf3 = doc2.serialize(); - EXPECT_TRUE(len != buf3->getPos()); + nbostream buf3 = doc2.serialize(); + EXPECT_TRUE(len != buf3.size()); } TEST(DocumentTest, testReadSerializedFileCompressed) @@ -595,14 +603,15 @@ TEST(DocumentTest, testReadSerializedFileCompressed) int fd = open(TEST_PATH("data/serializejava-compressed.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); } @@ -706,26 +715,25 @@ TEST(DocumentTest,testReadSerializedAllVersions) // you can copy this current to new test for new version) { //doc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0)); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_EQ(buf->getLength(), buf->getPos()); + nbostream buf = doc.serialize(); int fd = open(TEST_PATH("data/document-cpp-currentversion-uncompressed.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); } { CompressionConfig oldCfg(doc.getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_TRUE(buf->getPos() <= buf->getLength()); + nbostream buf = doc.serialize(); + EXPECT_TRUE(buf.size() <= buf.capacity()); int fd = open(TEST_PATH("data/document-cpp-currentversion-lz4-9.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(oldCfg); } @@ -745,14 +753,15 @@ TEST(DocumentTest,testReadSerializedAllVersions) } int fd = open(tests[i]._dataFile.c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { - throw vespalib::Exception("read failed"); - } + if (read(fd, buf.get(), len) != len) { + throw vespalib::Exception("read failed"); + } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); IntFieldValue intVal; EXPECT_TRUE(doc.getValue(doc.getField("intfield"), intVal)); @@ -805,27 +814,14 @@ TEST(DocumentTest,testReadSerializedAllVersions) EXPECT_EQ(199, wset.get(StringFieldValue("Weighted 1"))); // Check that serialization doesn't cause any problems. - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); } } size_t getSerializedSize(const Document &doc) { - return doc.serialize()->getLength(); -} - -size_t getSerializedSizeHeader(const Document &doc) { - nbostream stream; - doc.serializeHeader(stream); - return stream.size(); -} - -size_t getSerializedSizeBody(const Document &doc) { - nbostream stream; - doc.serializeBody(stream); - return stream.size(); + return doc.serialize().size(); } TEST(DocumentTest, testGenerateSerializedFile) @@ -864,30 +860,21 @@ TEST(DocumentTest, testGenerateSerializedFile) map.put(StringFieldValue("foo2"), StringFieldValue("bar2")); doc.setValue("mapfield", map); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); + nbostream buf = doc.serialize(); const std::string serializedDir = TEST_PATH("../test/document/"); int fd = open((serializedDir + "/serializecpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.peek(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); - ByteBuffer hBuf(getSerializedSizeHeader(doc)); + vespalib::nbostream hBuf; doc.serializeHeader(hBuf); fd = open((serializedDir + "/serializecppsplit_header.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, hBuf.getBuffer(), hBuf.getPos()) != (ssize_t)hBuf.getPos()) { - throw vespalib::Exception("write failed"); - } - close(fd); - - ByteBuffer bBuf(getSerializedSizeBody(doc)); - doc.serializeBody(bBuf); - fd = open((serializedDir+ "/serializecppsplit_body.dat").c_str(), - O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, bBuf.getBuffer(), bBuf.getPos()) != (ssize_t)bBuf.getPos()) { + if (write(fd, hBuf.peek(), hBuf.size()) != (ssize_t)hBuf.size()) { throw vespalib::Exception("write failed"); } close(fd); @@ -895,62 +882,30 @@ TEST(DocumentTest, testGenerateSerializedFile) CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - ByteBuffer lz4buf(getSerializedSize(doc)); - - doc.serialize(lz4buf); - lz4buf.flip(); + nbostream lz4buf = doc.serialize(); fd = open((serializedDir + "/serializecpp-lz4-level9.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, lz4buf.getBufferAtPos(), lz4buf.getRemaining()) != (ssize_t)lz4buf.getRemaining()) { + if (write(fd, lz4buf.data(), lz4buf.size()) != (ssize_t)lz4buf.size()) { throw vespalib::Exception("write failed"); } close(fd); } -TEST(DocumentTest, testGetURIFromSerialized) -{ - TestDocRepo test_repo; - Document doc(*test_repo.getDocumentType("testdoctype1"), DocumentId("id:ns:testdoctype1::1")); - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - EXPECT_EQ( - vespalib::string("id:ns:testdoctype1::1"), - Document::getIdFromSerialized(*serialized).toString()); - - EXPECT_EQ(vespalib::string("testdoctype1"), - Document::getDocTypeFromSerialized( - test_repo.getTypeRepo(), - *serialized)->getName()); - } - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - Document doc2(test_repo.getTypeRepo(), *serialized, false, NULL); - EXPECT_EQ(vespalib::string("id:ns:testdoctype1::1"), doc2.getId().toString()); - EXPECT_EQ(vespalib::string("testdoctype1"), doc2.getType().getName()); - } -} - TEST(DocumentTest, testBogusserialize) { TestDocRepo test_repo; try { - auto buf = std::make_unique<ByteBuffer>("aoifjweprjwoejr203r+2+4r823++!",100); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("aoifjweprjwoejr203r+2+4r823++!",100); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing bogus data"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Unrecognized serialization version")); } try { - auto buf = std::make_unique<ByteBuffer>("",0); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("",0); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing empty buffer"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Buffer out of bounds")); @@ -967,24 +922,23 @@ TEST(DocumentTest, testCRC32) uint32_t crc = doc.calculateChecksum(); EXPECT_EQ(3987392271u, crc); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); + nbostream buf = doc.serialize(); int pos = 30; // Corrupt serialization. - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; // Create document. Byte corrupted above is in data area and // shouldn't fail deserialization. try { - Document doc2(test_repo.getTypeRepo(), *buf); - buf->setPos(0); + Document doc2(test_repo.getTypeRepo(), buf); + buf.rp(0); EXPECT_TRUE(crc != doc2.calculateChecksum()); } catch (document::DeserializeException& e) { EXPECT_TRUE(false); } // Return original value and retry - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; /// \todo TODO (was warning): Cannot test for in memory representation altered, as there is no syntax for getting internal refs to data from document. Add test when this is added. } @@ -1001,13 +955,12 @@ TEST(DocumentTest, testHasChanged) // Still changed after setting a value of course. EXPECT_TRUE(doc.hasChanged()); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); - + nbostream buf; + doc.serialize(buf); // Setting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); EXPECT_TRUE(!doc2.hasChanged()); doc2.set("headerval", 13); @@ -1015,16 +968,16 @@ TEST(DocumentTest, testHasChanged) } // Overwriting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.set("hstringval", "bla bla bla bla bla"); EXPECT_TRUE(doc2.hasChanged()); } // Clearing value tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.clear(); EXPECT_TRUE(doc2.hasChanged()); @@ -1032,35 +985,6 @@ TEST(DocumentTest, testHasChanged) // Add more tests here when we allow non-const refs to internals } -TEST(DocumentTest, testSplitSerialization) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - doc->set("headerval", 50); - - ByteBuffer buf(getSerializedSizeHeader(*doc)); - doc->serializeHeader(buf); - buf.flip(); - - ByteBuffer buf2(getSerializedSizeBody(*doc)); - doc->serializeBody(buf2); - buf2.flip(); - - EXPECT_EQ(size_t(65), buf.getLength()); - EXPECT_EQ(size_t(73), buf2.getLength()); - - Document headerDoc(testDocMan.getTypeRepo(), buf); - EXPECT_TRUE(headerDoc.hasValue("headerval")); - EXPECT_TRUE(!headerDoc.hasValue("content")); - - buf.setPos(0); - Document fullDoc(testDocMan.getTypeRepo(), buf, buf2); - EXPECT_TRUE(fullDoc.hasValue("headerval")); - EXPECT_TRUE(fullDoc.hasValue("content")); - - EXPECT_EQ(*doc, fullDoc); -} - TEST(DocumentTest, testSliceSerialize) { // Test that document doesn't need its own bytebuffer, such that we @@ -1076,17 +1000,15 @@ TEST(DocumentTest, testSliceSerialize) val.add(RawFieldValue("hei der", 7)); doc2->setValue(doc2->getField("rawarray"), val); - ByteBuffer buf(getSerializedSize(*doc) + getSerializedSize(*doc2)); - doc->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + nbostream buf = doc->serialize(); + EXPECT_EQ(getSerializedSize(*doc), buf.size()); doc2->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); - buf.flip(); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.size()); Document doc3(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc), buf.rp()); Document doc4(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.rp()); EXPECT_EQ(*doc, doc3); EXPECT_EQ(*doc2, doc4); @@ -1102,21 +1024,19 @@ TEST(DocumentTest, testCompression) doc->setValue("hstringval", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc->serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed = doc->serialize(); CompressionConfig oldCfg(doc->getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf_lz4 = doc->serialize(); - buf_lz4->flip(); + nbostream buf_lz4 = doc->serialize(); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(oldCfg); - EXPECT_TRUE(buf_lz4->getRemaining() < buf_uncompressed->getRemaining()); + EXPECT_TRUE(buf_lz4.size() < buf_uncompressed.size()); - Document doc_lz4(testDocMan.getTypeRepo(), *buf_lz4); + Document doc_lz4(testDocMan.getTypeRepo(), buf_lz4); EXPECT_EQ(*doc, doc_lz4); } @@ -1136,10 +1056,10 @@ TEST(DocumentTest, testCompressionConfigured) for (int i = 0; i < 8; ++i) { bigString += bigString; } doc_uncompressed.setValue("stringfield", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc_uncompressed.serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed; + doc_uncompressed.serialize(buf_uncompressed); - size_t uncompressedSize = buf_uncompressed->getRemaining(); + size_t uncompressedSize = buf_uncompressed.size(); DocumenttypesConfigBuilderHelper builder2; builder2.document(43, "serializetest", @@ -1151,22 +1071,22 @@ TEST(DocumentTest, testCompressionConfigured) 9, 99, 0)); DocumentTypeRepo repo2(builder2.config()); - Document doc(repo2, *buf_uncompressed); + Document doc(repo2, buf_uncompressed); - std::unique_ptr<ByteBuffer> buf_compressed = doc.serialize(); - buf_compressed->flip(); - size_t compressedSize = buf_compressed->getRemaining(); + nbostream buf_compressed; + doc.serialize(buf_compressed); + size_t compressedSize = buf_compressed.size(); EXPECT_TRUE(compressedSize < uncompressedSize); - Document doc2(repo2, *buf_compressed); + Document doc2(repo2, buf_compressed); - std::unique_ptr<ByteBuffer> buf_compressed2 = doc2.serialize(); - buf_compressed2->flip(); + nbostream buf_compressed2; + doc2.serialize(buf_compressed2); - EXPECT_EQ(compressedSize, buf_compressed2->getRemaining()); + EXPECT_EQ(compressedSize, buf_compressed2.size()); - Document doc3(repo2, *buf_compressed2); + Document doc3(repo2, buf_compressed2); EXPECT_EQ(doc2, doc_uncompressed); EXPECT_EQ(doc2, doc3); @@ -1198,59 +1118,31 @@ TEST(DocumentTest, testUnknownEntries) doc1.setValue(field3, IntFieldValue(3)); doc1.setValue(field4, IntFieldValue(4)); - uint32_t headerLen = getSerializedSizeHeader(doc1); - document::ByteBuffer header(headerLen); - doc1.serializeHeader(header); - header.flip(); - - uint32_t bodyLen = getSerializedSizeBody(doc1); - document::ByteBuffer body(bodyLen); - doc1.serializeBody(body); - body.flip(); - - uint32_t totalLen = getSerializedSize(doc1); - document::ByteBuffer total(totalLen); - doc1.serialize(total); - total.flip(); + vespalib::nbostream os; + doc1.serialize(os); Document doc2; - doc2.deserialize(repo, total); - - Document doc3; - doc3.deserializeHeader(repo, header); - doc3.deserializeBody(repo, body); + doc2.deserialize(repo, os); EXPECT_EQ(std::string( "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" "<int3>3</int3>\n" "<int4>4</int4>\n" "</document>"), doc2.toXml()); - EXPECT_EQ(std::string( - "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" - "<int3>3</int3>\n" - "<int4>4</int4>\n" - "</document>"), doc3.toXml()); EXPECT_EQ(3, doc2.getValue(field3)->getAsInt()); EXPECT_EQ(4, doc2.getValue(field4)->getAsInt()); - EXPECT_EQ(3, doc3.getValue(field3)->getAsInt()); - EXPECT_EQ(4, doc3.getValue(field4)->getAsInt()); // The fields are actually accessible as long as you ask with field of // correct type. EXPECT_TRUE(doc2.hasValue(field1)); EXPECT_TRUE(doc2.hasValue(field2)); - EXPECT_TRUE(doc3.hasValue(field1)); - EXPECT_TRUE(doc3.hasValue(field2)); EXPECT_EQ(1, doc2.getValue(field1)->getAsInt()); EXPECT_EQ(2, doc2.getValue(field2)->getAsInt()); - EXPECT_EQ(1, doc3.getValue(field1)->getAsInt()); - EXPECT_EQ(2, doc3.getValue(field2)->getAsInt()); EXPECT_EQ(size_t(2), doc2.getSetFieldCount()); - EXPECT_EQ(size_t(2), doc3.getSetFieldCount()); } TEST(DocumentTest, testAnnotationDeserialization) @@ -1280,14 +1172,15 @@ TEST(DocumentTest, testAnnotationDeserialization) int fd = open(TEST_PATH("data/serializejavawithannotations.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream1(buf.get(), len); + Document doc(repo, stream1); StringFieldValue strVal; EXPECT_TRUE(doc.getValue(doc.getField("story"), strVal)); @@ -1326,14 +1219,6 @@ TEST(DocumentTest, testAnnotationDeserialization) EXPECT_EQ((int64_t)2384LL, longVal.getAsLong()); } -TEST(DocumentTest, testGetSerializedSize) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - - EXPECT_EQ(getSerializedSize(*doc), doc->getSerializedSize()); -} - TEST(DocumentTest, testDeserializeMultiple) { TestDocRepo testDocRepo; diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 47a529adfc8..5543cb48ba4 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -43,14 +43,13 @@ namespace document { namespace { -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } nbostream serialize(const ValueUpdate & update) @@ -83,25 +82,25 @@ void testRoundtripSerialize(const UpdateType& update, const DataType &type) { } void -writeBufferToFile(const ByteBuffer &buf, const vespalib::string &fileName) +writeBufferToFile(const nbostream &buf, const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::out | std::ios::binary); - file.write(buf.getBuffer(), buf.getPos()); + file.write(buf.data(), buf.size()); assert(file.good()); file.close(); } -ByteBuffer::UP +nbostream readBufferFromFile(const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate); auto size = file.tellg(); - auto result = std::make_unique<ByteBuffer>(size); file.seekg(0); - file.read(result->getBuffer(), size); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); + file.read(static_cast<char *>(buf.get()), size); assert(file.good()); file.close(); - return result; + return nbostream(std::move(buf), size); } } @@ -132,9 +131,8 @@ TEST(DocumentUpdateTest, testSimpleUsage) // Test that a document update can be serialized DocumentUpdate docUpdate(repo, *docType, DocumentId("id:ns:test::1")); docUpdate.addUpdate(fieldUpdateCopy); - ByteBuffer::UP docBuf = serializeHEAD(docUpdate); - docBuf->flip(); - auto docUpdateCopy(DocumentUpdate::createHEAD(repo, nbostream(docBuf->getBufferAtPos(), docBuf->getRemaining()))); + nbostream docBuf = serializeHEAD(docUpdate); + auto docUpdateCopy(DocumentUpdate::createHEAD(repo, docBuf)); // Create a test document Document doc(*docType, DocumentId("id:ns:test::1")); @@ -236,7 +234,7 @@ TEST(DocumentUpdateTest, testUpdateArray) // Create a document. TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); // Assign array field. ArrayFieldValue myarray(doc->getType().getField("tags").getDataType()); @@ -459,8 +457,7 @@ TEST(DocumentUpdateTest, testReadSerializedFile) const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); - auto buf = readBufferFromFile("data/serializeupdatejava.dat"); - nbostream is(buf->getBufferAtPos(), buf->getRemaining()); + auto is = readBufferFromFile("data/serializeupdatejava.dat"); DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, is)); DocumentUpdate& upd(*updp); @@ -539,8 +536,8 @@ TEST(DocumentUpdateTest, testGenerateSerializedFile) ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 2))) .addUpdate(MapValueUpdate(StringFieldValue("foo"), ArithmeticValueUpdate(ArithmeticValueUpdate::Mul, 2)))); - ByteBuffer::UP buf(serializeHEAD(upd)); - writeBufferToFile(*buf, "data/serializeupdatecpp.dat"); + nbostream buf(serializeHEAD(upd)); + writeBufferToFile(buf, "data/serializeupdatecpp.dat"); } @@ -549,7 +546,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) // Create a test document TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("headerval")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); // Assign a float value to an int field. DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); @@ -561,7 +558,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) update.applyTo(*doc); // Verify that the field is NOT set in the document. - EXPECT_EQ((document::FieldValue*)NULL, + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); } @@ -569,7 +566,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoParams) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update.addUpdate(FieldUpdate(doc->getField("tags")).addUpdate(AssignValueUpdate())); @@ -1110,13 +1107,12 @@ struct TensorUpdateSerializeFixture { } void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) { - ByteBuffer::UP buf = serializeHEAD(update); - writeBufferToFile(*buf, fileName); + nbostream buf = serializeHEAD(update); + writeBufferToFile(buf, fileName); } DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) { - auto buf = readBufferFromFile(fileName); - nbostream stream(buf->getBufferAtPos(), buf->getRemaining()); + auto stream = readBufferFromFile(fileName); return DocumentUpdate::createHEAD(*repo, stream); } @@ -1196,10 +1192,9 @@ TEST(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeseriali { CreateIfNonExistentFixture f; - ByteBuffer::UP buf(serializeHEAD(*f.update)); - buf->flip(); + nbostream buf(serializeHEAD(*f.update)); - DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), *buf); + DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), buf); EXPECT_EQ(*f.update, *deserialized); EXPECT_TRUE(deserialized->getCreateIfNonExistent()); } @@ -1231,9 +1226,8 @@ TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) ArrayUpdateFixture f; auto buffer = serializeHEAD(*f.update); - buffer->flip(); - auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), *buffer); + auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), buffer); EXPECT_EQ(*f.update, *deserialized); } diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 82443f13716..213ac2e5432 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -19,6 +19,7 @@ #include <gtest/gtest.h> using vespalib::Identifiable; +using vespalib::nbostream; using namespace document::config_builder; namespace document { @@ -133,23 +134,21 @@ createTestDocument(const DocumentTypeRepo &repo) return doc; } -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { vespalib::nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { try{ - ByteBuffer::UP bb(serializeHEAD(a)); - bb->flip(); - DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb)); + auto bb(serializeHEAD(a)); + DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, bb)); - EXPECT_EQ(size_t(0), bb->getRemaining()); + EXPECT_EQ(size_t(0), bb.size()); EXPECT_EQ(a.getId().toString(), b->getId().toString()); EXPECT_EQ(a.getUpdates().size(), b->getUpdates().size()); for (size_t i(0); i < a.getUpdates().size(); i++) { @@ -157,8 +156,7 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { const FieldUpdate & ub = b->getUpdates()[i]; EXPECT_EQ(&ua.getField(), &ub.getField()); - EXPECT_EQ(ua.getUpdates().size(), - ub.getUpdates().size()); + EXPECT_EQ(ua.getUpdates().size(), ub.getUpdates().size()); for (size_t j(0); j < ua.getUpdates().size(); j++) { EXPECT_EQ(ua.getUpdates()[j]->getType(), ub.getUpdates()[j]->getType()); } @@ -1073,14 +1071,14 @@ TEST_F(FieldPathUpdateTestCase, testReadSerializedFile) int fd = open(TEST_PATH("data/serialize-fieldpathupdate-java.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, buf)); + DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, nbostream(std::move(buf), len))); DocumentUpdate& upd(*updp); DocumentUpdate::UP compare(createDocumentUpdateForSerialization(repo)); @@ -1094,11 +1092,11 @@ TEST_F(FieldPathUpdateTestCase, testGenerateSerializedFile) // Tests nothing, only generates a file for java test DocumentUpdate::UP upd(createDocumentUpdateForSerialization(repo)); - ByteBuffer::UP buf(serializeHEAD(*upd)); + nbostream buf(serializeHEAD(*upd)); int fd = open(TEST_PATH("data/serialize-fieldpathupdate-cpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.data(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp index 58592e50cfe..8a5daf05f05 100644 --- a/document/src/tests/primitivefieldvaluetest.cpp +++ b/document/src/tests/primitivefieldvaluetest.cpp @@ -14,9 +14,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -89,20 +88,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Serialization Type t; - std::unique_ptr<ByteBuffer> buf(smallest.serialize()); - buf->flip(); - deserialize(*buf, t); + nbostream buf(smallest.serialize()); + deserialize(buf, t); EXPECT_EQ(smallest, t); buf = medium1.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(medium1, t); EXPECT_EQ(medium2, t); buf = largest.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(largest, t); // Assignment @@ -160,20 +156,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Test that a just deserialized value can be serialized again // (literals have lazy deserialization so behaves diff then value = "foo"; - std::unique_ptr<ByteBuffer> buf(value.serialize()); - buf->flip(); + nbostream buf(value.serialize()); Literal value2("Other"); - deserialize(*buf, value2); + deserialize(buf, value2); buf = value2.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(value, value2); // Verify that get value ref gives us ref within original bytebuffer // (operator== use above should not modify this) buf = value.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(size_t(3), value2.getValueRef().size()); // Zero termination diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 7feea4e51d2..f680b3e6cff 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -912,7 +912,7 @@ void DeserializedTensorDoc::setup(const DocumentTypeRepo &docTypeRepo, const vespalib::nbostream &blob) { vespalib::nbostream wrapStream(blob.peek(), blob.size()); - _doc = std::make_unique<Document>(docTypeRepo, wrapStream, nullptr); + _doc = std::make_unique<Document>(docTypeRepo, wrapStream); _fieldValue = _doc->getValue(tensor_field_name); } diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp index 76cf065a36a..9cf4b38be91 100644 --- a/document/src/tests/structfieldvaluetest.cpp +++ b/document/src/tests/structfieldvaluetest.cpp @@ -28,10 +28,9 @@ protected: namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value, const FixedTypeRepo &repo) +void deserialize(nbostream & stream, T &value, const FixedTypeRepo &repo) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); } @@ -61,13 +60,11 @@ TEST_F(StructFieldValueTest, testEmptyStruct) StructFieldValue value(type); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value == value2); } @@ -101,14 +98,12 @@ TEST_F(StructFieldValueTest, testStruct) EXPECT_EQ(2, value.getValue(longF)->getAsInt()); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); @@ -116,15 +111,15 @@ TEST_F(StructFieldValueTest, testStruct) // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); diff --git a/document/src/tests/testbytebuffer.cpp b/document/src/tests/testbytebuffer.cpp index 17807fb4ff5..b4ad558b4ad 100644 --- a/document/src/tests/testbytebuffer.cpp +++ b/document/src/tests/testbytebuffer.cpp @@ -3,12 +3,14 @@ #include <vespa/document/util/stringutil.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/document/fieldvalue/serializablearray.h> -#include <iostream> -#include <vespa/vespalib/util/macro.h> #include <vespa/document/util/bufferexceptions.h> +#include <vespa/vespalib/util/macro.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <gtest/gtest.h> + using namespace document; +using vespalib::GrowableByteBuffer; namespace { @@ -22,111 +24,28 @@ void assign(S &lhs, const S &rhs) TEST(ByteBuffer_Test, test_constructors) { - ByteBuffer* simple=new ByteBuffer(); - delete simple; - - ByteBuffer* less_simple=new ByteBuffer("hei",3); - EXPECT_TRUE(strcmp(less_simple->getBufferAtPos(),"hei")==0); - delete less_simple; -} - -TEST(ByteBuffer_Test, test_assignment_operator) -{ - try { - ByteBuffer b1; - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); - - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - int test = 0; - b2.flip(); - b2.getInt(test); - EXPECT_EQ(1,test); - b2.getInt(test); - EXPECT_EQ(2,test); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - // Test Selfassignment == no change - // - assign(b2, b2); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - ByteBuffer b3; - // Empty - b2 = b3; - - EXPECT_EQ((size_t) 0,b2.getPos()); - EXPECT_EQ((size_t) 0,b2.getLength()); - EXPECT_EQ((size_t) 0,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } + ByteBuffer less_simple("hei",3); + EXPECT_TRUE(strcmp(less_simple.getBufferAtPos(),"hei")==0); } TEST(ByteBuffer_Test, test_copy_constructor) { try { - // Empty buffer first - ByteBuffer b1; - ByteBuffer b2(b1); - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); + GrowableByteBuffer gb(100); + gb.putInt(1); + gb.putInt(2); + ByteBuffer b1(gb.getBuffer(), gb.position()); ByteBuffer b2(b1); EXPECT_EQ(b1.getPos(),b2.getPos()); EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); int test = 0; - b2.flip(); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(1,test); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(2,test); } catch (std::exception &e) { @@ -134,327 +53,6 @@ TEST(ByteBuffer_Test, test_copy_constructor) } } -TEST(ByteBuffer_Test, test_slice) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer* slice = new ByteBuffer; - slice->sliceFrom(*newBuf, 0,newBuf->getLength()); - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice->getBufferAtPos(),"hei der")==0); - - ByteBuffer* slice2 = new ByteBuffer; - slice2->sliceFrom(*slice, 4, slice->getLength()); - delete slice; - slice = NULL; - - EXPECT_TRUE(strcmp(slice2->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2->getBuffer(),"hei der")==0); - delete slice2; - slice2 = NULL; - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - ByteBuffer* slice3=new ByteBuffer; - ByteBuffer* slice4=new ByteBuffer; - - slice3->sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice4->sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice3->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice3; - slice3 = NULL; - - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice4; - slice4 = NULL; -} - -TEST(ByteBuffer_Test, test_slice2) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer slice; - slice.sliceFrom(*newBuf, 0, newBuf->getLength()); - - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"hei der")==0); - - ByteBuffer slice2; - slice2.sliceFrom(slice, 4, slice.getLength()); - - EXPECT_TRUE(strcmp(slice2.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - - slice.sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice2.sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); -} - - -TEST(ByteBuffer_Test, test_putGetFlip) -{ - ByteBuffer* newBuf=new ByteBuffer(100); - - try { - newBuf->putInt(10); - int test; - newBuf->flip(); - - newBuf->getInt(test); - EXPECT_TRUE(test==10); - - newBuf->clear(); - newBuf->putDouble(3.35); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==sizeof(double)); - double test2; - newBuf->getDouble(test2); - EXPECT_TRUE(test2==3.35); - - newBuf->clear(); - newBuf->putBytes("heisann",8); - newBuf->putInt(4); - EXPECT_TRUE(newBuf->getPos()==12); - EXPECT_TRUE(newBuf->getLength()==100); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==12); - - char testStr[12]; - newBuf->getBytes(testStr, 8); - EXPECT_TRUE(strcmp(testStr,"heisann")==0); - newBuf->getInt(test); - EXPECT_TRUE(test==4); - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - delete newBuf; -} - - -TEST(ByteBuffer_Test, test_NumberEncodings) -{ - ByteBuffer* buf=new ByteBuffer(1024); - - // Check 0 - buf->putInt1_2_4Bytes(124); - buf->putInt2_4_8Bytes(124); - buf->putInt1_4Bytes(124); - // Check 1 - buf->putInt1_2_4Bytes(127); - buf->putInt2_4_8Bytes(127); - buf->putInt1_4Bytes(127); - // Check 2 - buf->putInt1_2_4Bytes(128); - buf->putInt2_4_8Bytes(128); - buf->putInt1_4Bytes(128); - // Check 3 - buf->putInt1_2_4Bytes(255); - buf->putInt2_4_8Bytes(255); - buf->putInt1_4Bytes(255); - // Check 4 - buf->putInt1_2_4Bytes(256); - buf->putInt2_4_8Bytes(256); - buf->putInt1_4Bytes(256); - // Check 5 - buf->putInt1_2_4Bytes(0); - buf->putInt2_4_8Bytes(0); - buf->putInt1_4Bytes(0); - // Check 6 - buf->putInt1_2_4Bytes(1); - buf->putInt2_4_8Bytes(1); - buf->putInt1_4Bytes(1); - - // Check 7 - try { - buf->putInt1_2_4Bytes(0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - buf->putInt2_4_8Bytes(0x7FFFFFFFll); - buf->putInt1_4Bytes(0x7FFFFFFF); - - try { - buf->putInt2_4_8Bytes(0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - buf->putInt1_2_4Bytes(0x7FFF); - // Check 8 - buf->putInt2_4_8Bytes(0x7FFFll); - buf->putInt1_4Bytes(0x7FFF); - buf->putInt1_2_4Bytes(0x7F); - // Check 9 - buf->putInt2_4_8Bytes(0x7Fll); - buf->putInt1_4Bytes(0x7F); - - try { - buf->putInt1_2_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt1_2_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - uint32_t endWritePos = buf->getPos(); - buf->setPos(0); - - int32_t tmp32; - int64_t tmp64; - - // Check 0 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)124, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - // Check 1 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)127, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - // Check 2 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)128, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - // Check 3 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)255, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - // Check 4 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)256, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - // Check 5 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - // Check 6 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)1, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - // Check 7 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFFFFFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFFFFFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - // Check 8 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - // Check 9 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7F, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - - uint32_t endReadPos = buf->getPos(); - EXPECT_EQ(endWritePos, endReadPos); - - delete buf; -} - -TEST(ByteBuffer_Test, test_NumberLengths) -{ - ByteBuffer b; - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(127)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(128)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(129)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(255)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(256)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(0x7FFFFFFF)); - - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(0)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(1)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(4)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(31)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(126)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(32767)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32768)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32769)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(1030493)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(0x3FFFFFFF)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000000)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000001)); - - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(16383)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16384)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16385)); -} - TEST(ByteBuffer_Test, test_SerializableArray) { SerializableArray array; diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp index 1ec52791a4a..71dc9f1d1a9 100644 --- a/document/src/tests/weightedsetfieldvaluetest.cpp +++ b/document/src/tests/weightedsetfieldvaluetest.cpp @@ -16,9 +16,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -94,26 +93,25 @@ TEST(WeightedSetFieldValueTest, testWeightedSet) EXPECT_EQ(6, value.get(IntFieldValue(3))); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); WeightedSetFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(buffer, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); EXPECT_EQ(size_t(3), value2.size()); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 29414c901f8..969684f104b 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -8,12 +8,10 @@ #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/base/exceptions.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <sstream> -#include <limits> using vespalib::nbostream; using vespalib::make_string; @@ -24,10 +22,6 @@ using namespace vespalib::xml; namespace document { namespace { -bool isLegalVersion(uint16_t version) { - return (6 <= version) && (version <= 8); -} - void documentTypeError(vespalib::stringref name) __attribute__((noinline)); void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) __attribute__((noinline)); @@ -93,22 +87,13 @@ Document::Document(const DataType &type, DocumentId documentId) } } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserialize(repo, buffer); -} - void Document::setRepo(const DocumentTypeRepo& repo) { _fields.setRepo(repo); } -Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), +Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is) + : StructuredFieldValue(*DataType::DOCUMENT), _id(), _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), _lastModified(0) @@ -116,33 +101,6 @@ Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is, const deserialize(repo, is); } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - if (!includeContent) { - const DocumentType *newDocType = deserializeDocHeaderAndType(repo, buffer, _id, static_cast<const DocumentType*>(anticipatedType)); - if (newDocType) { - setType(*newDocType); - } - } else { - deserialize(repo, buffer); - } -} - - -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserializeHeader(repo, header); - deserializeBody(repo, body); -} - Document::~Document() = default; const DocumentType& @@ -170,30 +128,6 @@ Document::hasChanged() const return _fields.hasChanged(); } -DocumentId -Document::getIdFromSerialized(ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - deserializeDocHeader(buf, retVal); - buf.setPos(position); - - return retVal; -} - -const DocumentType * -Document::getDocTypeFromSerialized(const DocumentTypeRepo& repo, ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - const DocumentType *docType(deserializeDocHeaderAndType(repo, buf, retVal, nullptr)); - buf.setPos(position); - - return docType; -} - FieldValue& Document::assign(const FieldValue& value) { @@ -274,120 +208,11 @@ Document::calculateChecksum() const return calculator.checksum() ^ _fields.calculateChecksum(); } -const DocumentType * -Document::deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, DocumentId& id, - const DocumentType * docType) -{ - deserializeDocHeader(buffer, id); - - vespalib::stringref docTypeName(buffer.getBufferAtPos()); - buffer.incPos(docTypeName.size() + 1); // Skip 0-byte too - { - int16_t docTypeVersion; // version not supported anymore - buffer.getShortNetwork(docTypeVersion); - } - const DocumentType *docTypeNew = nullptr; - - if (! ((docType != nullptr) && (docType->getName() == docTypeName))) { - docTypeNew = repo.getDocumentType(docTypeName); - if (!docTypeNew) { - throw DocumentTypeNotFoundException(docTypeName, VESPA_STRLOC); - } - } - return docTypeNew; -} - -namespace { -[[noreturn]] void versionError(uint16_t version) __attribute__((noinline)); -[[noreturn]] void mainDocumentError(int64_t len) __attribute__((noinline)); -[[noreturn]] void notEnoughDocumentError(int32_t len, int64_t remaining) __attribute__((noinline)); - -void versionError(uint16_t version) { - throw DeserializeException(make_string( "Unrecognized serialization version %d", version), VESPA_STRLOC); -} - -void mainDocumentError(int64_t len) { - throw DeserializeException(make_string( - "Document lengths past %i is not supported. Corrupt data said length is %" PRId64 " bytes", - std::numeric_limits<int>::max(), len), VESPA_STRLOC); -} - -void notEnoughDocumentError(int32_t len, int64_t remaining) { - throw DeserializeException(make_string( "Buffer said document length is %d bytes, but only %" PRId64 " bytes remain in buffer", len, remaining)); -} - -} - -void -Document::deserializeDocHeader(ByteBuffer& buffer, DocumentId& id) { - int16_t version; - int32_t len; - buffer.getShortNetwork(version); - - if ( ! isLegalVersion(version) ) { - versionError(version); - } else if (version < 7) { - int64_t tmpLen = 0; - buffer.getInt2_4_8Bytes(tmpLen); - if (tmpLen > std::numeric_limits<int>::max()) { - mainDocumentError(tmpLen); - } else { - len = static_cast<int32_t>(tmpLen) - - ByteBuffer::getSerializedSize2_4_8Bytes(tmpLen) - - sizeof(uint16_t); - } - } else { - buffer.getIntNetwork(len); - } - - if (len > (long)buffer.getRemaining()) { - notEnoughDocumentError(len, buffer.getRemaining()); - } else { - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - id = DocumentId(stream); - buffer.incPos(stream.rp()); - unsigned char contentByte; - buffer.getByte(contentByte); - } -} - -void Document::serializeHeader(ByteBuffer& buffer) const { - nbostream stream; - serializeHeader(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - void Document::serializeHeader(nbostream& stream) const { VespaDocumentSerializer serializer(stream); serializer.write(*this, WITHOUT_BODY); } -void Document::serializeBody(ByteBuffer& buffer) const { - nbostream stream; - serializeBody(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -bool Document::hasBodyField() const { - for (document::StructuredFieldValue::const_iterator it(getFields().begin()), mt(getFields().end()); - it != mt; - ++it) - { - if ( ! it.field().isHeaderField() ) { - return true; - } - } - return false; -} - -void Document::serializeBody(nbostream& stream) const { - if (hasBodyField()) { - VespaDocumentSerializer serializer(stream); - serializer.write(_fields, BodyFields()); - } -} - void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os) { VespaDocumentDeserializer deserializer(repo, os, 0); try { @@ -397,38 +222,19 @@ void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & o } } -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& data) { - nbostream stream(data.getBufferAtPos(), data.getRemaining()); - deserialize(repo, stream); - data.incPos(data.getRemaining() - stream.size()); -} - -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body) { +void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & header, vespalib::nbostream & body) { deserializeHeader(repo, header); deserializeBody(repo, body); } -void Document::deserializeHeader(const DocumentTypeRepo& repo, - ByteBuffer& header) { - nbostream stream(header.getBufferAtPos(), header.getRemaining()); +void Document::deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { VespaDocumentDeserializer deserializer(repo, stream, 0); deserializer.read(*this); - header.incPos(header.getRemaining() - stream.size()); } -void Document::deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body) { - nbostream body_stream(body.getBufferAtPos(), body.getRemaining()); - VespaDocumentDeserializer - body_deserializer(repo, body_stream, getFields().getVersion()); - body_deserializer.readStructNoReset(getFields()); - body.incPos(body.getRemaining() - body_stream.size()); -} - -size_t -Document::getSerializedSize() const -{ - // Temporary non-optimal (but guaranteed correct) implementation. - return serialize()->getLength(); +void Document::deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { + VespaDocumentDeserializer deserializer(repo, stream, getFields().getVersion()); + deserializer.readStructNoReset(getFields()); } StructuredFieldValue::StructuredIterator::UP diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h index 3e1ec0da3e6..f20ef969e0c 100644 --- a/document/src/vespa/document/fieldvalue/document.h +++ b/document/src/vespa/document/fieldvalue/document.h @@ -27,10 +27,11 @@ private: DocumentId _id; StructFieldValue _fields; - // To avoid having to return another container object out of docblocks - // the meta data has been added to document. This will not be serialized - // with the document and really doesn't belong here! + // To avoid having to return another container object out of docblocks + // the meta data has been added to document. This will not be serialized + // with the document and really doesn't belong here! int64_t _lastModified; + public: typedef std::unique_ptr<Document> UP; typedef std::shared_ptr<Document> SP; @@ -42,13 +43,7 @@ public: Document(); Document(const Document&); Document(const DataType &, DocumentId id); - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType = nullptr); - Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream, const DataType *anticipatedType = nullptr); - /** - Constructor to deserialize only document and type from a buffer. Only relevant if includeContent is false. - */ - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType); - Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType = nullptr); + Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream); ~Document() override; void setRepo(const DocumentTypeRepo & repo); @@ -86,21 +81,6 @@ public: bool hasChanged() const override; - /** - * Returns a pointer to the Id of a serialized document, without performing - * the deserialization. buffer must point to the start position of the - * serialization. If the buffer doesn't have enough data remaining to have - * a legal Id in it, method returns NULL. - */ - static DocumentId getIdFromSerialized(ByteBuffer&); - - /** - * Returns a pointer to the document type of a serialized header, without - * performing the deserialization. Buffer must point to the start position - * of the serialization. - */ - static const DocumentType *getDocTypeFromSerialized(const DocumentTypeRepo&, ByteBuffer&); - // FieldValue implementation. FieldValue& assign(const FieldValue&) override; int compare(const FieldValue& other) const override; @@ -108,22 +88,12 @@ public: void printXml(XmlOutputStream& out) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - // Specialized serialization functions - void serializeHeader(ByteBuffer& buffer) const; + // Specialized serialization functions, Only used for testing legacy stuff void serializeHeader(vespalib::nbostream& stream) const; - void serializeBody(ByteBuffer& buffer) const; - void serializeBody(vespalib::nbostream& stream) const; - - /** Deserialize document contained in given bytebuffer. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& data); void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os); /** Deserialize document contained in given bytebuffers. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& body, ByteBuffer& header); - void deserializeHeader(const DocumentTypeRepo& repo, ByteBuffer& header); - void deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body); - - size_t getSerializedSize() const; + void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & body, vespalib::nbostream & header); /** Undo fieldvalue's toXml override for document. */ std::string toXml() const { return toXml(""); } @@ -137,18 +107,14 @@ public: void setFieldValue(const Field& field, FieldValue::UP data) override; private: - bool hasBodyField() const; + void deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & header); + void deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & body); bool hasFieldValue(const Field& field) const override { return _fields.hasValue(field); } void removeFieldValue(const Field& field) override { _fields.remove(field); } FieldValue::UP getFieldValue(const Field& field) const override { return _fields.getValue(field); } bool getFieldValue(const Field& field, FieldValue& value) const override { return _fields.getValue(field, value); } StructuredIterator::UP getIterator(const Field* first) const override; - - static void deserializeDocHeader(ByteBuffer& buffer, DocumentId& id); - static const DocumentType *deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, - DocumentId& id, const DocumentType * docType); }; } // document diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index 999747688eb..7737d03b45a 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -35,22 +35,11 @@ void FieldValue::serialize(nbostream &stream) const { serializer.write(*this); } -void FieldValue::serialize(ByteBuffer& buffer) const { +nbostream +FieldValue::serialize() const { nbostream stream; serialize(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -std::unique_ptr<ByteBuffer> FieldValue::serialize() const { - nbostream stream; - serialize(stream); - - nbostream::Buffer buf; - stream.swap(buf); - size_t sz = buf.size(); - auto bb = std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); - bb->setPos(sz); - return bb; + return stream; } size_t @@ -58,7 +47,7 @@ FieldValue::hash() const { vespalib::nbostream os; serialize(os); - return vespalib::hashValue(os.c_str(), os.size()) ; + return vespalib::hashValue(os.data(), os.size()) ; } bool diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index a152f74cc09..9ed8d522b17 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -19,15 +19,11 @@ #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/util/polymorphicarraybase.h> -namespace vespalib { - class nbostream; -} +namespace vespalib { class nbostream; } namespace document { -namespace fieldvalue { - class IteratorHandler; -} +namespace fieldvalue { class IteratorHandler; } class ByteBuffer; class DataType; @@ -73,8 +69,7 @@ public: virtual bool isA(const FieldValue& other) const; void serialize(vespalib::nbostream &stream) const; - void serialize(ByteBuffer& buffer) const; - std::unique_ptr<ByteBuffer> serialize() const; + vespalib::nbostream serialize() const; /** * Compares this fieldvalue with another fieldvalue. diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 5dfd8eff891..722904659ef 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -26,15 +26,30 @@ public: } -SerializableArray::Statistics SerializableArray::_stats; - SerializableArray::SerializableArray() : _serializedCompression(CompressionConfig::NONE), _uncompressedLength(0) { } -serializablearray::BufferMap & ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { +SerializableArray::SerializableArray(EntryMap entries, ByteBuffer::UP buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length) + : _entries(std::move(entries)), + _owned(), + _serializedCompression(comp_type) +{ + + if (CompressionConfig::isCompressed(_serializedCompression)) { + _compSerData = std::move(buffer); + _uncompressedLength = uncompressed_length; + } else { + _uncompressedLength = buffer->getRemaining(); + _uncompSerData = std::move(buffer); + } +} + +serializablearray::BufferMap & +ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { if (!owned) { owned = std::make_unique<serializablearray::BufferMap>(); } @@ -45,8 +60,8 @@ SerializableArray::SerializableArray(const SerializableArray& other) : Cloneable(), _entries(other._entries), _owned(), - _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : NULL), - _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : NULL), + _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : nullptr), + _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : nullptr), _serializedCompression(other._serializedCompression), _uncompressedLength(other._uncompressedLength) { @@ -66,17 +81,6 @@ SerializableArray::SerializableArray(const SerializableArray& other) } } -void -SerializableArray::swap(SerializableArray& other) -{ - _entries.swap(other._entries); - _owned.swap(other._owned); - std::swap(_uncompSerData, other._uncompSerData); - std::swap(_compSerData, other._compSerData); - std::swap(_serializedCompression, other._serializedCompression); - std::swap(_uncompressedLength, other._uncompressedLength); -} - void SerializableArray::clear() { _entries.clear(); @@ -86,9 +90,7 @@ void SerializableArray::clear() _uncompressedLength = 0; } -SerializableArray::~SerializableArray() -{ -} +SerializableArray::~SerializableArray() = default; void SerializableArray::invalidate() @@ -102,7 +104,7 @@ SerializableArray::set(int id, ByteBuffer::UP buffer) maybeDecompress(); Entry e(id, buffer->getRemaining(), buffer->getBuffer()); ensure(_owned)[id] = std::move(buffer); - EntryMap::iterator it = find(id); + auto it = find(id); if (it == _entries.end()) { _entries.push_back(e); } else { @@ -139,7 +141,7 @@ SerializableArray::get(int id) const { vespalib::ConstBufferRef buf; if ( !maybeDecompressAndCatch() ) { - EntryMap::const_iterator found = find(id); + auto found = find(id); if (found != _entries.end()) { const Entry& entry = *found; @@ -168,7 +170,7 @@ void SerializableArray::clear(int id) { maybeDecompress(); - EntryMap::iterator it = find(id); + auto it = find(id); if (it != _entries.end()) { _entries.erase(it); if (_owned) { @@ -193,7 +195,7 @@ SerializableArray::deCompress() // throw (DeserializeException) _uncompSerData = std::move(_compSerData); LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); } else { - ByteBuffer::UP newSerialization(new ByteBuffer(_uncompressedLength)); + auto newSerialization = std::make_unique<ByteBuffer>(vespalib::alloc::Alloc::alloc(_uncompressedLength), _uncompressedLength); vespalib::DataBuffer unCompressed(newSerialization->getBuffer(), newSerialization->getLength()); unCompressed.clear(); try { @@ -210,35 +212,16 @@ SerializableArray::deCompress() // throw (DeserializeException) if (unCompressed.getDataLen() != (size_t)_uncompressedLength) { throw DeserializeException( - make_string("Did not decompress to the expected length: had %zu, wanted %d, got %zu", + make_string("Did not decompress to the expected length: had %u, wanted %d, got %zu", _compSerData->getRemaining(), _uncompressedLength, unCompressed.getDataLen()), VESPA_STRLOC); } assert(newSerialization->getBuffer() == unCompressed.getData()); - newSerialization->setLimit(_uncompressedLength); _uncompSerData = std::move(newSerialization); LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); } } -void SerializableArray::assign(EntryMap & entries, - ByteBuffer::UP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length) -{ - _serializedCompression = comp_type; - - _entries.clear(); - _entries.swap(entries); - if (CompressionConfig::isCompressed(_serializedCompression)) { - _compSerData.reset(buffer.release()); - _uncompressedLength = uncompressed_length; - } else { - _uncompressedLength = buffer->getRemaining(); - _uncompSerData.reset(buffer.release()); - } -} - vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2f7d65938aa..1e766599dff 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -26,7 +26,6 @@ namespace document { -class SerializableArrayIterator; class ByteBuffer; namespace serializablearray { @@ -36,25 +35,6 @@ namespace serializablearray { class SerializableArray : public vespalib::Cloneable { public: - // Counts set during serialization, in order to provide metrics for how - // often we use cached version, and how often we compress. - struct Statistics { - uint64_t _usedCachedSerializationCount; - uint64_t _compressedDocumentCount; - uint64_t _compressionDidntHelpCount; - uint64_t _uncompressableCount; - uint64_t _serializedUncompressed; - uint64_t _inputWronglySerialized; - - Statistics() - : _usedCachedSerializationCount(0), - _compressedDocumentCount(0), - _compressionDidntHelpCount(0), - _uncompressableCount(0), - _serializedUncompressed(0), - _inputWronglySerialized(0) {} - }; - /** * Contains the id of a field, the size and a buffer reference that is either * a relative offset to a common buffer, or the buffer itself it it is not. @@ -98,12 +78,6 @@ public: static const uint32_t ReservedId = 100; static const uint32_t ReservedIdUpper = 128; - -private: - static Statistics _stats; - -public: - static Statistics& getStatistics() { return _stats; } using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; @@ -111,9 +85,9 @@ public: using CompressionInfo = vespalib::compression::CompressionInfo; SerializableArray(); - virtual ~SerializableArray(); - - void swap(SerializableArray& other); + SerializableArray(EntryMap entries, ByteBufferUP buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length); + ~SerializableArray() override; /** * Stores a value in the array. @@ -141,9 +115,6 @@ public: /** @return Returns true if the given ID is Set in the array. */ bool has(int id) const; - /** @return Number of elements in array */ - bool hasAnyElems() const { return !_entries.empty(); } - /** * clears an attribute. * @@ -157,16 +128,6 @@ public: CompressionConfig::Type getCompression() const { return _serializedCompression; } CompressionInfo getCompressionInfo() const; - /** - * Sets the serialized data that is the basis for this object's - * content. This is used by deserialization. Any existing entries - * are cleared. - */ - void assign(EntryMap &entries, - ByteBufferUP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length); - bool empty() const { return _entries.empty(); } const ByteBuffer* getSerializedBuffer() const { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 87c8d09648c..b861537f6f7 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -48,15 +48,14 @@ StructFieldValue::Chunks::~Chunks() = default; void StructFieldValue::Chunks::push_back(SerializableArray::UP item) { - assert(_sz < 2); - _chunks[_sz++].reset(item.release()); + assert(size() < 2); + _chunks[size()] = std::move(item); } void StructFieldValue::Chunks::clear() { _chunks[0].reset(); _chunks[1].reset(); - _sz = 0; } const StructDataType & @@ -81,13 +80,11 @@ StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, _doc_type = &repo.getDocumentType(); _version = version; - _chunks.push_back(std::make_unique<SerializableArray>()); - _chunks.back().assign(fm, std::move(buffer), comp_type, uncompressed_length); + _chunks.push_back(std::make_unique<SerializableArray>(std::move(fm), std::move(buffer), comp_type, uncompressed_length)); _hasChanged = false; } -bool StructFieldValue::serializeField(int field_id, uint16_t version, - FieldValueWriter &writer) const { +bool StructFieldValue::serializeField(int field_id, uint16_t version, FieldValueWriter &writer) const { if (version == _version) { for (int i = _chunks.size() - 1; i >= 0; --i) { vespalib::ConstBufferRef buf = _chunks[i].get(field_id); @@ -183,9 +180,9 @@ StructFieldValue::getFieldValue(const Field& field) const if (buf.size() != 0) { nbostream stream(buf.c_str(), buf.size()); FieldValue::UP value(field.getDataType().createFieldValue()); - if ((_repo == NULL) && (_doc_type != NULL)) { - std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); - createFV(*value, *tmpRepo, stream, *_doc_type, _version); + if ((_repo == nullptr) && (_doc_type != nullptr)) { + DocumentTypeRepo tmpRepo(*_doc_type); + createFV(*value, tmpRepo, stream, *_doc_type, _version); } else { createFV(*value, *_repo, stream, *_doc_type, _version); } @@ -216,7 +213,7 @@ StructFieldValue::getFieldValue(const Field& field, FieldValue& value) const vespalib::ConstBufferRef buf = getRawField(fieldId); if (buf.size() > 0) { nbostream_longlivedbuf stream(buf.c_str(), buf.size()); - if ((_repo == NULL) && (_doc_type != NULL)) { + if ((_repo == nullptr) && (_doc_type != nullptr)) { std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); createFV(value, *tmpRepo, stream, *_doc_type, _version); } else { @@ -239,17 +236,29 @@ StructFieldValue::hasFieldValue(const Field& field) const return false; } +namespace { + +std::unique_ptr<ByteBuffer> +serializeDoc(const FieldValue & fv) { + nbostream stream = fv.serialize(); + nbostream::Buffer buf; + stream.swap(buf); + size_t sz = buf.size(); + return std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); +} + +} void StructFieldValue::setFieldValue(const Field& field, FieldValue::UP value) { int fieldId = field.getId(); - std::unique_ptr<ByteBuffer> serialized(value->serialize()); - serialized->flip(); + + std::unique_ptr<ByteBuffer> serialized = serializeDoc(*value); if (_chunks.empty()) { - _chunks.push_back(SerializableArray::UP(new SerializableArray())); + _chunks.push_back(std::make_unique<SerializableArray>()); } - _chunks.back().set(fieldId, std::move(serialized)); + _chunks[0].set(fieldId, std::move(serialized)); _hasChanged = true; } @@ -274,7 +283,7 @@ StructFieldValue::clear() FieldValue& StructFieldValue::assign(const FieldValue& value) { - const StructFieldValue& other(dynamic_cast<const StructFieldValue&>(value)); + const auto & other(dynamic_cast<const StructFieldValue&>(value)); return operator=(other); } @@ -285,7 +294,7 @@ StructFieldValue::compare(const FieldValue& otherOrg) const if (comp != 0) { return comp; } - const StructFieldValue& other = static_cast<const StructFieldValue&>(otherOrg); + const auto & other = static_cast<const StructFieldValue&>(otherOrg); std::vector<int> a; getRawFieldIds(a); @@ -315,9 +324,9 @@ StructFieldValue::compare(const FieldValue& otherOrg) const uint32_t StructFieldValue::calculateChecksum() const { - ByteBuffer::UP buffer(serialize()); + nbostream buffer(serialize()); vespalib::crc_32_type calculator; - calculator.process_bytes(buffer->getBuffer(), buffer->getPos()); + calculator.process_bytes(buffer.peek(), buffer.size()); return calculator.checksum(); } @@ -386,7 +395,7 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { std::vector<int> _ids; std::vector<int>::iterator _cur; - FieldIterator(const StructFieldValue& s) + explicit FieldIterator(const StructFieldValue& s) : _struct(s), _ids(), _cur(_ids.begin()) @@ -412,7 +421,7 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { LOG(debug, "struct data type: %s", _struct.getType().toString(true).c_str()); } } - return 0; + return nullptr; } }; @@ -421,10 +430,10 @@ StructFieldValue::getIterator(const Field* toFind) const { StructuredIterator::UP ret; - FieldIterator *fi = new FieldIterator(*this); + auto *fi = new FieldIterator(*this); ret.reset(fi); - if (toFind != NULL) { + if (toFind != nullptr) { fi->skipTo(toFind->getId()); } return ret; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index b5b15e9dfce..30500229813 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -27,19 +27,16 @@ class StructFieldValue : public StructuredFieldValue public: class Chunks { public: - Chunks() : _sz(0) { } + Chunks() { } ~Chunks(); SerializableArray & operator [] (size_t i) { return *_chunks[i]; } const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; } VESPA_DLL_LOCAL void push_back(SerializableArray::UP item); - SerializableArray & back() { return *_chunks[_sz-1]; } - const SerializableArray & back() const { return *_chunks[_sz-1]; } - size_t size() const { return _sz; } - bool empty() const { return _sz == 0; } + size_t size() const { return _chunks[1] ? 2 : _chunks[0] ? 1 : 0; } + bool empty() const { return !_chunks[0]; } VESPA_DLL_LOCAL void clear(); private: SerializableArray::CP _chunks[2]; - size_t _sz; }; private: Chunks _chunks; diff --git a/document/src/vespa/document/serialization/util.h b/document/src/vespa/document/serialization/util.h index cbcc2c97017..05953f1de2b 100644 --- a/document/src/vespa/document/serialization/util.h +++ b/document/src/vespa/document/serialization/util.h @@ -97,32 +97,5 @@ void putInt2_4_8Bytes(Output &out, uint64_t val) { } } -inline uint32_t sizeOfInt1_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt1_2_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else if (val < 0x4000) { - return 2; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt2_4_8Bytes(uint64_t val) { - if (val < 0x8000) { - return 2; - } else if (val < 0x40000000) { - return 4; - } else { - return 8; - } -} } // namespace document diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 570a51907da..4d60befba52 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -293,10 +293,10 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { using vespalib::compression::compress; - vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); + vespalib::ConstBufferRef buf(stream.data(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, - vespalib::ConstBufferRef(stream.c_str(), stream.size()), + vespalib::ConstBufferRef(stream.data(), stream.size()), compressed_data, false); if (compressedType != config.type || ! compressionSufficient(config, stream.size(), compressed_data.getDataLen())) diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index b43b7a59c5b..f289e6a8f27 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -7,7 +7,6 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/bufferexceptions.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> @@ -231,20 +230,18 @@ DocumentUpdate::serializeFlags(int size_) const } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream && stream) { - vespalib::nbostream is(buffer.getBufferAtPos(), buffer.getRemaining()); auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, is); - buffer.setPos(buffer.getPos() + is.rp()); + update->initHEAD(repo, std::move(stream)); return update; } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream stream) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, std::move(stream)); + update->initHEAD(repo, stream); return update; } diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index a672d028f35..fb3d6c0f7a3 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -32,6 +32,7 @@ namespace document { +class ByteBuffer; class Document; class VespaDocumentSerializer; /** @@ -50,8 +51,8 @@ public: /** * Create new style document update, possibly with field path updates. */ - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream stream); - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, ByteBuffer & buffer); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); DocumentUpdate(); /** @@ -124,8 +125,8 @@ private: bool _needHardReserialize; int deserializeFlags(int sizeAndFlags); - void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void deserializeBody(const DocumentTypeRepo &repo, vespalib::nbostream &stream); void lazyDeserialize(const DocumentTypeRepo & repo, vespalib::nbostream & stream); void ensureDeserialized() const; diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 3e19420d0d0..0dbf8e5cc9e 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -7,7 +7,6 @@ namespace document { -class ByteBuffer; class DocumentTypeRepo; class Field; class FieldValue; @@ -35,7 +34,7 @@ public: using SP = std::shared_ptr<FieldPathUpdate>; using CP = vespalib::CloneablePtr<FieldPathUpdate>; - ~FieldPathUpdate(); + ~FieldPathUpdate() override; enum FieldPathUpdateType { Add = IDENTIFIABLE_CLASSID(AddFieldPathUpdate), diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 8cb148abe25..48ca7bd36d7 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp printable.cpp - serializable.cpp + serializableexceptions.cpp stringutil.cpp DEPENDS AFTER diff --git a/document/src/vespa/document/util/bufferexceptions.h b/document/src/vespa/document/util/bufferexceptions.h index 8a3215f6c79..aee7f3ae568 100644 --- a/document/src/vespa/document/util/bufferexceptions.h +++ b/document/src/vespa/document/util/bufferexceptions.h @@ -15,13 +15,5 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(BufferOutOfBoundsException) }; -class InputOutOfRangeException : public vespalib::IoException { -public: - InputOutOfRangeException(const vespalib::string& msg, - const vespalib::string& location = ""); - - VESPA_DEFINE_EXCEPTION_SPINE(InputOutOfRangeException) -}; - } diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp index ccbc2bc7790..0fbc6ca1c63 100644 --- a/document/src/vespa/document/util/bytebuffer.cpp +++ b/document/src/vespa/document/util/bytebuffer.cpp @@ -12,732 +12,160 @@ #include <sstream> #include <arpa/inet.h> -#define LOG_DEBUG1(a) -// Enable this macros instead to see what bytebuffer calls come -//#define LOG_DEBUG1(a) std::cerr << "ByteBuffer(" << ((void*) this) << " " << a << ")\n"; - -#define LOG_DEBUG2(a,b) LOG_DEBUG1(vespalib::make_string(a,b)); -#define LOG_DEBUG3(a,b,c) LOG_DEBUG1(vespalib::make_string(a,b,c)); -#define LOG_DEBUG4(a,b,c,d) LOG_DEBUG1(vespalib::make_string(a,b,c,d)); - using vespalib::alloc::Alloc; +using vespalib::make_string; namespace document { -VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(InputOutOfRangeException); +namespace { -vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { - vespalib::asciistream ost; - ost << pos << " > " << len; - return ost.str(); -} +static void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline, noreturn)); -BufferOutOfBoundsException::BufferOutOfBoundsException( - size_t pos, size_t len, const vespalib::string& location) - : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) +void throwOutOfBounds(size_t want, size_t has) { + throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); } -InputOutOfRangeException::InputOutOfRangeException( - const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::INTERNAL_FAILURE, location, 1) -{ } -ByteBuffer::ByteBuffer() : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(NULL, 0); - LOG_DEBUG1("Created empty bytebuffer"); +#if defined(__i386__) || defined(__x86_64__) + +template<typename T> +void +ByteBuffer::getDoubleLongNetwork(T &val) { + //TODO: Change this if we move to big-endian hardware + if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { + throwOutOfBounds(sizeof(T), getRemaining()); + } + + auto * data = reinterpret_cast<unsigned char*>(&val); + for (int i=sizeof(T)-1; i>=0; --i) { + getByte(data[i]); + } } -ByteBuffer::ByteBuffer(size_t len) : - ByteBuffer(Alloc::alloc(len), len) +#else +#error "getDoubleLongNetwork is undefined for this arcitecture" +#endif + +VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); + +vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { + vespalib::asciistream ost; + ost << pos << " > " << len; + return ost.str(); +} + +BufferOutOfBoundsException::BufferOutOfBoundsException(size_t pos, size_t len, const vespalib::string& location) + : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) { } -ByteBuffer::ByteBuffer(const char* buffer, size_t len) : - _buffer(NULL), - _len(0), +ByteBuffer::ByteBuffer(const char* buffer, uint32_t len) : + _buffer(const_cast<char *>(buffer)), + _len(len), _pos(0), - _limit(0), - _bufHolder(NULL), _ownedBuffer() { - set(buffer, len); } -ByteBuffer::ByteBuffer(Alloc buffer, size_t len) : +ByteBuffer::ByteBuffer(Alloc buffer, uint32_t len) : _buffer(static_cast<char *>(buffer.get())), _len(len), _pos(0), - _limit(len), - _bufHolder(NULL), _ownedBuffer(std::move(buffer)) { } -ByteBuffer::ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit) : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(buf, pos, len, limit); - LOG_DEBUG3("Created copy of byte buffer of length %" PRIu64 " with " - "limit %" PRIu64 ".", len, limit); -} - -ByteBuffer::ByteBuffer(const ByteBuffer& bb) : - _buffer(0), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), +ByteBuffer::ByteBuffer(const ByteBuffer& rhs) : + _buffer(nullptr), + _len(rhs._len), + _pos(rhs._pos), _ownedBuffer() { - LOG_DEBUG1("Created empty byte buffer to assign to."); - *this = bb; -} - -ByteBuffer& ByteBuffer::operator=(const ByteBuffer & org) -{ - if (this != & org) { - cleanUp(); - if (org._len > 0 && org._buffer) { - Alloc::alloc(org._len + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer,org._buffer,org._len); - _buffer[org._len] = 0; - } - _len = org._len; - _pos = org._pos; - _limit = org._limit; - LOG_DEBUG4("Assignment created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); - } - return *this; -} - -void -ByteBuffer::set(BufferHolder* buf, size_t pos, size_t len, size_t limit) -{ - cleanUp(); - _bufHolder = buf; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=pos; - _len=len; - _limit=limit; - LOG_DEBUG4("set() created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); -} - -ByteBuffer::~ByteBuffer() -{ - if (_bufHolder) { - _bufHolder->subRef(); + if (rhs._len > 0 && rhs._buffer) { + Alloc buf = Alloc::alloc(rhs._len); + memcpy(buf.get(), rhs._buffer, rhs._len); + _ownedBuffer = std::move(buf); + _buffer = static_cast<const char *>(_ownedBuffer.get()); } } -std::unique_ptr<ByteBuffer> -ByteBuffer::sliceCopy() const -{ - ByteBuffer* buf = new ByteBuffer; - buf->sliceFrom(*this, _pos, _limit); - - LOG_DEBUG3("Created slice at pos %" PRIu64 " with limit %" PRIu64 ".", - _pos, _limit); - return std::unique_ptr<ByteBuffer>(buf); -} +ByteBuffer::~ByteBuffer() = default; -void ByteBuffer::throwOutOfBounds(size_t want, size_t has) -{ - LOG_DEBUG1("Throwing out of bounds exception"); - throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); -} - -void -ByteBuffer::sliceFrom(const ByteBuffer& buf, size_t from, size_t to) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Created slice from buffer from %" PRIu64 " to %" PRIu64 ".", - from, to); - if (from > buf._len) { - throwOutOfBounds(from, buf._len); - } else if (to > buf._len) { - throwOutOfBounds(to, buf._len); - } else if (to < from) { - throwOutOfBounds(to, from); - } else { - - if (!buf._buffer) { - clear(); - return; - } - - // Slicing from someone that doesn't own their buffer, must make own copy. - if (( buf._ownedBuffer.get() == NULL ) && (buf._bufHolder == NULL)) { - cleanUp(); - Alloc::alloc(to-from + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer, buf._buffer + from, to-from); - _buffer[to-from] = 0; - _pos = 0; - _len = _limit = to-from; - return; - } - - // Slicing from someone that owns, but hasn't made a reference counter yet. - if (!buf._bufHolder) { - buf._bufHolder=new BufferHolder(std::move(const_cast<Alloc &>(buf._ownedBuffer))); - } - - // Slicing from refcounter. - cleanUp(); - - _bufHolder = buf._bufHolder; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=from; - _len=to; - _limit=to; - } -} - -ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, size_t len) +ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, uint32_t len) { if (buffer && len) { - Alloc newBuf = Alloc::alloc(len + 1); + Alloc newBuf = Alloc::alloc(len); memcpy(newBuf.get(), buffer, len); - static_cast<char *>(newBuf.get())[len] = 0; return new ByteBuffer(std::move(newBuf), len); } else { - return NULL; - } -} - -void -ByteBuffer::setPos(size_t pos) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting pos to be %" PRIu64 ", limit is %" PRIu64 ".", - pos, _limit); - if (pos>_limit) { - throwOutOfBounds(pos, _limit); - } else { - _pos=pos; - } -} - -void -ByteBuffer::setLimit(size_t limit) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting limit to %" PRIu64 ", (size is %" PRIu64 ").", limit, _len); - if (limit>_len) { - throwOutOfBounds(limit, _len); - } else { - _limit=limit; + return nullptr; } } - -ByteBuffer::BufferHolder::BufferHolder(Alloc buffer) - : _buffer(std::move(buffer)) -{ -} - -ByteBuffer::BufferHolder::~BufferHolder() = default; -void ByteBuffer::dump() const +void ByteBuffer::incPos(uint32_t pos) { - fprintf(stderr, "ByteBuffer: Length %lu, Pos %lu, Limit %lu\n", - _len, _pos, _limit); - for (size_t i=0; i<_len; i++) { - if (_buffer[i]>32 && _buffer[i]<126) { - fprintf(stderr, "%c", _buffer[i]); - } else { - fprintf(stderr, "[%d]",_buffer[i]); - } - } -} - -void ByteBuffer::incPos(size_t pos) -{ - LOG_DEBUG2("incPos(%" PRIu64 ")", pos); - if (_pos + pos > _limit) { - throwOutOfBounds(_pos + pos, _limit); + if (_pos + pos > _len) { + throwOutOfBounds(_pos + pos, _len); } else { _pos+=pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif } } void ByteBuffer::getNumeric(uint8_t & v) { - LOG_DEBUG2("getNumeric8(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(uint8_t *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(uint8_t v) { - LOG_DEBUG2("putNumeric8(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - *(uint8_t *) getBufferAtPos() = v; + v = *reinterpret_cast<const uint8_t *>(getBufferAtPos()); incPosNoCheck(sizeof(v)); } } -size_t ByteBuffer::forceValgrindStart2Pos() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(_buffer), *e(c + _pos); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - -size_t ByteBuffer::forceValgrindPos2Lim() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(getBufferAtPos()), *e(c + getRemaining()); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - - void ByteBuffer::getNumericNetwork(int16_t & v) { - LOG_DEBUG2("getNumericNetwork16(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint16_t val = *(uint16_t *) (void *) getBufferAtPos(); + uint16_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohs(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int16_t & v) { - LOG_DEBUG2("getNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int16_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(int16_t v) { - LOG_DEBUG2("putNumericNetwork16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint16_t val = htons(v); - *(uint16_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int16_t v) { - LOG_DEBUG2("putNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int16_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - void ByteBuffer::getNumericNetwork(int32_t & v) { - LOG_DEBUG2("getNumericNetwork32(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint32_t val = *(uint32_t *) (void *) getBufferAtPos(); + uint32_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohl(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int32_t & v) { - LOG_DEBUG2("getNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int32_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - - -void ByteBuffer::putNumericNetwork(int32_t v) { - LOG_DEBUG2("putNumericNetwork32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint32_t val = htonl(v); - *(uint32_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int32_t v) { - LOG_DEBUG2("putNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int32_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::getNumericNetwork(float & v) { - LOG_DEBUG2("getNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(uint32_t) == 4 - // and endianness same for float and ints - int32_t val; - getIntNetwork(val); - memcpy(&v, &val, sizeof(v)); -} - -void ByteBuffer::getNumeric(float & v) { - LOG_DEBUG2("getNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(float *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(float v) { - LOG_DEBUG2("putNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(int32_t) == 4 - // and endianness same for float and ints - int32_t val; - memcpy(&val, &v, sizeof(val)); - putIntNetwork(val); -} - -void ByteBuffer::putNumeric(float v) { - LOG_DEBUG2("putNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(float *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} void ByteBuffer::getNumeric(int64_t& v) { - LOG_DEBUG2("getNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int64_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(int64_t v) { - LOG_DEBUG2("putNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int64_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::getNumeric(double& v) { - LOG_DEBUG2("getNumericDouble(%f)", v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - v = *(double *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(double v) { - LOG_DEBUG2("putNumericDouble(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(double *) (void *) getBufferAtPos() = v; + memcpy(&v, getBufferAtPos(), sizeof(v)); incPosNoCheck(sizeof(v)); } } void ByteBuffer::getNumericNetwork(double & v) { - LOG_DEBUG2("getNumericNetworkDouble(%f)", v); getDoubleLongNetwork(v); } -void ByteBuffer::putNumericNetwork(int64_t v) { - LOG_DEBUG2("putNumericNetwork64(%" PRId64 ")", v); - putDoubleLongNetwork(v); -} -void ByteBuffer::putNumericNetwork(double v) { - LOG_DEBUG2("putNumericNetworkDouble(%f)", v); - putDoubleLongNetwork(v); -} + void ByteBuffer::getNumericNetwork(int64_t & v) { - LOG_DEBUG2("getNumericNetwork64(%" PRId64 ")", v); getDoubleLongNetwork(v); } -void ByteBuffer::putInt2_4_8Bytes(int64_t number, size_t len) { - LOG_DEBUG3("putInt2_4_8(%" PRId64 ", %" PRIu64 ")", number, len); - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (len == 0) { - if (number < 0x8000ll) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (number < 0x40000000ll) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } - } else if (len == 2) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (len == 4) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else if (len == 8) { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } else { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number using %d bytes.", (int)len), VESPA_STRLOC); - } -} - -void ByteBuffer::getInt2_4_8Bytes(int64_t & v) { - LOG_DEBUG2("getInt2_4_8(%" PRId64 ")", v); - if (getRemaining() >= 2) { - uint8_t flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 8 bytes - int64_t tmp; - getLongNetwork(tmp); - v = tmp & 0x3FFFFFFFFFFFFFFFll; - } else { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = (int64_t) (tmp & 0x3FFFFFFF); - } - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int64_t) tmp; - } - } else { - throwOutOfBounds(getRemaining(), 2); - } -} - -size_t ByteBuffer::getSerializedSize2_4_8Bytes(int64_t number) { - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (number < 0x8000ll) { - return 2; - } else if (number < 0x40000000ll) { - return 4; - } else { - return 8; - } -} - -void ByteBuffer::putInt1_2_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_2_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else if (number < 0x4000) { - putShortNetwork((int16_t) (((int16_t)number) | ((int16_t) 0x8000))); - } else { - putIntNetwork(number | 0xC0000000); - } -} - -void ByteBuffer::getInt1_2_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_2_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x3FFFFFFF; - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int32_t) (tmp & ((int16_t) 0x3FFF)); - } - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} - -size_t ByteBuffer::getSerializedSize1_2_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else if (number < 0x4000) { - return 2; - } else { - return 4; - } -} -void ByteBuffer::putInt1_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else { - putIntNetwork(number | 0x80000000); - } -} -void ByteBuffer::getInt1_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x7FFFFFFF; - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} -size_t ByteBuffer::getSerializedSize1_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else { - return 4; - } -} -void ByteBuffer::getBytes(void *buffer, size_t count) +void ByteBuffer::getBytes(void *buffer, uint32_t count) { - LOG_DEBUG3("getBytes(%p, %" PRIu64 ")", buffer, count); const char *v = getBufferAtPos(); incPos(count); memcpy(buffer, v, count); } -void ByteBuffer::putBytes(const void *buf, size_t count) { - LOG_DEBUG3("putBytes(%p, %" PRIu64 ")", buf, count); - if (__builtin_expect(getRemaining() < count, 0)) { - throwOutOfBounds(getRemaining(), sizeof(count)); - } else { - memcpy(getBufferAtPos(), buf, count); - incPosNoCheck(count); - } -} -std::string ByteBuffer::toString() { - std::ostringstream ost; - StringUtil::printAsHex(ost, getBuffer(), getLength()); - return ost.str(); -} - -void ByteBuffer::swap(ByteBuffer& other) { - LOG_DEBUG2("swap(%p)", &other); - std::swap(_bufHolder, other._bufHolder); - std::swap(_buffer, other._buffer); - std::swap(_len, other._len); - std::swap(_pos, other._pos); - std::swap(_limit, other._limit); -} - -void ByteBuffer::cleanUp() { - LOG_DEBUG1("cleanUp()"); - if (_bufHolder) { - _bufHolder->subRef(); - _bufHolder = NULL; - } else { - Alloc().swap(_ownedBuffer); - } - _buffer = NULL; -} } // document diff --git a/document/src/vespa/document/util/bytebuffer.h b/document/src/vespa/document/util/bytebuffer.h index 6467e6d8bf0..17ab0322a7d 100644 --- a/document/src/vespa/document/util/bytebuffer.h +++ b/document/src/vespa/document/util/bytebuffer.h @@ -15,7 +15,6 @@ #pragma once #include <vespa/vespalib/util/alloc.h> -#include <vespa/vespalib/util/referencecounter.h> namespace document { @@ -23,27 +22,21 @@ class ByteBuffer { public: typedef std::unique_ptr<ByteBuffer> UP; - /** - * Creates a byte buffer with no underlying buffer. - * Use set() to set the buffer. - */ - ByteBuffer(); ByteBuffer(const ByteBuffer &); - ByteBuffer& operator=(const ByteBuffer &); + ByteBuffer& operator=(const ByteBuffer &) = delete; + ByteBuffer(ByteBuffer &&) = default; + ByteBuffer& operator=(ByteBuffer &&) = delete; ~ByteBuffer(); - /** Allocates buffer with len bytes. */ - ByteBuffer(size_t len); - /** * Create a buffer with the given content. * * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(const char* buffer, size_t len); + ByteBuffer(const char* buffer, uint32_t len); /** * Create a buffer with the given content. @@ -51,22 +44,7 @@ public: * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(vespalib::alloc::Alloc buffer, size_t len); - - /** - * Sets the buffer pointed to by this buffer. Allows for multiple - * usages of the same ByteBuffer. - */ - void set(const char* buffer, size_t len) { - cleanUp(); - _buffer = const_cast<char*>(buffer); - _len=len; - _limit=len; - _pos=0; - } - - /** Clear this buffer, and set free the underlying BufferHolder. */ - void reset() { set(NULL, 0); } + ByteBuffer(vespalib::alloc::Alloc buffer, uint32_t len); /** * Creates a ByteBuffer object from another buffer. allocates @@ -75,62 +53,28 @@ public: * @param buffer The buffer to copy. * @param len The length of the buffer. * - * @return Returns a newly created bytebuffer object, or NULL - * if buffer was NULL, or len was <=0. + * @return Returns a newly created bytebuffer object, or nullptr + * if buffer was nullptr, or len was <=0. */ - static ByteBuffer* copyBuffer(const char* buffer, size_t len); - - std::unique_ptr<ByteBuffer> sliceCopy() const; - - /** - * @throws BufferOutOfBoundsException If faulty range is given. - */ - void sliceFrom(const ByteBuffer& buf, size_t from, size_t to); + static ByteBuffer* copyBuffer(const char* buffer, uint32_t len); /** @return Returns the buffer pointed to by this object (at position 0) */ - char* getBuffer() const { return _buffer; } + const char* getBuffer() const { return _buffer; } /** @return Returns the length of the buffer pointed to by this object. */ - size_t getLength() const { return _len; } - - /** - * Adjust the length of the buffer. Only sane to shorten it, as you do not - * know what is ahead. - */ - void setLength(size_t len) { _len = len; } + uint32_t getLength() const { return _len; } /** @return Returns a pointer to the current position in the buffer. */ - char* getBufferAtPos() const { return _buffer + _pos; } + const char* getBufferAtPos() const { return _buffer + _pos; } /** @return Returns the index of the current position in the buffer. */ - size_t getPos() const { return _pos; } - - /** @return Returns the limit. */ - size_t getLimit() const { return _limit; } + uint32_t getPos() const { return _pos; } /** * @return Returns the number of bytes remaining in the buffer - that is, - * getLimit()-getPos(). + * getLength()-getPos(). */ - size_t getRemaining() const { return _limit-_pos; } - - /** - * Changes the position in the buffer. - * - * @throws BufferOutOfBoundsException; - */ - void setPos(size_t pos); - - /** - * Sets the buffer limit. - * - * @param limit The new limit. - * @return True if the limit is legal (less than the length) - * @throws BufferOutOfBoundsException; - */ - void setLimit(size_t limit); - size_t forceValgrindStart2Pos() const __attribute__ ((noinline)); - size_t forceValgrindPos2Lim() const __attribute__ ((noinline)); + uint32_t getRemaining() const { return _len -_pos; } /** * Moves the position in the buffer. @@ -138,267 +82,37 @@ public: * @param pos The number of bytes to move the position. The new position * will be oldPos + pos. This is the same as doing * setPos(getPos()+pos) - * @return True if the position could be moved (it was inside the limit - * of the buffer). * @throws BufferOutOfBoundsException; */ - void incPos(size_t pos); - - void incPosNoCheck(size_t pos) { - _pos += pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif - } - - /** - * Resets pos to 0, and sets limit to old pos. Use this before reading - * from a buffer you have written to - */ - void flip() { - _limit = _pos; - _pos = 0; - } + void incPos(uint32_t pos); - /** - * Sets pos to 0 and limit to length. Use this to start writing from the - * start of the buffer. - */ - void clear() { - _pos=0; - _limit=_len; - } - - void getNumericNetwork(uint8_t & v) { getNumeric(v); } void getNumeric(uint8_t & v); - void putNumericNetwork(uint8_t v) { putNumeric(v); } - void putNumeric(uint8_t v); void getNumericNetwork(int16_t & v); - void getNumeric(int16_t & v); - void putNumericNetwork(int16_t v); - void putNumeric(int16_t v); void getNumericNetwork(int32_t & v); - void getNumeric(int32_t & v); - void putNumericNetwork(int32_t v); - void putNumeric(int32_t v); - void getNumericNetwork(float & v); - void getNumeric(float & v); - void putNumericNetwork(float v); - void putNumeric(float v); + void getNumericNetwork(int64_t & v); void getNumeric(int64_t& v); - void putNumericNetwork(int64_t v); - void putNumeric(int64_t v); void getNumericNetwork(double & v); - void getNumeric(double& v); - void putNumericNetwork(double v); - void putNumeric(double v); + void getChar(char & val) { unsigned char t;getByte(t); val=t; } void getByte(uint8_t & v) { getNumeric(v); } - void putByte(uint8_t v) { putNumeric(v); } void getShortNetwork(int16_t & v) { getNumericNetwork(v); } - void getShort(int16_t & v) { getNumeric(v); } - void putShortNetwork(int16_t v) { putNumericNetwork(v); } - void putShort(int16_t v) { putNumeric(v); } void getIntNetwork(int32_t & v) { getNumericNetwork(v); } - void getInt(int32_t & v) { getNumeric(v); } - void putIntNetwork(int32_t v) { putNumericNetwork(v); } - void putInt(int32_t v) { putNumeric(v); } - void getFloatNetwork(float & v) { getNumericNetwork(v); } - void getFloat(float & v) { getNumeric(v); } - void putFloatNetwork(float v) { putNumericNetwork(v); } - void putFloat(float v) { putNumeric(v); } void getLongNetwork(int64_t & v) { getNumericNetwork(v); } void getLong(int64_t& v) { getNumeric(v); } - void putLongNetwork(int64_t v) { putNumericNetwork(v); } - void putLong(int64_t v) { putNumeric(v); } void getDoubleNetwork(double & v) { getNumericNetwork(v); } - void getDouble(double& v) { getNumeric(v); } - void putDoubleNetwork(double v) { putNumericNetwork(v); } - void putDouble(double v) { putNumeric(v); } - - private: - void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline,noreturn)); - uint8_t peekByte() const { return *getBufferAtPos(); } - -#if defined(__i386__) || defined(__x86_64__) - - template<typename T> - void putDoubleLongNetwork(T val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - putByte(data[i]); - } - } + void getBytes(void *buffer, uint32_t count); +private: template<typename T> - void getDoubleLongNetwork(T &val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } + void getDoubleLongNetwork(T &val); - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - getByte(data[i]); - } - } -#else - #error "getDoubleLongNetwork is undefined for this arcitecture" -#endif + void incPosNoCheck(uint32_t pos) { _pos += pos; } - public: - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - */ - void putInt2_4_8Bytes(int64_t number) { - putInt2_4_8Bytes(number, 0); - } - - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - * @param len if non-zero, force writing number using len bytes, possibly - * with truncation - */ - void putInt2_4_8Bytes(int64_t number, size_t len); - - /** - * Reads a 62-bit positive integer from the buffer, which was written using - * 2, 4, or 8 bytes. - * - * @param v the integer read - */ - void getInt2_4_8Bytes(int64_t & v); - - /** - * Computes the size used for storing the given integer using 2, 4 or 8 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 2, 4 or 8 - */ - static size_t getSerializedSize2_4_8Bytes(int64_t number); - - /** - * Writes a 30-bit positive integer to the buffer, using 1, 2, or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_2_4Bytes(int32_t number); - - /** - * Reads a 30-bit positive integer from the buffer, which was written using - * 1, 2, or 4 bytes. - * - * @param v the integer read - */ - void getInt1_2_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1, 2 or 4 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1, 2 or 4 - */ - static size_t getSerializedSize1_2_4Bytes(int32_t number); - - /** - * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_4Bytes(int32_t number); - - /** - * Reads a 31-bit positive integer from the buffer, which was written using - * 1 or 4 bytes. - * - * @param v the integer read - */ - void getInt1_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1 or 4 bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1 or 4 - */ - static size_t getSerializedSize1_4Bytes(int32_t number); - - /** - * Writes a 8 bit integer to the buffer at the current position, and - * increases the positition accordingly. - * - * @param val the int to store - * @return True if the value could be stored, false if end of buffer is - * reached - */ - void getChar(char & val) { unsigned char t;getByte(t); val=t; } - void putChar(char val) { putByte(static_cast<unsigned char>(val)); } - - /** - * Reads the given number of bytes into the given pointer, and updates the - * positition accordingly - * - * @param buffer where to store the bytes - * @param count number of bytes to read - * @return True if all the bytes could be read, false if end of - * buffer is reached - */ - void getBytes(void *buffer, size_t count); - - /** - * Writes the given number of bytes into the ByteBuffer at the current - * position, and updates the positition accordingly - * - * @param buf the bytes to store - * @param count number of bytes to store - */ - void putBytes(const void *buf, size_t count); - - /** Debug */ - void dump() const; - - class BufferHolder : public vespalib::ReferenceCounter - { - private: - BufferHolder(const BufferHolder &); - BufferHolder& operator=(const BufferHolder &); - - public: - BufferHolder(vespalib::alloc::Alloc buffer); - virtual ~BufferHolder(); - - vespalib::alloc::Alloc _buffer; - }; - - ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit); - - void set(BufferHolder* buf, size_t pos, size_t len, size_t limit); - -private: - char * _buffer; - size_t _len; - size_t _pos; - size_t _limit; - mutable BufferHolder * _bufHolder; + const char * _buffer; + const uint32_t _len; + uint32_t _pos; vespalib::alloc::Alloc _ownedBuffer; -public: - - std::string toString(); - - void swap(ByteBuffer& other); - - void cleanUp(); }; } // document diff --git a/document/src/vespa/document/util/serializable.cpp b/document/src/vespa/document/util/serializable.cpp deleted file mode 100644 index 32c7bef90f0..00000000000 --- a/document/src/vespa/document/util/serializable.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "serializable.h" -#include "bufferexceptions.h" -#include "serializableexceptions.h" -#include "bytebuffer.h" - -namespace document { - -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Serializable, vespalib::Identifiable); -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Deserializable, Serializable); -VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(SerializeException); - -std::unique_ptr<ByteBuffer> Serializable::serialize() const -{ - size_t len = getSerializedSize(); - std::unique_ptr<ByteBuffer> retVal(new ByteBuffer(len)); - serialize(*retVal.get()); - return retVal; -} - -DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -DeserializeException::DeserializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -SerializeException::SerializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -SerializeException::SerializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -void -Serializable::serialize(ByteBuffer& buffer) const { - int pos = buffer.getPos(); - try{ - onSerialize(buffer); - } catch (...) { - buffer.setPos(pos); - throw; - } -} - -void -Deserializable::deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) { - int pos = buffer.getPos(); - try { - onDeserialize(repo, buffer); - } catch (const DeserializeException &) { - buffer.setPos(pos); - throw; - } catch (const BufferOutOfBoundsException &) { - buffer.setPos(pos); - throw; - } -} -} diff --git a/document/src/vespa/document/util/serializable.h b/document/src/vespa/document/util/serializable.h deleted file mode 100644 index 8818feceb9a..00000000000 --- a/document/src/vespa/document/util/serializable.h +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file serializable.h - * @ingroup document - * - * @brief Interfaces to be used for serializing of objects. - * - * @author Thomas F. Gundersen, H�kon Humberset - * @date 2004-03-15 - * @version $Id$ - */ - -#pragma once - -#include <vespa/vespalib/objects/cloneable.h> -#include <vespa/vespalib/objects/identifiable.h> - -#include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/identifiableid.h> - -namespace document { -class DocumentTypeRepo; - -/** - * Base class for classes that can be converted into a bytestream, - * normally used later to create a similar instance. - */ - -class Serializable : public vespalib::Identifiable -{ -protected: - virtual void onSerialize(ByteBuffer& buffer) const = 0; -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Serializable); - - virtual ~Serializable() {} - - /** - * @return An upper limit to how many bytes serialization of this instance - * need, providing instance is not altered before serialization. - */ - virtual size_t getSerializedSize() const = 0; - - /** - * Serializes the instance into the buffer given. Use getSerializedSize() - * before calling this method to be sure buffer is big enough. - * On success, the given buffers position will be just past the serialized - * version of this instance, on failure, position will be reset to whatever - * it was prior to calling this function. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - void serialize(ByteBuffer& buffer) const; - - /** - * Creates a bytebuffer with enough space to serialize this instance - * and serialize this instance into it. - * - * @return The created bytebuffer, positioned after the serialization. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - std::unique_ptr<ByteBuffer> serialize() const; -}; - -/** - * Base class for instances that can be overwritten from a bytestream, - * given that the bytestream is created from a similar instance. - */ -class Deserializable : public vespalib::Cloneable, public Serializable -{ -protected: - virtual void onDeserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) = 0; - -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Deserializable); - virtual ~Deserializable() {} - - /** - * Overwrite this object with the object represented by the given - * bytestream. On success, buffer will be positioned after the bytestream - * representing the instance we've just deserialized, on failure, bytebuffer - * will be pointing to where it was pointing before calling this function. - * - * @throw DeserializeException If read data doesn't represent a legal object - * of this type. - * @throw BufferOutOfBoundsException If instance wants to read more data - * than is available in the buffer. - */ - void deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer); -}; - -} - diff --git a/document/src/vespa/document/util/serializableexceptions.cpp b/document/src/vespa/document/util/serializableexceptions.cpp new file mode 100644 index 00000000000..e80e38015e8 --- /dev/null +++ b/document/src/vespa/document/util/serializableexceptions.cpp @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serializableexceptions.h" + +namespace document { + +VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); + +DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, location, 1) +{ +} + +DeserializeException::DeserializeException( + const vespalib::string& msg, const vespalib::Exception& cause, + const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) +{ +} + +} diff --git a/document/src/vespa/document/util/serializableexceptions.h b/document/src/vespa/document/util/serializableexceptions.h index fcfed810bfc..1b692aa27b7 100644 --- a/document/src/vespa/document/util/serializableexceptions.h +++ b/document/src/vespa/document/util/serializableexceptions.h @@ -24,12 +24,4 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(DeserializeException) }; -class SerializeException : public vespalib::IoException { -public: - SerializeException(const vespalib::string& msg, const vespalib::string& location = ""); - SerializeException(const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location = ""); - VESPA_DEFINE_EXCEPTION_SPINE(SerializeException) -}; - } |