diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-01-21 20:57:49 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-01-21 20:57:49 +0100 |
| commit | c9db06db7b01a2da9228c38783fd9f95c4fc7bba (patch) | |
| tree | 715ee5ac9cc3f925e1a72d6ba7db54ba2510cc79 | |
| parent | 85b84fc8e4db0976a6e3e9b97b2e4822e8764ff4 (diff) | |
| parent | aa0bfb11d05381a555c197a2ab26ee117c0758e6 (diff) | |
Merge pull request #11822 from vespa-engine/balder/reduce-bytebuffer-exposure
Balder/reduce bytebuffer exposure
107 files changed, 815 insertions, 2967 deletions
diff --git a/document/src/tests/arrayfieldvaluetest.cpp b/document/src/tests/arrayfieldvaluetest.cpp index 014884e44cd..0bdf51194fb 100644 --- a/document/src/tests/arrayfieldvaluetest.cpp +++ b/document/src/tests/arrayfieldvaluetest.cpp @@ -15,9 +15,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -53,40 +52,39 @@ TEST(ArrayFieldValueTest, testArray) EXPECT_EQ(IntFieldValue(3), (IntFieldValue&) value[2]); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream stream(value.serialize()); ArrayFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(stream, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By index - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.remove(1); EXPECT_TRUE(!value2.contains(IntFieldValue(2))); EXPECT_EQ(size_t(2), value2.size()); // By value - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // By value with multiple present - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.add(IntFieldValue(1)); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + stream.rp(0); + deserialize(stream, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp index d75a16ea1a6..7735ccdbb1d 100644 --- a/document/src/tests/documenttestcase.cpp +++ b/document/src/tests/documenttestcase.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocman.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/document/datatype/annotationreferencedatatype.h> #include <vespa/document/fieldvalue/iteratorhandler.h> #include <vespa/document/repo/configbuilder.h> @@ -9,6 +8,9 @@ #include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> @@ -28,10 +30,14 @@ using namespace fieldvalue; TEST(DocumentTest, testSizeOf) { + EXPECT_EQ(24u, sizeof(std::vector<char>)); + EXPECT_EQ(24u, sizeof(vespalib::alloc::Alloc)); + EXPECT_EQ(40u, sizeof(ByteBuffer)); + EXPECT_EQ(32u, sizeof(vespalib::GrowableByteBuffer)); EXPECT_EQ(88ul, sizeof(IdString)); EXPECT_EQ(104ul, sizeof(DocumentId)); - EXPECT_EQ(208ul, sizeof(Document)); - EXPECT_EQ(72ul, sizeof(StructFieldValue)); + EXPECT_EQ(200ul, sizeof(Document)); + EXPECT_EQ(64ul, sizeof(StructFieldValue)); EXPECT_EQ(24ul, sizeof(StructuredFieldValue)); EXPECT_EQ(64ul, sizeof(SerializableArray)); } @@ -63,7 +69,7 @@ TEST(DocumentTest, testFieldPath) class Handler : public fieldvalue::IteratorHandler { public: Handler(); - ~Handler(); + ~Handler() override; const std::string & getResult() const { return _result; } private: void onPrimitive(uint32_t, const Content&) override { @@ -386,13 +392,13 @@ TEST(DocumentTest, testSimpleUsage) EXPECT_EQ(1, value.getValue(intF)->getAsInt()); EXPECT_EQ(2, value.getValue(longF)->getAsInt()); - // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + // Serialize & equality + nbostream buffer; + value.serialize(buffer); Document value2(*repo.getDocumentType("test"), DocumentId("id::test:n=3:foo")); EXPECT_TRUE(value != value2); - value2.deserialize(repo, *buffer); + value2.deserialize(repo, buffer); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); EXPECT_EQ(DocumentId("id:ns:test::1"), value2.getId()); @@ -400,15 +406,15 @@ TEST(DocumentTest, testSimpleUsage) // Various ways of removing { // By value - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - value2.deserialize(repo, *buffer); + buffer.rp(0); + value2.deserialize(repo, buffer); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); @@ -561,29 +567,31 @@ TEST(DocumentTest, testReadSerializedFile) int fd = open(TEST_PATH("data/serializejava.dat").c_str(), O_RDONLY); size_t len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != (ssize_t)len) { + if (read(fd, buf.get(), len) != (ssize_t)len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); verifyJavaDocument(doc2); - EXPECT_EQ(len, buf2->getPos()); - EXPECT_TRUE(memcmp(buf2->getBuffer(), buf.getBuffer(), buf2->getPos()) == 0); + EXPECT_TRUE(buf2.empty()); + buf2.rp(0); + EXPECT_EQ(len, buf2.size()); + EXPECT_TRUE(memcmp(buf2.peek(), buf.get(), buf2.size()) == 0); doc2.setValue("stringfield", StringFieldValue("hei")); - std::unique_ptr<ByteBuffer> buf3 = doc2.serialize(); - EXPECT_TRUE(len != buf3->getPos()); + nbostream buf3 = doc2.serialize(); + EXPECT_TRUE(len != buf3.size()); } TEST(DocumentTest, testReadSerializedFileCompressed) @@ -595,14 +603,15 @@ TEST(DocumentTest, testReadSerializedFileCompressed) int fd = open(TEST_PATH("data/serializejava-compressed.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); verifyJavaDocument(doc); } @@ -706,26 +715,25 @@ TEST(DocumentTest,testReadSerializedAllVersions) // you can copy this current to new test for new version) { //doc.setCompression(CompressionConfig(CompressionConfig::NONE, 0, 0)); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_EQ(buf->getLength(), buf->getPos()); + nbostream buf = doc.serialize(); int fd = open(TEST_PATH("data/document-cpp-currentversion-uncompressed.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); } { CompressionConfig oldCfg(doc.getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - EXPECT_TRUE(buf->getPos() <= buf->getLength()); + nbostream buf = doc.serialize(); + EXPECT_TRUE(buf.size() <= buf.capacity()); int fd = open(TEST_PATH("data/document-cpp-currentversion-lz4-9.dat").c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); EXPECT_TRUE(fd > 0); - size_t len = write(fd, buf->getBuffer(), buf->getPos()); - EXPECT_EQ(buf->getPos(), len); + size_t len = write(fd, buf.peek(), buf.size()); + EXPECT_EQ(buf.size(), len); close(fd); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(oldCfg); } @@ -745,14 +753,15 @@ TEST(DocumentTest,testReadSerializedAllVersions) } int fd = open(tests[i]._dataFile.c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { - throw vespalib::Exception("read failed"); - } + if (read(fd, buf.get(), len) != len) { + throw vespalib::Exception("read failed"); + } close(fd); - Document doc(repo, buf); + nbostream stream(buf.get(), len); + Document doc(repo, stream); IntFieldValue intVal; EXPECT_TRUE(doc.getValue(doc.getField("intfield"), intVal)); @@ -805,27 +814,14 @@ TEST(DocumentTest,testReadSerializedAllVersions) EXPECT_EQ(199, wset.get(StringFieldValue("Weighted 1"))); // Check that serialization doesn't cause any problems. - std::unique_ptr<ByteBuffer> buf2 = doc.serialize(); - buf2->flip(); + nbostream buf2 = doc.serialize(); - Document doc2(repo, *buf2); + Document doc2(repo, buf2); } } size_t getSerializedSize(const Document &doc) { - return doc.serialize()->getLength(); -} - -size_t getSerializedSizeHeader(const Document &doc) { - nbostream stream; - doc.serializeHeader(stream); - return stream.size(); -} - -size_t getSerializedSizeBody(const Document &doc) { - nbostream stream; - doc.serializeBody(stream); - return stream.size(); + return doc.serialize().size(); } TEST(DocumentTest, testGenerateSerializedFile) @@ -864,30 +860,21 @@ TEST(DocumentTest, testGenerateSerializedFile) map.put(StringFieldValue("foo2"), StringFieldValue("bar2")); doc.setValue("mapfield", map); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); + nbostream buf = doc.serialize(); const std::string serializedDir = TEST_PATH("../test/document/"); int fd = open((serializedDir + "/serializecpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.peek(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); - ByteBuffer hBuf(getSerializedSizeHeader(doc)); + vespalib::nbostream hBuf; doc.serializeHeader(hBuf); fd = open((serializedDir + "/serializecppsplit_header.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, hBuf.getBuffer(), hBuf.getPos()) != (ssize_t)hBuf.getPos()) { - throw vespalib::Exception("write failed"); - } - close(fd); - - ByteBuffer bBuf(getSerializedSizeBody(doc)); - doc.serializeBody(bBuf); - fd = open((serializedDir+ "/serializecppsplit_body.dat").c_str(), - O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, bBuf.getBuffer(), bBuf.getPos()) != (ssize_t)bBuf.getPos()) { + if (write(fd, hBuf.peek(), hBuf.size()) != (ssize_t)hBuf.size()) { throw vespalib::Exception("write failed"); } close(fd); @@ -895,62 +882,30 @@ TEST(DocumentTest, testGenerateSerializedFile) CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc.getType().getFieldsType()).setCompressionConfig(newCfg); - ByteBuffer lz4buf(getSerializedSize(doc)); - - doc.serialize(lz4buf); - lz4buf.flip(); + nbostream lz4buf = doc.serialize(); fd = open((serializedDir + "/serializecpp-lz4-level9.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, lz4buf.getBufferAtPos(), lz4buf.getRemaining()) != (ssize_t)lz4buf.getRemaining()) { + if (write(fd, lz4buf.data(), lz4buf.size()) != (ssize_t)lz4buf.size()) { throw vespalib::Exception("write failed"); } close(fd); } -TEST(DocumentTest, testGetURIFromSerialized) -{ - TestDocRepo test_repo; - Document doc(*test_repo.getDocumentType("testdoctype1"), DocumentId("id:ns:testdoctype1::1")); - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - EXPECT_EQ( - vespalib::string("id:ns:testdoctype1::1"), - Document::getIdFromSerialized(*serialized).toString()); - - EXPECT_EQ(vespalib::string("testdoctype1"), - Document::getDocTypeFromSerialized( - test_repo.getTypeRepo(), - *serialized)->getName()); - } - - { - std::unique_ptr<ByteBuffer> serialized = doc.serialize(); - serialized->flip(); - - Document doc2(test_repo.getTypeRepo(), *serialized, false, NULL); - EXPECT_EQ(vespalib::string("id:ns:testdoctype1::1"), doc2.getId().toString()); - EXPECT_EQ(vespalib::string("testdoctype1"), doc2.getType().getName()); - } -} - TEST(DocumentTest, testBogusserialize) { TestDocRepo test_repo; try { - auto buf = std::make_unique<ByteBuffer>("aoifjweprjwoejr203r+2+4r823++!",100); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("aoifjweprjwoejr203r+2+4r823++!",100); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing bogus data"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Unrecognized serialization version")); } try { - auto buf = std::make_unique<ByteBuffer>("",0); - Document doc(test_repo.getTypeRepo(), *buf); + nbostream stream("",0); + Document doc(test_repo.getTypeRepo(), stream); FAIL() << "Failed to throw exception deserializing empty buffer"; } catch (DeserializeException& e) { EXPECT_THAT(e.what(), HasSubstr("Buffer out of bounds")); @@ -967,24 +922,23 @@ TEST(DocumentTest, testCRC32) uint32_t crc = doc.calculateChecksum(); EXPECT_EQ(3987392271u, crc); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); + nbostream buf = doc.serialize(); int pos = 30; // Corrupt serialization. - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; // Create document. Byte corrupted above is in data area and // shouldn't fail deserialization. try { - Document doc2(test_repo.getTypeRepo(), *buf); - buf->setPos(0); + Document doc2(test_repo.getTypeRepo(), buf); + buf.rp(0); EXPECT_TRUE(crc != doc2.calculateChecksum()); } catch (document::DeserializeException& e) { EXPECT_TRUE(false); } // Return original value and retry - buf->getBuffer()[pos] ^= 72; + const_cast<char *>(buf.peek())[pos] ^= 72; /// \todo TODO (was warning): Cannot test for in memory representation altered, as there is no syntax for getting internal refs to data from document. Add test when this is added. } @@ -1001,13 +955,12 @@ TEST(DocumentTest, testHasChanged) // Still changed after setting a value of course. EXPECT_TRUE(doc.hasChanged()); - std::unique_ptr<ByteBuffer> buf = doc.serialize(); - buf->flip(); - + nbostream buf; + doc.serialize(buf); // Setting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); EXPECT_TRUE(!doc2.hasChanged()); doc2.set("headerval", 13); @@ -1015,16 +968,16 @@ TEST(DocumentTest, testHasChanged) } // Overwriting a value in doc tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.set("hstringval", "bla bla bla bla bla"); EXPECT_TRUE(doc2.hasChanged()); } // Clearing value tags us changed. { - buf->setPos(0); - Document doc2(test_repo.getTypeRepo(), *buf); + buf.rp(0); + Document doc2(test_repo.getTypeRepo(), buf); doc2.clear(); EXPECT_TRUE(doc2.hasChanged()); @@ -1032,35 +985,6 @@ TEST(DocumentTest, testHasChanged) // Add more tests here when we allow non-const refs to internals } -TEST(DocumentTest, testSplitSerialization) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - doc->set("headerval", 50); - - ByteBuffer buf(getSerializedSizeHeader(*doc)); - doc->serializeHeader(buf); - buf.flip(); - - ByteBuffer buf2(getSerializedSizeBody(*doc)); - doc->serializeBody(buf2); - buf2.flip(); - - EXPECT_EQ(size_t(65), buf.getLength()); - EXPECT_EQ(size_t(73), buf2.getLength()); - - Document headerDoc(testDocMan.getTypeRepo(), buf); - EXPECT_TRUE(headerDoc.hasValue("headerval")); - EXPECT_TRUE(!headerDoc.hasValue("content")); - - buf.setPos(0); - Document fullDoc(testDocMan.getTypeRepo(), buf, buf2); - EXPECT_TRUE(fullDoc.hasValue("headerval")); - EXPECT_TRUE(fullDoc.hasValue("content")); - - EXPECT_EQ(*doc, fullDoc); -} - TEST(DocumentTest, testSliceSerialize) { // Test that document doesn't need its own bytebuffer, such that we @@ -1076,17 +1000,15 @@ TEST(DocumentTest, testSliceSerialize) val.add(RawFieldValue("hei der", 7)); doc2->setValue(doc2->getField("rawarray"), val); - ByteBuffer buf(getSerializedSize(*doc) + getSerializedSize(*doc2)); - doc->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + nbostream buf = doc->serialize(); + EXPECT_EQ(getSerializedSize(*doc), buf.size()); doc2->serialize(buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); - buf.flip(); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.size()); Document doc3(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc), buf.rp()); Document doc4(testDocMan.getTypeRepo(), buf); - EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.getPos()); + EXPECT_EQ(getSerializedSize(*doc) + getSerializedSize(*doc2), buf.rp()); EXPECT_EQ(*doc, doc3); EXPECT_EQ(*doc2, doc4); @@ -1102,21 +1024,19 @@ TEST(DocumentTest, testCompression) doc->setValue("hstringval", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc->serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed = doc->serialize(); CompressionConfig oldCfg(doc->getType().getFieldsType().getCompressionConfig()); CompressionConfig newCfg(CompressionConfig::LZ4, 9, 95); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(newCfg); - std::unique_ptr<ByteBuffer> buf_lz4 = doc->serialize(); - buf_lz4->flip(); + nbostream buf_lz4 = doc->serialize(); const_cast<StructDataType &>(doc->getType().getFieldsType()).setCompressionConfig(oldCfg); - EXPECT_TRUE(buf_lz4->getRemaining() < buf_uncompressed->getRemaining()); + EXPECT_TRUE(buf_lz4.size() < buf_uncompressed.size()); - Document doc_lz4(testDocMan.getTypeRepo(), *buf_lz4); + Document doc_lz4(testDocMan.getTypeRepo(), buf_lz4); EXPECT_EQ(*doc, doc_lz4); } @@ -1136,10 +1056,10 @@ TEST(DocumentTest, testCompressionConfigured) for (int i = 0; i < 8; ++i) { bigString += bigString; } doc_uncompressed.setValue("stringfield", StringFieldValue(bigString)); - std::unique_ptr<ByteBuffer> buf_uncompressed = doc_uncompressed.serialize(); - buf_uncompressed->flip(); + nbostream buf_uncompressed; + doc_uncompressed.serialize(buf_uncompressed); - size_t uncompressedSize = buf_uncompressed->getRemaining(); + size_t uncompressedSize = buf_uncompressed.size(); DocumenttypesConfigBuilderHelper builder2; builder2.document(43, "serializetest", @@ -1151,22 +1071,22 @@ TEST(DocumentTest, testCompressionConfigured) 9, 99, 0)); DocumentTypeRepo repo2(builder2.config()); - Document doc(repo2, *buf_uncompressed); + Document doc(repo2, buf_uncompressed); - std::unique_ptr<ByteBuffer> buf_compressed = doc.serialize(); - buf_compressed->flip(); - size_t compressedSize = buf_compressed->getRemaining(); + nbostream buf_compressed; + doc.serialize(buf_compressed); + size_t compressedSize = buf_compressed.size(); EXPECT_TRUE(compressedSize < uncompressedSize); - Document doc2(repo2, *buf_compressed); + Document doc2(repo2, buf_compressed); - std::unique_ptr<ByteBuffer> buf_compressed2 = doc2.serialize(); - buf_compressed2->flip(); + nbostream buf_compressed2; + doc2.serialize(buf_compressed2); - EXPECT_EQ(compressedSize, buf_compressed2->getRemaining()); + EXPECT_EQ(compressedSize, buf_compressed2.size()); - Document doc3(repo2, *buf_compressed2); + Document doc3(repo2, buf_compressed2); EXPECT_EQ(doc2, doc_uncompressed); EXPECT_EQ(doc2, doc3); @@ -1198,59 +1118,31 @@ TEST(DocumentTest, testUnknownEntries) doc1.setValue(field3, IntFieldValue(3)); doc1.setValue(field4, IntFieldValue(4)); - uint32_t headerLen = getSerializedSizeHeader(doc1); - document::ByteBuffer header(headerLen); - doc1.serializeHeader(header); - header.flip(); - - uint32_t bodyLen = getSerializedSizeBody(doc1); - document::ByteBuffer body(bodyLen); - doc1.serializeBody(body); - body.flip(); - - uint32_t totalLen = getSerializedSize(doc1); - document::ByteBuffer total(totalLen); - doc1.serialize(total); - total.flip(); + vespalib::nbostream os; + doc1.serialize(os); Document doc2; - doc2.deserialize(repo, total); - - Document doc3; - doc3.deserializeHeader(repo, header); - doc3.deserializeBody(repo, body); + doc2.deserialize(repo, os); EXPECT_EQ(std::string( "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" "<int3>3</int3>\n" "<int4>4</int4>\n" "</document>"), doc2.toXml()); - EXPECT_EQ(std::string( - "<document documenttype=\"test\" documentid=\"id:ns:test::1\">\n" - "<int3>3</int3>\n" - "<int4>4</int4>\n" - "</document>"), doc3.toXml()); EXPECT_EQ(3, doc2.getValue(field3)->getAsInt()); EXPECT_EQ(4, doc2.getValue(field4)->getAsInt()); - EXPECT_EQ(3, doc3.getValue(field3)->getAsInt()); - EXPECT_EQ(4, doc3.getValue(field4)->getAsInt()); // The fields are actually accessible as long as you ask with field of // correct type. EXPECT_TRUE(doc2.hasValue(field1)); EXPECT_TRUE(doc2.hasValue(field2)); - EXPECT_TRUE(doc3.hasValue(field1)); - EXPECT_TRUE(doc3.hasValue(field2)); EXPECT_EQ(1, doc2.getValue(field1)->getAsInt()); EXPECT_EQ(2, doc2.getValue(field2)->getAsInt()); - EXPECT_EQ(1, doc3.getValue(field1)->getAsInt()); - EXPECT_EQ(2, doc3.getValue(field2)->getAsInt()); EXPECT_EQ(size_t(2), doc2.getSetFieldCount()); - EXPECT_EQ(size_t(2), doc3.getSetFieldCount()); } TEST(DocumentTest, testAnnotationDeserialization) @@ -1280,14 +1172,15 @@ TEST(DocumentTest, testAnnotationDeserialization) int fd = open(TEST_PATH("data/serializejavawithannotations.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - Document doc(repo, buf); + nbostream stream1(buf.get(), len); + Document doc(repo, stream1); StringFieldValue strVal; EXPECT_TRUE(doc.getValue(doc.getField("story"), strVal)); @@ -1326,14 +1219,6 @@ TEST(DocumentTest, testAnnotationDeserialization) EXPECT_EQ((int64_t)2384LL, longVal.getAsLong()); } -TEST(DocumentTest, testGetSerializedSize) -{ - TestDocMan testDocMan; - Document::UP doc = testDocMan.createDocument(); - - EXPECT_EQ(getSerializedSize(*doc), doc->getSerializedSize()); -} - TEST(DocumentTest, testDeserializeMultiple) { TestDocRepo testDocRepo; diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 47a529adfc8..5543cb48ba4 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -43,14 +43,13 @@ namespace document { namespace { -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } nbostream serialize(const ValueUpdate & update) @@ -83,25 +82,25 @@ void testRoundtripSerialize(const UpdateType& update, const DataType &type) { } void -writeBufferToFile(const ByteBuffer &buf, const vespalib::string &fileName) +writeBufferToFile(const nbostream &buf, const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::out | std::ios::binary); - file.write(buf.getBuffer(), buf.getPos()); + file.write(buf.data(), buf.size()); assert(file.good()); file.close(); } -ByteBuffer::UP +nbostream readBufferFromFile(const vespalib::string &fileName) { auto file = std::fstream(fileName, std::ios::in | std::ios::binary | std::ios::ate); auto size = file.tellg(); - auto result = std::make_unique<ByteBuffer>(size); file.seekg(0); - file.read(result->getBuffer(), size); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(size); + file.read(static_cast<char *>(buf.get()), size); assert(file.good()); file.close(); - return result; + return nbostream(std::move(buf), size); } } @@ -132,9 +131,8 @@ TEST(DocumentUpdateTest, testSimpleUsage) // Test that a document update can be serialized DocumentUpdate docUpdate(repo, *docType, DocumentId("id:ns:test::1")); docUpdate.addUpdate(fieldUpdateCopy); - ByteBuffer::UP docBuf = serializeHEAD(docUpdate); - docBuf->flip(); - auto docUpdateCopy(DocumentUpdate::createHEAD(repo, nbostream(docBuf->getBufferAtPos(), docBuf->getRemaining()))); + nbostream docBuf = serializeHEAD(docUpdate); + auto docUpdateCopy(DocumentUpdate::createHEAD(repo, docBuf)); // Create a test document Document doc(*docType, DocumentId("id:ns:test::1")); @@ -236,7 +234,7 @@ TEST(DocumentUpdateTest, testUpdateArray) // Create a document. TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); // Assign array field. ArrayFieldValue myarray(doc->getType().getField("tags").getDataType()); @@ -459,8 +457,7 @@ TEST(DocumentUpdateTest, testReadSerializedFile) const std::string file_name = "data/crossplatform-java-cpp-doctypes.cfg"; DocumentTypeRepo repo(readDocumenttypesConfig(file_name)); - auto buf = readBufferFromFile("data/serializeupdatejava.dat"); - nbostream is(buf->getBufferAtPos(), buf->getRemaining()); + auto is = readBufferFromFile("data/serializeupdatejava.dat"); DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, is)); DocumentUpdate& upd(*updp); @@ -539,8 +536,8 @@ TEST(DocumentUpdateTest, testGenerateSerializedFile) ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 2))) .addUpdate(MapValueUpdate(StringFieldValue("foo"), ArithmeticValueUpdate(ArithmeticValueUpdate::Mul, 2)))); - ByteBuffer::UP buf(serializeHEAD(upd)); - writeBufferToFile(*buf, "data/serializeupdatecpp.dat"); + nbostream buf(serializeHEAD(upd)); + writeBufferToFile(buf, "data/serializeupdatecpp.dat"); } @@ -549,7 +546,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) // Create a test document TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("headerval")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); // Assign a float value to an int field. DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); @@ -561,7 +558,7 @@ TEST(DocumentUpdateTest, testSetBadFieldTypes) update.applyTo(*doc); // Verify that the field is NOT set in the document. - EXPECT_EQ((document::FieldValue*)NULL, + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("headerval")).get()); } @@ -569,7 +566,7 @@ TEST(DocumentUpdateTest, testUpdateApplyNoParams) { TestDocMan docMan; Document::UP doc(docMan.createDocument()); - EXPECT_EQ((document::FieldValue*)NULL, doc->getValue(doc->getField("tags")).get()); + EXPECT_EQ((document::FieldValue*)nullptr, doc->getValue(doc->getField("tags")).get()); DocumentUpdate update(docMan.getTypeRepo(), *doc->getDataType(), doc->getId()); update.addUpdate(FieldUpdate(doc->getField("tags")).addUpdate(AssignValueUpdate())); @@ -1110,13 +1107,12 @@ struct TensorUpdateSerializeFixture { } void serializeUpdateToFile(const DocumentUpdate &update, const vespalib::string &fileName) { - ByteBuffer::UP buf = serializeHEAD(update); - writeBufferToFile(*buf, fileName); + nbostream buf = serializeHEAD(update); + writeBufferToFile(buf, fileName); } DocumentUpdate::UP deserializeUpdateFromFile(const vespalib::string &fileName) { - auto buf = readBufferFromFile(fileName); - nbostream stream(buf->getBufferAtPos(), buf->getRemaining()); + auto stream = readBufferFromFile(fileName); return DocumentUpdate::createHEAD(*repo, stream); } @@ -1196,10 +1192,9 @@ TEST(DocumentUpdateTest, testThatCreateIfNonExistentFlagIsSerializedAndDeseriali { CreateIfNonExistentFixture f; - ByteBuffer::UP buf(serializeHEAD(*f.update)); - buf->flip(); + nbostream buf(serializeHEAD(*f.update)); - DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), *buf); + DocumentUpdate::UP deserialized = DocumentUpdate::createHEAD(f.docMan.getTypeRepo(), buf); EXPECT_EQ(*f.update, *deserialized); EXPECT_TRUE(deserialized->getCreateIfNonExistent()); } @@ -1231,9 +1226,8 @@ TEST(DocumentUpdateTest, array_element_update_can_be_roundtrip_serialized) ArrayUpdateFixture f; auto buffer = serializeHEAD(*f.update); - buffer->flip(); - auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), *buffer); + auto deserialized = DocumentUpdate::createHEAD(f.doc_man.getTypeRepo(), buffer); EXPECT_EQ(*f.update, *deserialized); } diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index 82443f13716..213ac2e5432 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -19,6 +19,7 @@ #include <gtest/gtest.h> using vespalib::Identifiable; +using vespalib::nbostream; using namespace document::config_builder; namespace document { @@ -133,23 +134,21 @@ createTestDocument(const DocumentTypeRepo &repo) return doc; } -ByteBuffer::UP serializeHEAD(const DocumentUpdate & update) +nbostream +serializeHEAD(const DocumentUpdate & update) { vespalib::nbostream stream; VespaDocumentSerializer serializer(stream); serializer.writeHEAD(update); - ByteBuffer::UP retVal(new ByteBuffer(stream.size())); - retVal->putBytes(stream.peek(), stream.size()); - return retVal; + return stream; } void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { try{ - ByteBuffer::UP bb(serializeHEAD(a)); - bb->flip(); - DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, *bb)); + auto bb(serializeHEAD(a)); + DocumentUpdate::UP b(DocumentUpdate::createHEAD(repo, bb)); - EXPECT_EQ(size_t(0), bb->getRemaining()); + EXPECT_EQ(size_t(0), bb.size()); EXPECT_EQ(a.getId().toString(), b->getId().toString()); EXPECT_EQ(a.getUpdates().size(), b->getUpdates().size()); for (size_t i(0); i < a.getUpdates().size(); i++) { @@ -157,8 +156,7 @@ void testSerialize(const DocumentTypeRepo& repo, const DocumentUpdate& a) { const FieldUpdate & ub = b->getUpdates()[i]; EXPECT_EQ(&ua.getField(), &ub.getField()); - EXPECT_EQ(ua.getUpdates().size(), - ub.getUpdates().size()); + EXPECT_EQ(ua.getUpdates().size(), ub.getUpdates().size()); for (size_t j(0); j < ua.getUpdates().size(); j++) { EXPECT_EQ(ua.getUpdates()[j]->getType(), ub.getUpdates()[j]->getType()); } @@ -1073,14 +1071,14 @@ TEST_F(FieldPathUpdateTestCase, testReadSerializedFile) int fd = open(TEST_PATH("data/serialize-fieldpathupdate-java.dat").c_str(), O_RDONLY); int len = lseek(fd,0,SEEK_END); - ByteBuffer buf(len); + vespalib::alloc::Alloc buf = vespalib::alloc::Alloc::alloc(len); lseek(fd,0,SEEK_SET); - if (read(fd, buf.getBuffer(), len) != len) { + if (read(fd, buf.get(), len) != len) { throw vespalib::Exception("read failed"); } close(fd); - DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, buf)); + DocumentUpdate::UP updp(DocumentUpdate::createHEAD(repo, nbostream(std::move(buf), len))); DocumentUpdate& upd(*updp); DocumentUpdate::UP compare(createDocumentUpdateForSerialization(repo)); @@ -1094,11 +1092,11 @@ TEST_F(FieldPathUpdateTestCase, testGenerateSerializedFile) // Tests nothing, only generates a file for java test DocumentUpdate::UP upd(createDocumentUpdateForSerialization(repo)); - ByteBuffer::UP buf(serializeHEAD(*upd)); + nbostream buf(serializeHEAD(*upd)); int fd = open(TEST_PATH("data/serialize-fieldpathupdate-cpp.dat").c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644); - if (write(fd, buf->getBuffer(), buf->getPos()) != (ssize_t)buf->getPos()) { + if (write(fd, buf.data(), buf.size()) != (ssize_t)buf.size()) { throw vespalib::Exception("write failed"); } close(fd); diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp index 58592e50cfe..8a5daf05f05 100644 --- a/document/src/tests/primitivefieldvaluetest.cpp +++ b/document/src/tests/primitivefieldvaluetest.cpp @@ -14,9 +14,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -89,20 +88,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Serialization Type t; - std::unique_ptr<ByteBuffer> buf(smallest.serialize()); - buf->flip(); - deserialize(*buf, t); + nbostream buf(smallest.serialize()); + deserialize(buf, t); EXPECT_EQ(smallest, t); buf = medium1.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(medium1, t); EXPECT_EQ(medium2, t); buf = largest.serialize(); - buf->flip(); - deserialize(*buf, t); + deserialize(buf, t); EXPECT_EQ(largest, t); // Assignment @@ -160,20 +156,17 @@ void deserialize(const ByteBuffer &buffer, T &value) { // Test that a just deserialized value can be serialized again // (literals have lazy deserialization so behaves diff then value = "foo"; - std::unique_ptr<ByteBuffer> buf(value.serialize()); - buf->flip(); + nbostream buf(value.serialize()); Literal value2("Other"); - deserialize(*buf, value2); + deserialize(buf, value2); buf = value2.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(value, value2); // Verify that get value ref gives us ref within original bytebuffer // (operator== use above should not modify this) buf = value.serialize(); - buf->flip(); - deserialize(*buf, value2); + deserialize(buf, value2); EXPECT_EQ(size_t(3), value2.getValueRef().size()); // Zero termination diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 7feea4e51d2..f680b3e6cff 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -912,7 +912,7 @@ void DeserializedTensorDoc::setup(const DocumentTypeRepo &docTypeRepo, const vespalib::nbostream &blob) { vespalib::nbostream wrapStream(blob.peek(), blob.size()); - _doc = std::make_unique<Document>(docTypeRepo, wrapStream, nullptr); + _doc = std::make_unique<Document>(docTypeRepo, wrapStream); _fieldValue = _doc->getValue(tensor_field_name); } diff --git a/document/src/tests/structfieldvaluetest.cpp b/document/src/tests/structfieldvaluetest.cpp index 76cf065a36a..9cf4b38be91 100644 --- a/document/src/tests/structfieldvaluetest.cpp +++ b/document/src/tests/structfieldvaluetest.cpp @@ -28,10 +28,9 @@ protected: namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value, const FixedTypeRepo &repo) +void deserialize(nbostream & stream, T &value, const FixedTypeRepo &repo) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); } @@ -61,13 +60,11 @@ TEST_F(StructFieldValueTest, testEmptyStruct) StructFieldValue value(type); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value == value2); } @@ -101,14 +98,12 @@ TEST_F(StructFieldValueTest, testStruct) EXPECT_EQ(2, value.getValue(longF)->getAsInt()); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); - EXPECT_EQ(buffer->getLength(), buffer->getLimit()); StructFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2, repo); + deserialize(buffer, value2, repo); EXPECT_TRUE(value2.hasValue(intF)); EXPECT_EQ(value, value2); @@ -116,15 +111,15 @@ TEST_F(StructFieldValueTest, testStruct) // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.remove(intF); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(1), value2.getSetFieldCount()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2, repo); + buffer.rp(0); + deserialize(buffer, value2, repo); value2.clear(); EXPECT_TRUE(!value2.hasValue(intF)); EXPECT_EQ(size_t(0), value2.getSetFieldCount()); diff --git a/document/src/tests/testbytebuffer.cpp b/document/src/tests/testbytebuffer.cpp index 17807fb4ff5..b4ad558b4ad 100644 --- a/document/src/tests/testbytebuffer.cpp +++ b/document/src/tests/testbytebuffer.cpp @@ -3,12 +3,14 @@ #include <vespa/document/util/stringutil.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/document/fieldvalue/serializablearray.h> -#include <iostream> -#include <vespa/vespalib/util/macro.h> #include <vespa/document/util/bufferexceptions.h> +#include <vespa/vespalib/util/macro.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <gtest/gtest.h> + using namespace document; +using vespalib::GrowableByteBuffer; namespace { @@ -22,111 +24,28 @@ void assign(S &lhs, const S &rhs) TEST(ByteBuffer_Test, test_constructors) { - ByteBuffer* simple=new ByteBuffer(); - delete simple; - - ByteBuffer* less_simple=new ByteBuffer("hei",3); - EXPECT_TRUE(strcmp(less_simple->getBufferAtPos(),"hei")==0); - delete less_simple; -} - -TEST(ByteBuffer_Test, test_assignment_operator) -{ - try { - ByteBuffer b1; - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); - - ByteBuffer b2 = b1; - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - int test = 0; - b2.flip(); - b2.getInt(test); - EXPECT_EQ(1,test); - b2.getInt(test); - EXPECT_EQ(2,test); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - // Test Selfassignment == no change - // - assign(b2, b2); - - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ((size_t) 8,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - ByteBuffer b3; - // Empty - b2 = b3; - - EXPECT_EQ((size_t) 0,b2.getPos()); - EXPECT_EQ((size_t) 0,b2.getLength()); - EXPECT_EQ((size_t) 0,b2.getLimit()); - EXPECT_EQ((size_t) 0,b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } + ByteBuffer less_simple("hei",3); + EXPECT_TRUE(strcmp(less_simple.getBufferAtPos(),"hei")==0); } TEST(ByteBuffer_Test, test_copy_constructor) { try { - // Empty buffer first - ByteBuffer b1; - ByteBuffer b2(b1); - - EXPECT_EQ(b1.getPos(),b2.getPos()); - EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); - EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); - - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - - try { - ByteBuffer b1(100); - b1.putInt(1); - b1.putInt(2); + GrowableByteBuffer gb(100); + gb.putInt(1); + gb.putInt(2); + ByteBuffer b1(gb.getBuffer(), gb.position()); ByteBuffer b2(b1); EXPECT_EQ(b1.getPos(),b2.getPos()); EXPECT_EQ(b1.getLength(),b2.getLength()); - EXPECT_EQ(b1.getLimit(),b2.getLimit()); EXPECT_EQ(b1.getRemaining(),b2.getRemaining()); int test = 0; - b2.flip(); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(1,test); - b2.getInt(test); + b2.getIntNetwork(test); EXPECT_EQ(2,test); } catch (std::exception &e) { @@ -134,327 +53,6 @@ TEST(ByteBuffer_Test, test_copy_constructor) } } -TEST(ByteBuffer_Test, test_slice) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer* slice = new ByteBuffer; - slice->sliceFrom(*newBuf, 0,newBuf->getLength()); - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice->getBufferAtPos(),"hei der")==0); - - ByteBuffer* slice2 = new ByteBuffer; - slice2->sliceFrom(*slice, 4, slice->getLength()); - delete slice; - slice = NULL; - - EXPECT_TRUE(strcmp(slice2->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2->getBuffer(),"hei der")==0); - delete slice2; - slice2 = NULL; - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - ByteBuffer* slice3=new ByteBuffer; - ByteBuffer* slice4=new ByteBuffer; - - slice3->sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice4->sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice3->getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice3; - slice3 = NULL; - - EXPECT_TRUE(strcmp(slice4->getBuffer(),"hei der")==0); - - delete slice4; - slice4 = NULL; -} - -TEST(ByteBuffer_Test, test_slice2) -{ - ByteBuffer* newBuf=ByteBuffer::copyBuffer("hei der",8); - - ByteBuffer slice; - slice.sliceFrom(*newBuf, 0, newBuf->getLength()); - - delete newBuf; - newBuf = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"hei der")==0); - - ByteBuffer slice2; - slice2.sliceFrom(slice, 4, slice.getLength()); - - EXPECT_TRUE(strcmp(slice2.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); - - ByteBuffer* newBuf2=new ByteBuffer("hei der", 8); - - slice.sliceFrom(*newBuf2, 4, newBuf2->getLength()); - slice2.sliceFrom(*newBuf2, 0, newBuf2->getLength()); - delete newBuf2; - newBuf2 = NULL; - - EXPECT_TRUE(strcmp(slice.getBufferAtPos(),"der")==0); - EXPECT_TRUE(strcmp(slice2.getBuffer(),"hei der")==0); -} - - -TEST(ByteBuffer_Test, test_putGetFlip) -{ - ByteBuffer* newBuf=new ByteBuffer(100); - - try { - newBuf->putInt(10); - int test; - newBuf->flip(); - - newBuf->getInt(test); - EXPECT_TRUE(test==10); - - newBuf->clear(); - newBuf->putDouble(3.35); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==sizeof(double)); - double test2; - newBuf->getDouble(test2); - EXPECT_TRUE(test2==3.35); - - newBuf->clear(); - newBuf->putBytes("heisann",8); - newBuf->putInt(4); - EXPECT_TRUE(newBuf->getPos()==12); - EXPECT_TRUE(newBuf->getLength()==100); - newBuf->flip(); - EXPECT_TRUE(newBuf->getRemaining()==12); - - char testStr[12]; - newBuf->getBytes(testStr, 8); - EXPECT_TRUE(strcmp(testStr,"heisann")==0); - newBuf->getInt(test); - EXPECT_TRUE(test==4); - } catch (std::exception &e) { - FAIL() << "Unexpected exception at " << VESPA_STRLOC << ": \"" << e.what() << "\""; - } - delete newBuf; -} - - -TEST(ByteBuffer_Test, test_NumberEncodings) -{ - ByteBuffer* buf=new ByteBuffer(1024); - - // Check 0 - buf->putInt1_2_4Bytes(124); - buf->putInt2_4_8Bytes(124); - buf->putInt1_4Bytes(124); - // Check 1 - buf->putInt1_2_4Bytes(127); - buf->putInt2_4_8Bytes(127); - buf->putInt1_4Bytes(127); - // Check 2 - buf->putInt1_2_4Bytes(128); - buf->putInt2_4_8Bytes(128); - buf->putInt1_4Bytes(128); - // Check 3 - buf->putInt1_2_4Bytes(255); - buf->putInt2_4_8Bytes(255); - buf->putInt1_4Bytes(255); - // Check 4 - buf->putInt1_2_4Bytes(256); - buf->putInt2_4_8Bytes(256); - buf->putInt1_4Bytes(256); - // Check 5 - buf->putInt1_2_4Bytes(0); - buf->putInt2_4_8Bytes(0); - buf->putInt1_4Bytes(0); - // Check 6 - buf->putInt1_2_4Bytes(1); - buf->putInt2_4_8Bytes(1); - buf->putInt1_4Bytes(1); - - // Check 7 - try { - buf->putInt1_2_4Bytes(0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - buf->putInt2_4_8Bytes(0x7FFFFFFFll); - buf->putInt1_4Bytes(0x7FFFFFFF); - - try { - buf->putInt2_4_8Bytes(0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - buf->putInt1_2_4Bytes(0x7FFF); - // Check 8 - buf->putInt2_4_8Bytes(0x7FFFll); - buf->putInt1_4Bytes(0x7FFF); - buf->putInt1_2_4Bytes(0x7F); - // Check 9 - buf->putInt2_4_8Bytes(0x7Fll); - buf->putInt1_4Bytes(0x7F); - - try { - buf->putInt1_2_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-1); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt1_2_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - try { - buf->putInt1_4Bytes(-0x7FFFFFFF); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - try { - buf->putInt2_4_8Bytes(-0x7FFFFFFFFFFFFFFFll); - FAIL() << "Expected input out of range exception"; - } catch (InputOutOfRangeException& e) { } - - uint32_t endWritePos = buf->getPos(); - buf->setPos(0); - - int32_t tmp32; - int64_t tmp64; - - // Check 0 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)124, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(124, tmp32); - // Check 1 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)127, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(127, tmp32); - // Check 2 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)128, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(128, tmp32); - // Check 3 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)255, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(255, tmp32); - // Check 4 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)256, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(256, tmp32); - // Check 5 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0, tmp32); - // Check 6 - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)1, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(1, tmp32); - // Check 7 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFFFFFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFFFFFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - // Check 8 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7FFF, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7FFF, tmp32); - buf->getInt1_2_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - // Check 9 - buf->getInt2_4_8Bytes(tmp64); - EXPECT_EQ((int64_t)0x7F, tmp64); - buf->getInt1_4Bytes(tmp32); - EXPECT_EQ(0x7F, tmp32); - - uint32_t endReadPos = buf->getPos(); - EXPECT_EQ(endWritePos, endReadPos); - - delete buf; -} - -TEST(ByteBuffer_Test, test_NumberLengths) -{ - ByteBuffer b; - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_4Bytes(127)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(128)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(129)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(255)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(256)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_4Bytes(0x7FFFFFFF)); - - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(0)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(1)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(4)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(31)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(126)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize2_4_8Bytes(32767)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32768)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(32769)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(1030493)); - EXPECT_EQ((size_t) 4, b.getSerializedSize2_4_8Bytes(0x3FFFFFFF)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000000)); - EXPECT_EQ((size_t) 8, b.getSerializedSize2_4_8Bytes(0x40000001)); - - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(0)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(1)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(4)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(31)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(126)); - EXPECT_EQ((size_t) 1, b.getSerializedSize1_2_4Bytes(127)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(128)); - EXPECT_EQ((size_t) 2, b.getSerializedSize1_2_4Bytes(16383)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16384)); - EXPECT_EQ((size_t) 4, b.getSerializedSize1_2_4Bytes(16385)); -} - TEST(ByteBuffer_Test, test_SerializableArray) { SerializableArray array; diff --git a/document/src/tests/weightedsetfieldvaluetest.cpp b/document/src/tests/weightedsetfieldvaluetest.cpp index 1ec52791a4a..71dc9f1d1a9 100644 --- a/document/src/tests/weightedsetfieldvaluetest.cpp +++ b/document/src/tests/weightedsetfieldvaluetest.cpp @@ -16,9 +16,8 @@ namespace document { namespace { template <typename T> -void deserialize(const ByteBuffer &buffer, T &value) { +void deserialize(nbostream & stream, T &value) { uint16_t version = Document::getNewestSerializationVersion(); - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); DocumentTypeRepo repo; VespaDocumentDeserializer deserializer(repo, stream, version); deserializer.read(value); @@ -94,26 +93,25 @@ TEST(WeightedSetFieldValueTest, testWeightedSet) EXPECT_EQ(6, value.get(IntFieldValue(3))); // Serialize & equality - std::unique_ptr<ByteBuffer> buffer(value.serialize()); - buffer->flip(); + nbostream buffer(value.serialize()); WeightedSetFieldValue value2(type); EXPECT_TRUE(value != value2); - deserialize(*buffer, value2); + deserialize(buffer, value2); EXPECT_EQ(value, value2); // Various ways of removing { // By value - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); EXPECT_EQ(size_t(3), value2.size()); EXPECT_TRUE(value2.remove(IntFieldValue(1))); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(2), value2.size()); // Clearing all - buffer->setPos(0); - deserialize(*buffer, value2); + buffer.rp(0); + deserialize(buffer, value2); value2.clear(); EXPECT_TRUE(!value2.contains(IntFieldValue(1))); EXPECT_EQ(size_t(0), value2.size()); diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp index 29414c901f8..969684f104b 100644 --- a/document/src/vespa/document/fieldvalue/document.cpp +++ b/document/src/vespa/document/fieldvalue/document.cpp @@ -8,12 +8,10 @@ #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/serializableexceptions.h> -#include <vespa/document/base/exceptions.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/util/xmlstream.h> #include <sstream> -#include <limits> using vespalib::nbostream; using vespalib::make_string; @@ -24,10 +22,6 @@ using namespace vespalib::xml; namespace document { namespace { -bool isLegalVersion(uint16_t version) { - return (6 <= version) && (version <= 8); -} - void documentTypeError(vespalib::stringref name) __attribute__((noinline)); void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) __attribute__((noinline)); @@ -93,22 +87,13 @@ Document::Document(const DataType &type, DocumentId documentId) } } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserialize(repo, buffer); -} - void Document::setRepo(const DocumentTypeRepo& repo) { _fields.setRepo(repo); } -Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), +Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is) + : StructuredFieldValue(*DataType::DOCUMENT), _id(), _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), _lastModified(0) @@ -116,33 +101,6 @@ Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is, const deserialize(repo, is); } -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - if (!includeContent) { - const DocumentType *newDocType = deserializeDocHeaderAndType(repo, buffer, _id, static_cast<const DocumentType*>(anticipatedType)); - if (newDocType) { - setType(*newDocType); - } - } else { - deserialize(repo, buffer); - } -} - - -Document::Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType) - : StructuredFieldValue(anticipatedType ? verifyDocumentType(anticipatedType) : *DataType::DOCUMENT), - _id(), - _fields(static_cast<const DocumentType &>(getType()).getFieldsType()), - _lastModified(0) -{ - deserializeHeader(repo, header); - deserializeBody(repo, body); -} - Document::~Document() = default; const DocumentType& @@ -170,30 +128,6 @@ Document::hasChanged() const return _fields.hasChanged(); } -DocumentId -Document::getIdFromSerialized(ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - deserializeDocHeader(buf, retVal); - buf.setPos(position); - - return retVal; -} - -const DocumentType * -Document::getDocTypeFromSerialized(const DocumentTypeRepo& repo, ByteBuffer& buf) -{ - int position = buf.getPos(); - DocumentId retVal; - - const DocumentType *docType(deserializeDocHeaderAndType(repo, buf, retVal, nullptr)); - buf.setPos(position); - - return docType; -} - FieldValue& Document::assign(const FieldValue& value) { @@ -274,120 +208,11 @@ Document::calculateChecksum() const return calculator.checksum() ^ _fields.calculateChecksum(); } -const DocumentType * -Document::deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, DocumentId& id, - const DocumentType * docType) -{ - deserializeDocHeader(buffer, id); - - vespalib::stringref docTypeName(buffer.getBufferAtPos()); - buffer.incPos(docTypeName.size() + 1); // Skip 0-byte too - { - int16_t docTypeVersion; // version not supported anymore - buffer.getShortNetwork(docTypeVersion); - } - const DocumentType *docTypeNew = nullptr; - - if (! ((docType != nullptr) && (docType->getName() == docTypeName))) { - docTypeNew = repo.getDocumentType(docTypeName); - if (!docTypeNew) { - throw DocumentTypeNotFoundException(docTypeName, VESPA_STRLOC); - } - } - return docTypeNew; -} - -namespace { -[[noreturn]] void versionError(uint16_t version) __attribute__((noinline)); -[[noreturn]] void mainDocumentError(int64_t len) __attribute__((noinline)); -[[noreturn]] void notEnoughDocumentError(int32_t len, int64_t remaining) __attribute__((noinline)); - -void versionError(uint16_t version) { - throw DeserializeException(make_string( "Unrecognized serialization version %d", version), VESPA_STRLOC); -} - -void mainDocumentError(int64_t len) { - throw DeserializeException(make_string( - "Document lengths past %i is not supported. Corrupt data said length is %" PRId64 " bytes", - std::numeric_limits<int>::max(), len), VESPA_STRLOC); -} - -void notEnoughDocumentError(int32_t len, int64_t remaining) { - throw DeserializeException(make_string( "Buffer said document length is %d bytes, but only %" PRId64 " bytes remain in buffer", len, remaining)); -} - -} - -void -Document::deserializeDocHeader(ByteBuffer& buffer, DocumentId& id) { - int16_t version; - int32_t len; - buffer.getShortNetwork(version); - - if ( ! isLegalVersion(version) ) { - versionError(version); - } else if (version < 7) { - int64_t tmpLen = 0; - buffer.getInt2_4_8Bytes(tmpLen); - if (tmpLen > std::numeric_limits<int>::max()) { - mainDocumentError(tmpLen); - } else { - len = static_cast<int32_t>(tmpLen) - - ByteBuffer::getSerializedSize2_4_8Bytes(tmpLen) - - sizeof(uint16_t); - } - } else { - buffer.getIntNetwork(len); - } - - if (len > (long)buffer.getRemaining()) { - notEnoughDocumentError(len, buffer.getRemaining()); - } else { - nbostream stream(buffer.getBufferAtPos(), buffer.getRemaining()); - id = DocumentId(stream); - buffer.incPos(stream.rp()); - unsigned char contentByte; - buffer.getByte(contentByte); - } -} - -void Document::serializeHeader(ByteBuffer& buffer) const { - nbostream stream; - serializeHeader(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - void Document::serializeHeader(nbostream& stream) const { VespaDocumentSerializer serializer(stream); serializer.write(*this, WITHOUT_BODY); } -void Document::serializeBody(ByteBuffer& buffer) const { - nbostream stream; - serializeBody(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -bool Document::hasBodyField() const { - for (document::StructuredFieldValue::const_iterator it(getFields().begin()), mt(getFields().end()); - it != mt; - ++it) - { - if ( ! it.field().isHeaderField() ) { - return true; - } - } - return false; -} - -void Document::serializeBody(nbostream& stream) const { - if (hasBodyField()) { - VespaDocumentSerializer serializer(stream); - serializer.write(_fields, BodyFields()); - } -} - void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os) { VespaDocumentDeserializer deserializer(repo, os, 0); try { @@ -397,38 +222,19 @@ void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & o } } -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& data) { - nbostream stream(data.getBufferAtPos(), data.getRemaining()); - deserialize(repo, stream); - data.incPos(data.getRemaining() - stream.size()); -} - -void Document::deserialize(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body) { +void Document::deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & header, vespalib::nbostream & body) { deserializeHeader(repo, header); deserializeBody(repo, body); } -void Document::deserializeHeader(const DocumentTypeRepo& repo, - ByteBuffer& header) { - nbostream stream(header.getBufferAtPos(), header.getRemaining()); +void Document::deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { VespaDocumentDeserializer deserializer(repo, stream, 0); deserializer.read(*this); - header.incPos(header.getRemaining() - stream.size()); } -void Document::deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body) { - nbostream body_stream(body.getBufferAtPos(), body.getRemaining()); - VespaDocumentDeserializer - body_deserializer(repo, body_stream, getFields().getVersion()); - body_deserializer.readStructNoReset(getFields()); - body.incPos(body.getRemaining() - body_stream.size()); -} - -size_t -Document::getSerializedSize() const -{ - // Temporary non-optimal (but guaranteed correct) implementation. - return serialize()->getLength(); +void Document::deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { + VespaDocumentDeserializer deserializer(repo, stream, getFields().getVersion()); + deserializer.readStructNoReset(getFields()); } StructuredFieldValue::StructuredIterator::UP diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h index 3e1ec0da3e6..f20ef969e0c 100644 --- a/document/src/vespa/document/fieldvalue/document.h +++ b/document/src/vespa/document/fieldvalue/document.h @@ -27,10 +27,11 @@ private: DocumentId _id; StructFieldValue _fields; - // To avoid having to return another container object out of docblocks - // the meta data has been added to document. This will not be serialized - // with the document and really doesn't belong here! + // To avoid having to return another container object out of docblocks + // the meta data has been added to document. This will not be serialized + // with the document and really doesn't belong here! int64_t _lastModified; + public: typedef std::unique_ptr<Document> UP; typedef std::shared_ptr<Document> SP; @@ -42,13 +43,7 @@ public: Document(); Document(const Document&); Document(const DataType &, DocumentId id); - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, const DataType *anticipatedType = nullptr); - Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream, const DataType *anticipatedType = nullptr); - /** - Constructor to deserialize only document and type from a buffer. Only relevant if includeContent is false. - */ - Document(const DocumentTypeRepo& repo, ByteBuffer& buffer, bool includeContent, const DataType *anticipatedType); - Document(const DocumentTypeRepo& repo, ByteBuffer& header, ByteBuffer& body, const DataType *anticipatedType = nullptr); + Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream); ~Document() override; void setRepo(const DocumentTypeRepo & repo); @@ -86,21 +81,6 @@ public: bool hasChanged() const override; - /** - * Returns a pointer to the Id of a serialized document, without performing - * the deserialization. buffer must point to the start position of the - * serialization. If the buffer doesn't have enough data remaining to have - * a legal Id in it, method returns NULL. - */ - static DocumentId getIdFromSerialized(ByteBuffer&); - - /** - * Returns a pointer to the document type of a serialized header, without - * performing the deserialization. Buffer must point to the start position - * of the serialization. - */ - static const DocumentType *getDocTypeFromSerialized(const DocumentTypeRepo&, ByteBuffer&); - // FieldValue implementation. FieldValue& assign(const FieldValue&) override; int compare(const FieldValue& other) const override; @@ -108,22 +88,12 @@ public: void printXml(XmlOutputStream& out) const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; - // Specialized serialization functions - void serializeHeader(ByteBuffer& buffer) const; + // Specialized serialization functions, Only used for testing legacy stuff void serializeHeader(vespalib::nbostream& stream) const; - void serializeBody(ByteBuffer& buffer) const; - void serializeBody(vespalib::nbostream& stream) const; - - /** Deserialize document contained in given bytebuffer. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& data); void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & os); /** Deserialize document contained in given bytebuffers. */ - void deserialize(const DocumentTypeRepo& repo, ByteBuffer& body, ByteBuffer& header); - void deserializeHeader(const DocumentTypeRepo& repo, ByteBuffer& header); - void deserializeBody(const DocumentTypeRepo& repo, ByteBuffer& body); - - size_t getSerializedSize() const; + void deserialize(const DocumentTypeRepo& repo, vespalib::nbostream & body, vespalib::nbostream & header); /** Undo fieldvalue's toXml override for document. */ std::string toXml() const { return toXml(""); } @@ -137,18 +107,14 @@ public: void setFieldValue(const Field& field, FieldValue::UP data) override; private: - bool hasBodyField() const; + void deserializeHeader(const DocumentTypeRepo& repo, vespalib::nbostream & header); + void deserializeBody(const DocumentTypeRepo& repo, vespalib::nbostream & body); bool hasFieldValue(const Field& field) const override { return _fields.hasValue(field); } void removeFieldValue(const Field& field) override { _fields.remove(field); } FieldValue::UP getFieldValue(const Field& field) const override { return _fields.getValue(field); } bool getFieldValue(const Field& field, FieldValue& value) const override { return _fields.getValue(field, value); } StructuredIterator::UP getIterator(const Field* first) const override; - - static void deserializeDocHeader(ByteBuffer& buffer, DocumentId& id); - static const DocumentType *deserializeDocHeaderAndType( - const DocumentTypeRepo& repo, ByteBuffer& buffer, - DocumentId& id, const DocumentType * docType); }; } // document diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index 999747688eb..7737d03b45a 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -35,22 +35,11 @@ void FieldValue::serialize(nbostream &stream) const { serializer.write(*this); } -void FieldValue::serialize(ByteBuffer& buffer) const { +nbostream +FieldValue::serialize() const { nbostream stream; serialize(stream); - buffer.putBytes(stream.peek(), stream.size()); -} - -std::unique_ptr<ByteBuffer> FieldValue::serialize() const { - nbostream stream; - serialize(stream); - - nbostream::Buffer buf; - stream.swap(buf); - size_t sz = buf.size(); - auto bb = std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); - bb->setPos(sz); - return bb; + return stream; } size_t @@ -58,7 +47,7 @@ FieldValue::hash() const { vespalib::nbostream os; serialize(os); - return vespalib::hashValue(os.c_str(), os.size()) ; + return vespalib::hashValue(os.data(), os.size()) ; } bool diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index a152f74cc09..9ed8d522b17 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -19,15 +19,11 @@ #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/util/polymorphicarraybase.h> -namespace vespalib { - class nbostream; -} +namespace vespalib { class nbostream; } namespace document { -namespace fieldvalue { - class IteratorHandler; -} +namespace fieldvalue { class IteratorHandler; } class ByteBuffer; class DataType; @@ -73,8 +69,7 @@ public: virtual bool isA(const FieldValue& other) const; void serialize(vespalib::nbostream &stream) const; - void serialize(ByteBuffer& buffer) const; - std::unique_ptr<ByteBuffer> serialize() const; + vespalib::nbostream serialize() const; /** * Compares this fieldvalue with another fieldvalue. diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index 5dfd8eff891..722904659ef 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -26,15 +26,30 @@ public: } -SerializableArray::Statistics SerializableArray::_stats; - SerializableArray::SerializableArray() : _serializedCompression(CompressionConfig::NONE), _uncompressedLength(0) { } -serializablearray::BufferMap & ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { +SerializableArray::SerializableArray(EntryMap entries, ByteBuffer::UP buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length) + : _entries(std::move(entries)), + _owned(), + _serializedCompression(comp_type) +{ + + if (CompressionConfig::isCompressed(_serializedCompression)) { + _compSerData = std::move(buffer); + _uncompressedLength = uncompressed_length; + } else { + _uncompressedLength = buffer->getRemaining(); + _uncompSerData = std::move(buffer); + } +} + +serializablearray::BufferMap & +ensure(std::unique_ptr<serializablearray::BufferMap> & owned) { if (!owned) { owned = std::make_unique<serializablearray::BufferMap>(); } @@ -45,8 +60,8 @@ SerializableArray::SerializableArray(const SerializableArray& other) : Cloneable(), _entries(other._entries), _owned(), - _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : NULL), - _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : NULL), + _uncompSerData(other._uncompSerData.get() ? new ByteBuffer(*other._uncompSerData) : nullptr), + _compSerData(other._compSerData.get() ? new ByteBuffer(*other._compSerData) : nullptr), _serializedCompression(other._serializedCompression), _uncompressedLength(other._uncompressedLength) { @@ -66,17 +81,6 @@ SerializableArray::SerializableArray(const SerializableArray& other) } } -void -SerializableArray::swap(SerializableArray& other) -{ - _entries.swap(other._entries); - _owned.swap(other._owned); - std::swap(_uncompSerData, other._uncompSerData); - std::swap(_compSerData, other._compSerData); - std::swap(_serializedCompression, other._serializedCompression); - std::swap(_uncompressedLength, other._uncompressedLength); -} - void SerializableArray::clear() { _entries.clear(); @@ -86,9 +90,7 @@ void SerializableArray::clear() _uncompressedLength = 0; } -SerializableArray::~SerializableArray() -{ -} +SerializableArray::~SerializableArray() = default; void SerializableArray::invalidate() @@ -102,7 +104,7 @@ SerializableArray::set(int id, ByteBuffer::UP buffer) maybeDecompress(); Entry e(id, buffer->getRemaining(), buffer->getBuffer()); ensure(_owned)[id] = std::move(buffer); - EntryMap::iterator it = find(id); + auto it = find(id); if (it == _entries.end()) { _entries.push_back(e); } else { @@ -139,7 +141,7 @@ SerializableArray::get(int id) const { vespalib::ConstBufferRef buf; if ( !maybeDecompressAndCatch() ) { - EntryMap::const_iterator found = find(id); + auto found = find(id); if (found != _entries.end()) { const Entry& entry = *found; @@ -168,7 +170,7 @@ void SerializableArray::clear(int id) { maybeDecompress(); - EntryMap::iterator it = find(id); + auto it = find(id); if (it != _entries.end()) { _entries.erase(it); if (_owned) { @@ -193,7 +195,7 @@ SerializableArray::deCompress() // throw (DeserializeException) _uncompSerData = std::move(_compSerData); LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); } else { - ByteBuffer::UP newSerialization(new ByteBuffer(_uncompressedLength)); + auto newSerialization = std::make_unique<ByteBuffer>(vespalib::alloc::Alloc::alloc(_uncompressedLength), _uncompressedLength); vespalib::DataBuffer unCompressed(newSerialization->getBuffer(), newSerialization->getLength()); unCompressed.clear(); try { @@ -210,35 +212,16 @@ SerializableArray::deCompress() // throw (DeserializeException) if (unCompressed.getDataLen() != (size_t)_uncompressedLength) { throw DeserializeException( - make_string("Did not decompress to the expected length: had %zu, wanted %d, got %zu", + make_string("Did not decompress to the expected length: had %u, wanted %d, got %zu", _compSerData->getRemaining(), _uncompressedLength, unCompressed.getDataLen()), VESPA_STRLOC); } assert(newSerialization->getBuffer() == unCompressed.getData()); - newSerialization->setLimit(_uncompressedLength); _uncompSerData = std::move(newSerialization); LOG_ASSERT(_uncompressedLength == _uncompSerData->getRemaining()); } } -void SerializableArray::assign(EntryMap & entries, - ByteBuffer::UP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length) -{ - _serializedCompression = comp_type; - - _entries.clear(); - _entries.swap(entries); - if (CompressionConfig::isCompressed(_serializedCompression)) { - _compSerData.reset(buffer.release()); - _uncompressedLength = uncompressed_length; - } else { - _uncompressedLength = buffer->getRemaining(); - _uncompSerData.reset(buffer.release()); - } -} - vespalib::compression::CompressionInfo SerializableArray::getCompressionInfo() const { return CompressionInfo(_uncompressedLength, _compSerData->getRemaining()); diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h index 2f7d65938aa..1e766599dff 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.h +++ b/document/src/vespa/document/fieldvalue/serializablearray.h @@ -26,7 +26,6 @@ namespace document { -class SerializableArrayIterator; class ByteBuffer; namespace serializablearray { @@ -36,25 +35,6 @@ namespace serializablearray { class SerializableArray : public vespalib::Cloneable { public: - // Counts set during serialization, in order to provide metrics for how - // often we use cached version, and how often we compress. - struct Statistics { - uint64_t _usedCachedSerializationCount; - uint64_t _compressedDocumentCount; - uint64_t _compressionDidntHelpCount; - uint64_t _uncompressableCount; - uint64_t _serializedUncompressed; - uint64_t _inputWronglySerialized; - - Statistics() - : _usedCachedSerializationCount(0), - _compressedDocumentCount(0), - _compressionDidntHelpCount(0), - _uncompressableCount(0), - _serializedUncompressed(0), - _inputWronglySerialized(0) {} - }; - /** * Contains the id of a field, the size and a buffer reference that is either * a relative offset to a common buffer, or the buffer itself it it is not. @@ -98,12 +78,6 @@ public: static const uint32_t ReservedId = 100; static const uint32_t ReservedIdUpper = 128; - -private: - static Statistics _stats; - -public: - static Statistics& getStatistics() { return _stats; } using CP = vespalib::CloneablePtr<SerializableArray>; using UP = std::unique_ptr<SerializableArray>; using ByteBufferUP = std::unique_ptr<ByteBuffer>; @@ -111,9 +85,9 @@ public: using CompressionInfo = vespalib::compression::CompressionInfo; SerializableArray(); - virtual ~SerializableArray(); - - void swap(SerializableArray& other); + SerializableArray(EntryMap entries, ByteBufferUP buffer, + CompressionConfig::Type comp_type, uint32_t uncompressed_length); + ~SerializableArray() override; /** * Stores a value in the array. @@ -141,9 +115,6 @@ public: /** @return Returns true if the given ID is Set in the array. */ bool has(int id) const; - /** @return Number of elements in array */ - bool hasAnyElems() const { return !_entries.empty(); } - /** * clears an attribute. * @@ -157,16 +128,6 @@ public: CompressionConfig::Type getCompression() const { return _serializedCompression; } CompressionInfo getCompressionInfo() const; - /** - * Sets the serialized data that is the basis for this object's - * content. This is used by deserialization. Any existing entries - * are cleared. - */ - void assign(EntryMap &entries, - ByteBufferUP buffer, - CompressionConfig::Type comp_type, - uint32_t uncompressed_length); - bool empty() const { return _entries.empty(); } const ByteBuffer* getSerializedBuffer() const { diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp index 87c8d09648c..b861537f6f7 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp @@ -48,15 +48,14 @@ StructFieldValue::Chunks::~Chunks() = default; void StructFieldValue::Chunks::push_back(SerializableArray::UP item) { - assert(_sz < 2); - _chunks[_sz++].reset(item.release()); + assert(size() < 2); + _chunks[size()] = std::move(item); } void StructFieldValue::Chunks::clear() { _chunks[0].reset(); _chunks[1].reset(); - _sz = 0; } const StructDataType & @@ -81,13 +80,11 @@ StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo, _doc_type = &repo.getDocumentType(); _version = version; - _chunks.push_back(std::make_unique<SerializableArray>()); - _chunks.back().assign(fm, std::move(buffer), comp_type, uncompressed_length); + _chunks.push_back(std::make_unique<SerializableArray>(std::move(fm), std::move(buffer), comp_type, uncompressed_length)); _hasChanged = false; } -bool StructFieldValue::serializeField(int field_id, uint16_t version, - FieldValueWriter &writer) const { +bool StructFieldValue::serializeField(int field_id, uint16_t version, FieldValueWriter &writer) const { if (version == _version) { for (int i = _chunks.size() - 1; i >= 0; --i) { vespalib::ConstBufferRef buf = _chunks[i].get(field_id); @@ -183,9 +180,9 @@ StructFieldValue::getFieldValue(const Field& field) const if (buf.size() != 0) { nbostream stream(buf.c_str(), buf.size()); FieldValue::UP value(field.getDataType().createFieldValue()); - if ((_repo == NULL) && (_doc_type != NULL)) { - std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); - createFV(*value, *tmpRepo, stream, *_doc_type, _version); + if ((_repo == nullptr) && (_doc_type != nullptr)) { + DocumentTypeRepo tmpRepo(*_doc_type); + createFV(*value, tmpRepo, stream, *_doc_type, _version); } else { createFV(*value, *_repo, stream, *_doc_type, _version); } @@ -216,7 +213,7 @@ StructFieldValue::getFieldValue(const Field& field, FieldValue& value) const vespalib::ConstBufferRef buf = getRawField(fieldId); if (buf.size() > 0) { nbostream_longlivedbuf stream(buf.c_str(), buf.size()); - if ((_repo == NULL) && (_doc_type != NULL)) { + if ((_repo == nullptr) && (_doc_type != nullptr)) { std::unique_ptr<const DocumentTypeRepo> tmpRepo(new DocumentTypeRepo(*_doc_type)); createFV(value, *tmpRepo, stream, *_doc_type, _version); } else { @@ -239,17 +236,29 @@ StructFieldValue::hasFieldValue(const Field& field) const return false; } +namespace { + +std::unique_ptr<ByteBuffer> +serializeDoc(const FieldValue & fv) { + nbostream stream = fv.serialize(); + nbostream::Buffer buf; + stream.swap(buf); + size_t sz = buf.size(); + return std::make_unique<ByteBuffer>(nbostream::Buffer::stealAlloc(std::move(buf)), sz); +} + +} void StructFieldValue::setFieldValue(const Field& field, FieldValue::UP value) { int fieldId = field.getId(); - std::unique_ptr<ByteBuffer> serialized(value->serialize()); - serialized->flip(); + + std::unique_ptr<ByteBuffer> serialized = serializeDoc(*value); if (_chunks.empty()) { - _chunks.push_back(SerializableArray::UP(new SerializableArray())); + _chunks.push_back(std::make_unique<SerializableArray>()); } - _chunks.back().set(fieldId, std::move(serialized)); + _chunks[0].set(fieldId, std::move(serialized)); _hasChanged = true; } @@ -274,7 +283,7 @@ StructFieldValue::clear() FieldValue& StructFieldValue::assign(const FieldValue& value) { - const StructFieldValue& other(dynamic_cast<const StructFieldValue&>(value)); + const auto & other(dynamic_cast<const StructFieldValue&>(value)); return operator=(other); } @@ -285,7 +294,7 @@ StructFieldValue::compare(const FieldValue& otherOrg) const if (comp != 0) { return comp; } - const StructFieldValue& other = static_cast<const StructFieldValue&>(otherOrg); + const auto & other = static_cast<const StructFieldValue&>(otherOrg); std::vector<int> a; getRawFieldIds(a); @@ -315,9 +324,9 @@ StructFieldValue::compare(const FieldValue& otherOrg) const uint32_t StructFieldValue::calculateChecksum() const { - ByteBuffer::UP buffer(serialize()); + nbostream buffer(serialize()); vespalib::crc_32_type calculator; - calculator.process_bytes(buffer->getBuffer(), buffer->getPos()); + calculator.process_bytes(buffer.peek(), buffer.size()); return calculator.checksum(); } @@ -386,7 +395,7 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { std::vector<int> _ids; std::vector<int>::iterator _cur; - FieldIterator(const StructFieldValue& s) + explicit FieldIterator(const StructFieldValue& s) : _struct(s), _ids(), _cur(_ids.begin()) @@ -412,7 +421,7 @@ struct StructFieldValue::FieldIterator : public StructuredIterator { LOG(debug, "struct data type: %s", _struct.getType().toString(true).c_str()); } } - return 0; + return nullptr; } }; @@ -421,10 +430,10 @@ StructFieldValue::getIterator(const Field* toFind) const { StructuredIterator::UP ret; - FieldIterator *fi = new FieldIterator(*this); + auto *fi = new FieldIterator(*this); ret.reset(fi); - if (toFind != NULL) { + if (toFind != nullptr) { fi->skipTo(toFind->getId()); } return ret; diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h index b5b15e9dfce..30500229813 100644 --- a/document/src/vespa/document/fieldvalue/structfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h @@ -27,19 +27,16 @@ class StructFieldValue : public StructuredFieldValue public: class Chunks { public: - Chunks() : _sz(0) { } + Chunks() { } ~Chunks(); SerializableArray & operator [] (size_t i) { return *_chunks[i]; } const SerializableArray & operator [] (size_t i) const { return *_chunks[i]; } VESPA_DLL_LOCAL void push_back(SerializableArray::UP item); - SerializableArray & back() { return *_chunks[_sz-1]; } - const SerializableArray & back() const { return *_chunks[_sz-1]; } - size_t size() const { return _sz; } - bool empty() const { return _sz == 0; } + size_t size() const { return _chunks[1] ? 2 : _chunks[0] ? 1 : 0; } + bool empty() const { return !_chunks[0]; } VESPA_DLL_LOCAL void clear(); private: SerializableArray::CP _chunks[2]; - size_t _sz; }; private: Chunks _chunks; diff --git a/document/src/vespa/document/serialization/util.h b/document/src/vespa/document/serialization/util.h index cbcc2c97017..05953f1de2b 100644 --- a/document/src/vespa/document/serialization/util.h +++ b/document/src/vespa/document/serialization/util.h @@ -97,32 +97,5 @@ void putInt2_4_8Bytes(Output &out, uint64_t val) { } } -inline uint32_t sizeOfInt1_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt1_2_4Bytes(uint32_t val) { - if (val < 0x80) { - return 1; - } else if (val < 0x4000) { - return 2; - } else { - return 4; - } -} - -inline uint32_t sizeOfInt2_4_8Bytes(uint64_t val) { - if (val < 0x8000) { - return 2; - } else if (val < 0x40000000) { - return 4; - } else { - return 8; - } -} } // namespace document diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 570a51907da..4d60befba52 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -293,10 +293,10 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { using vespalib::compression::compress; - vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); + vespalib::ConstBufferRef buf(stream.data(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, - vespalib::ConstBufferRef(stream.c_str(), stream.size()), + vespalib::ConstBufferRef(stream.data(), stream.size()), compressed_data, false); if (compressedType != config.type || ! compressionSufficient(config, stream.size(), compressed_data.getDataLen())) diff --git a/document/src/vespa/document/update/documentupdate.cpp b/document/src/vespa/document/update/documentupdate.cpp index b43b7a59c5b..f289e6a8f27 100644 --- a/document/src/vespa/document/update/documentupdate.cpp +++ b/document/src/vespa/document/update/documentupdate.cpp @@ -7,7 +7,6 @@ #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/util/bufferexceptions.h> -#include <vespa/document/util/bytebuffer.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> @@ -231,20 +230,18 @@ DocumentUpdate::serializeFlags(int size_) const } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, ByteBuffer& buffer) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream && stream) { - vespalib::nbostream is(buffer.getBufferAtPos(), buffer.getRemaining()); auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, is); - buffer.setPos(buffer.getPos() + is.rp()); + update->initHEAD(repo, std::move(stream)); return update; } DocumentUpdate::UP -DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream stream) +DocumentUpdate::createHEAD(const DocumentTypeRepo& repo, vespalib::nbostream & stream) { auto update = std::make_unique<DocumentUpdate>(); - update->initHEAD(repo, std::move(stream)); + update->initHEAD(repo, stream); return update; } diff --git a/document/src/vespa/document/update/documentupdate.h b/document/src/vespa/document/update/documentupdate.h index a672d028f35..fb3d6c0f7a3 100644 --- a/document/src/vespa/document/update/documentupdate.h +++ b/document/src/vespa/document/update/documentupdate.h @@ -32,6 +32,7 @@ namespace document { +class ByteBuffer; class Document; class VespaDocumentSerializer; /** @@ -50,8 +51,8 @@ public: /** * Create new style document update, possibly with field path updates. */ - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream stream); - static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, ByteBuffer & buffer); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + static DocumentUpdate::UP createHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); DocumentUpdate(); /** @@ -124,8 +125,8 @@ private: bool _needHardReserialize; int deserializeFlags(int sizeAndFlags); - void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream & stream); + void initHEAD(const DocumentTypeRepo & repo, vespalib::nbostream && stream); void deserializeBody(const DocumentTypeRepo &repo, vespalib::nbostream &stream); void lazyDeserialize(const DocumentTypeRepo & repo, vespalib::nbostream & stream); void ensureDeserialized() const; diff --git a/document/src/vespa/document/update/fieldpathupdate.h b/document/src/vespa/document/update/fieldpathupdate.h index 3e19420d0d0..0dbf8e5cc9e 100644 --- a/document/src/vespa/document/update/fieldpathupdate.h +++ b/document/src/vespa/document/update/fieldpathupdate.h @@ -7,7 +7,6 @@ namespace document { -class ByteBuffer; class DocumentTypeRepo; class Field; class FieldValue; @@ -35,7 +34,7 @@ public: using SP = std::shared_ptr<FieldPathUpdate>; using CP = vespalib::CloneablePtr<FieldPathUpdate>; - ~FieldPathUpdate(); + ~FieldPathUpdate() override; enum FieldPathUpdateType { Add = IDENTIFIABLE_CLASSID(AddFieldPathUpdate), diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index 8cb148abe25..48ca7bd36d7 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_library(document_util OBJECT SOURCES bytebuffer.cpp printable.cpp - serializable.cpp + serializableexceptions.cpp stringutil.cpp DEPENDS AFTER diff --git a/document/src/vespa/document/util/bufferexceptions.h b/document/src/vespa/document/util/bufferexceptions.h index 8a3215f6c79..aee7f3ae568 100644 --- a/document/src/vespa/document/util/bufferexceptions.h +++ b/document/src/vespa/document/util/bufferexceptions.h @@ -15,13 +15,5 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(BufferOutOfBoundsException) }; -class InputOutOfRangeException : public vespalib::IoException { -public: - InputOutOfRangeException(const vespalib::string& msg, - const vespalib::string& location = ""); - - VESPA_DEFINE_EXCEPTION_SPINE(InputOutOfRangeException) -}; - } diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp index ccbc2bc7790..0fbc6ca1c63 100644 --- a/document/src/vespa/document/util/bytebuffer.cpp +++ b/document/src/vespa/document/util/bytebuffer.cpp @@ -12,732 +12,160 @@ #include <sstream> #include <arpa/inet.h> -#define LOG_DEBUG1(a) -// Enable this macros instead to see what bytebuffer calls come -//#define LOG_DEBUG1(a) std::cerr << "ByteBuffer(" << ((void*) this) << " " << a << ")\n"; - -#define LOG_DEBUG2(a,b) LOG_DEBUG1(vespalib::make_string(a,b)); -#define LOG_DEBUG3(a,b,c) LOG_DEBUG1(vespalib::make_string(a,b,c)); -#define LOG_DEBUG4(a,b,c,d) LOG_DEBUG1(vespalib::make_string(a,b,c,d)); - using vespalib::alloc::Alloc; +using vespalib::make_string; namespace document { -VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(InputOutOfRangeException); +namespace { -vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { - vespalib::asciistream ost; - ost << pos << " > " << len; - return ost.str(); -} +static void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline, noreturn)); -BufferOutOfBoundsException::BufferOutOfBoundsException( - size_t pos, size_t len, const vespalib::string& location) - : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) +void throwOutOfBounds(size_t want, size_t has) { + throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); } -InputOutOfRangeException::InputOutOfRangeException( - const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::INTERNAL_FAILURE, location, 1) -{ } -ByteBuffer::ByteBuffer() : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(NULL, 0); - LOG_DEBUG1("Created empty bytebuffer"); +#if defined(__i386__) || defined(__x86_64__) + +template<typename T> +void +ByteBuffer::getDoubleLongNetwork(T &val) { + //TODO: Change this if we move to big-endian hardware + if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { + throwOutOfBounds(sizeof(T), getRemaining()); + } + + auto * data = reinterpret_cast<unsigned char*>(&val); + for (int i=sizeof(T)-1; i>=0; --i) { + getByte(data[i]); + } } -ByteBuffer::ByteBuffer(size_t len) : - ByteBuffer(Alloc::alloc(len), len) +#else +#error "getDoubleLongNetwork is undefined for this arcitecture" +#endif + +VESPA_IMPLEMENT_EXCEPTION_SPINE(BufferOutOfBoundsException); + +vespalib::string BufferOutOfBoundsException::createMessage(size_t pos, size_t len) { + vespalib::asciistream ost; + ost << pos << " > " << len; + return ost.str(); +} + +BufferOutOfBoundsException::BufferOutOfBoundsException(size_t pos, size_t len, const vespalib::string& location) + : IoException(createMessage(pos, len), IoException::NO_SPACE, location, 1) { } -ByteBuffer::ByteBuffer(const char* buffer, size_t len) : - _buffer(NULL), - _len(0), +ByteBuffer::ByteBuffer(const char* buffer, uint32_t len) : + _buffer(const_cast<char *>(buffer)), + _len(len), _pos(0), - _limit(0), - _bufHolder(NULL), _ownedBuffer() { - set(buffer, len); } -ByteBuffer::ByteBuffer(Alloc buffer, size_t len) : +ByteBuffer::ByteBuffer(Alloc buffer, uint32_t len) : _buffer(static_cast<char *>(buffer.get())), _len(len), _pos(0), - _limit(len), - _bufHolder(NULL), _ownedBuffer(std::move(buffer)) { } -ByteBuffer::ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit) : - _buffer(NULL), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), - _ownedBuffer() -{ - set(buf, pos, len, limit); - LOG_DEBUG3("Created copy of byte buffer of length %" PRIu64 " with " - "limit %" PRIu64 ".", len, limit); -} - -ByteBuffer::ByteBuffer(const ByteBuffer& bb) : - _buffer(0), - _len(0), - _pos(0), - _limit(0), - _bufHolder(NULL), +ByteBuffer::ByteBuffer(const ByteBuffer& rhs) : + _buffer(nullptr), + _len(rhs._len), + _pos(rhs._pos), _ownedBuffer() { - LOG_DEBUG1("Created empty byte buffer to assign to."); - *this = bb; -} - -ByteBuffer& ByteBuffer::operator=(const ByteBuffer & org) -{ - if (this != & org) { - cleanUp(); - if (org._len > 0 && org._buffer) { - Alloc::alloc(org._len + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer,org._buffer,org._len); - _buffer[org._len] = 0; - } - _len = org._len; - _pos = org._pos; - _limit = org._limit; - LOG_DEBUG4("Assignment created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); - } - return *this; -} - -void -ByteBuffer::set(BufferHolder* buf, size_t pos, size_t len, size_t limit) -{ - cleanUp(); - _bufHolder = buf; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=pos; - _len=len; - _limit=limit; - LOG_DEBUG4("set() created new buffer of size %" PRIu64 " at pos " - "%" PRIu64 " with limit %" PRIu64 ".", - _len, _pos, _limit); -} - -ByteBuffer::~ByteBuffer() -{ - if (_bufHolder) { - _bufHolder->subRef(); + if (rhs._len > 0 && rhs._buffer) { + Alloc buf = Alloc::alloc(rhs._len); + memcpy(buf.get(), rhs._buffer, rhs._len); + _ownedBuffer = std::move(buf); + _buffer = static_cast<const char *>(_ownedBuffer.get()); } } -std::unique_ptr<ByteBuffer> -ByteBuffer::sliceCopy() const -{ - ByteBuffer* buf = new ByteBuffer; - buf->sliceFrom(*this, _pos, _limit); - - LOG_DEBUG3("Created slice at pos %" PRIu64 " with limit %" PRIu64 ".", - _pos, _limit); - return std::unique_ptr<ByteBuffer>(buf); -} +ByteBuffer::~ByteBuffer() = default; -void ByteBuffer::throwOutOfBounds(size_t want, size_t has) -{ - LOG_DEBUG1("Throwing out of bounds exception"); - throw BufferOutOfBoundsException(want, has, VESPA_STRLOC); -} - -void -ByteBuffer::sliceFrom(const ByteBuffer& buf, size_t from, size_t to) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Created slice from buffer from %" PRIu64 " to %" PRIu64 ".", - from, to); - if (from > buf._len) { - throwOutOfBounds(from, buf._len); - } else if (to > buf._len) { - throwOutOfBounds(to, buf._len); - } else if (to < from) { - throwOutOfBounds(to, from); - } else { - - if (!buf._buffer) { - clear(); - return; - } - - // Slicing from someone that doesn't own their buffer, must make own copy. - if (( buf._ownedBuffer.get() == NULL ) && (buf._bufHolder == NULL)) { - cleanUp(); - Alloc::alloc(to-from + 1).swap(_ownedBuffer); - _buffer = static_cast<char *>(_ownedBuffer.get()); - memcpy(_buffer, buf._buffer + from, to-from); - _buffer[to-from] = 0; - _pos = 0; - _len = _limit = to-from; - return; - } - - // Slicing from someone that owns, but hasn't made a reference counter yet. - if (!buf._bufHolder) { - buf._bufHolder=new BufferHolder(std::move(const_cast<Alloc &>(buf._ownedBuffer))); - } - - // Slicing from refcounter. - cleanUp(); - - _bufHolder = buf._bufHolder; - _bufHolder->addRef(); - _buffer = static_cast<char *>(_bufHolder->_buffer.get()); - _pos=from; - _len=to; - _limit=to; - } -} - -ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, size_t len) +ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, uint32_t len) { if (buffer && len) { - Alloc newBuf = Alloc::alloc(len + 1); + Alloc newBuf = Alloc::alloc(len); memcpy(newBuf.get(), buffer, len); - static_cast<char *>(newBuf.get())[len] = 0; return new ByteBuffer(std::move(newBuf), len); } else { - return NULL; - } -} - -void -ByteBuffer::setPos(size_t pos) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting pos to be %" PRIu64 ", limit is %" PRIu64 ".", - pos, _limit); - if (pos>_limit) { - throwOutOfBounds(pos, _limit); - } else { - _pos=pos; - } -} - -void -ByteBuffer::setLimit(size_t limit) // throw (BufferOutOfBoundsException) -{ - LOG_DEBUG3("Setting limit to %" PRIu64 ", (size is %" PRIu64 ").", limit, _len); - if (limit>_len) { - throwOutOfBounds(limit, _len); - } else { - _limit=limit; + return nullptr; } } - -ByteBuffer::BufferHolder::BufferHolder(Alloc buffer) - : _buffer(std::move(buffer)) -{ -} - -ByteBuffer::BufferHolder::~BufferHolder() = default; -void ByteBuffer::dump() const +void ByteBuffer::incPos(uint32_t pos) { - fprintf(stderr, "ByteBuffer: Length %lu, Pos %lu, Limit %lu\n", - _len, _pos, _limit); - for (size_t i=0; i<_len; i++) { - if (_buffer[i]>32 && _buffer[i]<126) { - fprintf(stderr, "%c", _buffer[i]); - } else { - fprintf(stderr, "[%d]",_buffer[i]); - } - } -} - -void ByteBuffer::incPos(size_t pos) -{ - LOG_DEBUG2("incPos(%" PRIu64 ")", pos); - if (_pos + pos > _limit) { - throwOutOfBounds(_pos + pos, _limit); + if (_pos + pos > _len) { + throwOutOfBounds(_pos + pos, _len); } else { _pos+=pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif } } void ByteBuffer::getNumeric(uint8_t & v) { - LOG_DEBUG2("getNumeric8(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(uint8_t *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(uint8_t v) { - LOG_DEBUG2("putNumeric8(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - *(uint8_t *) getBufferAtPos() = v; + v = *reinterpret_cast<const uint8_t *>(getBufferAtPos()); incPosNoCheck(sizeof(v)); } } -size_t ByteBuffer::forceValgrindStart2Pos() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(_buffer), *e(c + _pos); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - -size_t ByteBuffer::forceValgrindPos2Lim() const -{ - size_t zeroCount(0); - if (_buffer) { - for(const char * c(getBufferAtPos()), *e(c + getRemaining()); c < e; c++) { - if (*c == 0) { - zeroCount++; - } - } - } - return zeroCount; -} - - void ByteBuffer::getNumericNetwork(int16_t & v) { - LOG_DEBUG2("getNumericNetwork16(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint16_t val = *(uint16_t *) (void *) getBufferAtPos(); + uint16_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohs(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int16_t & v) { - LOG_DEBUG2("getNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int16_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(int16_t v) { - LOG_DEBUG2("putNumericNetwork16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint16_t val = htons(v); - *(uint16_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int16_t v) { - LOG_DEBUG2("putNumeric16(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int16_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - void ByteBuffer::getNumericNetwork(int32_t & v) { - LOG_DEBUG2("getNumericNetwork32(%d)", (int) v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - uint32_t val = *(uint32_t *) (void *) getBufferAtPos(); + uint32_t val; + memcpy(&val, getBufferAtPos(), sizeof(val)); v = ntohl(val); incPosNoCheck(sizeof(v)); } } -void ByteBuffer::getNumeric(int32_t & v) { - LOG_DEBUG2("getNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int32_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - - -void ByteBuffer::putNumericNetwork(int32_t v) { - LOG_DEBUG2("putNumericNetwork32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - uint32_t val = htonl(v); - *(uint32_t *) (void *) getBufferAtPos() = val; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumeric(int32_t v) { - LOG_DEBUG2("putNumeric32(%d)", (int) v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int32_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::getNumericNetwork(float & v) { - LOG_DEBUG2("getNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(uint32_t) == 4 - // and endianness same for float and ints - int32_t val; - getIntNetwork(val); - memcpy(&v, &val, sizeof(v)); -} - -void ByteBuffer::getNumeric(float & v) { - LOG_DEBUG2("getNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(float *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} - -void ByteBuffer::putNumericNetwork(float v) { - LOG_DEBUG2("putNumericNetworkFloat(%f)", v); - // XXX depends on sizeof(float) == sizeof(int32_t) == 4 - // and endianness same for float and ints - int32_t val; - memcpy(&val, &v, sizeof(val)); - putIntNetwork(val); -} - -void ByteBuffer::putNumeric(float v) { - LOG_DEBUG2("putNumericFloat(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(float *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} void ByteBuffer::getNumeric(int64_t& v) { - LOG_DEBUG2("getNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - v = *(int64_t *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(int64_t v) { - LOG_DEBUG2("putNumeric64(%" PRId64 ")", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(int64_t *) (void *) getBufferAtPos() = v; - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::getNumeric(double& v) { - LOG_DEBUG2("getNumericDouble(%f)", v); if (__builtin_expect(getRemaining() < sizeof(v), 0)) { throwOutOfBounds(getRemaining(), sizeof(v)); } else { - v = *(double *) (void *) getBufferAtPos(); - incPosNoCheck(sizeof(v)); - } -} -void ByteBuffer::putNumeric(double v) { - LOG_DEBUG2("putNumericDouble(%f)", v); - if (__builtin_expect(getRemaining() < sizeof(v), 0)) { - throwOutOfBounds(getRemaining(), sizeof(v)); - } else { - *(double *) (void *) getBufferAtPos() = v; + memcpy(&v, getBufferAtPos(), sizeof(v)); incPosNoCheck(sizeof(v)); } } void ByteBuffer::getNumericNetwork(double & v) { - LOG_DEBUG2("getNumericNetworkDouble(%f)", v); getDoubleLongNetwork(v); } -void ByteBuffer::putNumericNetwork(int64_t v) { - LOG_DEBUG2("putNumericNetwork64(%" PRId64 ")", v); - putDoubleLongNetwork(v); -} -void ByteBuffer::putNumericNetwork(double v) { - LOG_DEBUG2("putNumericNetworkDouble(%f)", v); - putDoubleLongNetwork(v); -} + void ByteBuffer::getNumericNetwork(int64_t & v) { - LOG_DEBUG2("getNumericNetwork64(%" PRId64 ")", v); getDoubleLongNetwork(v); } -void ByteBuffer::putInt2_4_8Bytes(int64_t number, size_t len) { - LOG_DEBUG3("putInt2_4_8(%" PRId64 ", %" PRIu64 ")", number, len); - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (len == 0) { - if (number < 0x8000ll) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (number < 0x40000000ll) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } - } else if (len == 2) { - //length 2 bytes - putShortNetwork((int16_t) number); - } else if (len == 4) { - //length 4 bytes - putIntNetwork(((int32_t) number) | 0x80000000); - } else if (len == 8) { - //length 8 bytes - putLongNetwork(number | 0xC000000000000000ll); - } else { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number using %d bytes.", (int)len), VESPA_STRLOC); - } -} - -void ByteBuffer::getInt2_4_8Bytes(int64_t & v) { - LOG_DEBUG2("getInt2_4_8(%" PRId64 ")", v); - if (getRemaining() >= 2) { - uint8_t flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 8 bytes - int64_t tmp; - getLongNetwork(tmp); - v = tmp & 0x3FFFFFFFFFFFFFFFll; - } else { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = (int64_t) (tmp & 0x3FFFFFFF); - } - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int64_t) tmp; - } - } else { - throwOutOfBounds(getRemaining(), 2); - } -} - -size_t ByteBuffer::getSerializedSize2_4_8Bytes(int64_t number) { - if (number < 0ll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFFFFFFFFFFll) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^62."), VESPA_STRLOC); - } - - if (number < 0x8000ll) { - return 2; - } else if (number < 0x40000000ll) { - return 4; - } else { - return 8; - } -} - -void ByteBuffer::putInt1_2_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_2_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else if (number < 0x4000) { - putShortNetwork((int16_t) (((int16_t)number) | ((int16_t) 0x8000))); - } else { - putIntNetwork(number | 0xC0000000); - } -} - -void ByteBuffer::getInt1_2_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_2_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - if (flagByte & 0x40) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x3FFFFFFF; - } else { - //length 2 bytes - int16_t tmp; - getShortNetwork(tmp); - v = (int32_t) (tmp & ((int16_t) 0x3FFF)); - } - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} - -size_t ByteBuffer::getSerializedSize1_2_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x3FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^30."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else if (number < 0x4000) { - return 2; - } else { - return 4; - } -} -void ByteBuffer::putInt1_4Bytes(int32_t number) { - LOG_DEBUG2("putInt1_4Bytes(%i)", number); - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - putByte((unsigned char) number); - } else { - putIntNetwork(number | 0x80000000); - } -} -void ByteBuffer::getInt1_4Bytes(int32_t & v) { - LOG_DEBUG2("getInt1_4Bytes(%i)", v); - if (getRemaining() >= 1) { - unsigned char flagByte = peekByte(); - - if (flagByte & 0x80) { - //length 4 bytes - int32_t tmp; - getIntNetwork(tmp); - v = tmp & 0x7FFFFFFF; - } else { - v = (int32_t) flagByte; - incPosNoCheck(1); - } - } else { - throwOutOfBounds(getRemaining(), 1); - } -} -size_t ByteBuffer::getSerializedSize1_4Bytes(int32_t number) { - if (number < 0) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode negative number."), VESPA_STRLOC); - } else if (number > 0x7FFFFFFF) { - throw InputOutOfRangeException(vespalib::make_string( - "Cannot encode number larger than 2^31."), VESPA_STRLOC); - } - - if (number < 0x80) { - return 1; - } else { - return 4; - } -} -void ByteBuffer::getBytes(void *buffer, size_t count) +void ByteBuffer::getBytes(void *buffer, uint32_t count) { - LOG_DEBUG3("getBytes(%p, %" PRIu64 ")", buffer, count); const char *v = getBufferAtPos(); incPos(count); memcpy(buffer, v, count); } -void ByteBuffer::putBytes(const void *buf, size_t count) { - LOG_DEBUG3("putBytes(%p, %" PRIu64 ")", buf, count); - if (__builtin_expect(getRemaining() < count, 0)) { - throwOutOfBounds(getRemaining(), sizeof(count)); - } else { - memcpy(getBufferAtPos(), buf, count); - incPosNoCheck(count); - } -} -std::string ByteBuffer::toString() { - std::ostringstream ost; - StringUtil::printAsHex(ost, getBuffer(), getLength()); - return ost.str(); -} - -void ByteBuffer::swap(ByteBuffer& other) { - LOG_DEBUG2("swap(%p)", &other); - std::swap(_bufHolder, other._bufHolder); - std::swap(_buffer, other._buffer); - std::swap(_len, other._len); - std::swap(_pos, other._pos); - std::swap(_limit, other._limit); -} - -void ByteBuffer::cleanUp() { - LOG_DEBUG1("cleanUp()"); - if (_bufHolder) { - _bufHolder->subRef(); - _bufHolder = NULL; - } else { - Alloc().swap(_ownedBuffer); - } - _buffer = NULL; -} } // document diff --git a/document/src/vespa/document/util/bytebuffer.h b/document/src/vespa/document/util/bytebuffer.h index 6467e6d8bf0..17ab0322a7d 100644 --- a/document/src/vespa/document/util/bytebuffer.h +++ b/document/src/vespa/document/util/bytebuffer.h @@ -15,7 +15,6 @@ #pragma once #include <vespa/vespalib/util/alloc.h> -#include <vespa/vespalib/util/referencecounter.h> namespace document { @@ -23,27 +22,21 @@ class ByteBuffer { public: typedef std::unique_ptr<ByteBuffer> UP; - /** - * Creates a byte buffer with no underlying buffer. - * Use set() to set the buffer. - */ - ByteBuffer(); ByteBuffer(const ByteBuffer &); - ByteBuffer& operator=(const ByteBuffer &); + ByteBuffer& operator=(const ByteBuffer &) = delete; + ByteBuffer(ByteBuffer &&) = default; + ByteBuffer& operator=(ByteBuffer &&) = delete; ~ByteBuffer(); - /** Allocates buffer with len bytes. */ - ByteBuffer(size_t len); - /** * Create a buffer with the given content. * * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(const char* buffer, size_t len); + ByteBuffer(const char* buffer, uint32_t len); /** * Create a buffer with the given content. @@ -51,22 +44,7 @@ public: * @param buffer The buffer to represent. * @param len The length of the buffer */ - ByteBuffer(vespalib::alloc::Alloc buffer, size_t len); - - /** - * Sets the buffer pointed to by this buffer. Allows for multiple - * usages of the same ByteBuffer. - */ - void set(const char* buffer, size_t len) { - cleanUp(); - _buffer = const_cast<char*>(buffer); - _len=len; - _limit=len; - _pos=0; - } - - /** Clear this buffer, and set free the underlying BufferHolder. */ - void reset() { set(NULL, 0); } + ByteBuffer(vespalib::alloc::Alloc buffer, uint32_t len); /** * Creates a ByteBuffer object from another buffer. allocates @@ -75,62 +53,28 @@ public: * @param buffer The buffer to copy. * @param len The length of the buffer. * - * @return Returns a newly created bytebuffer object, or NULL - * if buffer was NULL, or len was <=0. + * @return Returns a newly created bytebuffer object, or nullptr + * if buffer was nullptr, or len was <=0. */ - static ByteBuffer* copyBuffer(const char* buffer, size_t len); - - std::unique_ptr<ByteBuffer> sliceCopy() const; - - /** - * @throws BufferOutOfBoundsException If faulty range is given. - */ - void sliceFrom(const ByteBuffer& buf, size_t from, size_t to); + static ByteBuffer* copyBuffer(const char* buffer, uint32_t len); /** @return Returns the buffer pointed to by this object (at position 0) */ - char* getBuffer() const { return _buffer; } + const char* getBuffer() const { return _buffer; } /** @return Returns the length of the buffer pointed to by this object. */ - size_t getLength() const { return _len; } - - /** - * Adjust the length of the buffer. Only sane to shorten it, as you do not - * know what is ahead. - */ - void setLength(size_t len) { _len = len; } + uint32_t getLength() const { return _len; } /** @return Returns a pointer to the current position in the buffer. */ - char* getBufferAtPos() const { return _buffer + _pos; } + const char* getBufferAtPos() const { return _buffer + _pos; } /** @return Returns the index of the current position in the buffer. */ - size_t getPos() const { return _pos; } - - /** @return Returns the limit. */ - size_t getLimit() const { return _limit; } + uint32_t getPos() const { return _pos; } /** * @return Returns the number of bytes remaining in the buffer - that is, - * getLimit()-getPos(). + * getLength()-getPos(). */ - size_t getRemaining() const { return _limit-_pos; } - - /** - * Changes the position in the buffer. - * - * @throws BufferOutOfBoundsException; - */ - void setPos(size_t pos); - - /** - * Sets the buffer limit. - * - * @param limit The new limit. - * @return True if the limit is legal (less than the length) - * @throws BufferOutOfBoundsException; - */ - void setLimit(size_t limit); - size_t forceValgrindStart2Pos() const __attribute__ ((noinline)); - size_t forceValgrindPos2Lim() const __attribute__ ((noinline)); + uint32_t getRemaining() const { return _len -_pos; } /** * Moves the position in the buffer. @@ -138,267 +82,37 @@ public: * @param pos The number of bytes to move the position. The new position * will be oldPos + pos. This is the same as doing * setPos(getPos()+pos) - * @return True if the position could be moved (it was inside the limit - * of the buffer). * @throws BufferOutOfBoundsException; */ - void incPos(size_t pos); - - void incPosNoCheck(size_t pos) { - _pos += pos; -#ifdef __FORCE_VALGRIND_ON_SERIALIZE__ - forceValgrindStart2Pos(); -#endif - } - - /** - * Resets pos to 0, and sets limit to old pos. Use this before reading - * from a buffer you have written to - */ - void flip() { - _limit = _pos; - _pos = 0; - } + void incPos(uint32_t pos); - /** - * Sets pos to 0 and limit to length. Use this to start writing from the - * start of the buffer. - */ - void clear() { - _pos=0; - _limit=_len; - } - - void getNumericNetwork(uint8_t & v) { getNumeric(v); } void getNumeric(uint8_t & v); - void putNumericNetwork(uint8_t v) { putNumeric(v); } - void putNumeric(uint8_t v); void getNumericNetwork(int16_t & v); - void getNumeric(int16_t & v); - void putNumericNetwork(int16_t v); - void putNumeric(int16_t v); void getNumericNetwork(int32_t & v); - void getNumeric(int32_t & v); - void putNumericNetwork(int32_t v); - void putNumeric(int32_t v); - void getNumericNetwork(float & v); - void getNumeric(float & v); - void putNumericNetwork(float v); - void putNumeric(float v); + void getNumericNetwork(int64_t & v); void getNumeric(int64_t& v); - void putNumericNetwork(int64_t v); - void putNumeric(int64_t v); void getNumericNetwork(double & v); - void getNumeric(double& v); - void putNumericNetwork(double v); - void putNumeric(double v); + void getChar(char & val) { unsigned char t;getByte(t); val=t; } void getByte(uint8_t & v) { getNumeric(v); } - void putByte(uint8_t v) { putNumeric(v); } void getShortNetwork(int16_t & v) { getNumericNetwork(v); } - void getShort(int16_t & v) { getNumeric(v); } - void putShortNetwork(int16_t v) { putNumericNetwork(v); } - void putShort(int16_t v) { putNumeric(v); } void getIntNetwork(int32_t & v) { getNumericNetwork(v); } - void getInt(int32_t & v) { getNumeric(v); } - void putIntNetwork(int32_t v) { putNumericNetwork(v); } - void putInt(int32_t v) { putNumeric(v); } - void getFloatNetwork(float & v) { getNumericNetwork(v); } - void getFloat(float & v) { getNumeric(v); } - void putFloatNetwork(float v) { putNumericNetwork(v); } - void putFloat(float v) { putNumeric(v); } void getLongNetwork(int64_t & v) { getNumericNetwork(v); } void getLong(int64_t& v) { getNumeric(v); } - void putLongNetwork(int64_t v) { putNumericNetwork(v); } - void putLong(int64_t v) { putNumeric(v); } void getDoubleNetwork(double & v) { getNumericNetwork(v); } - void getDouble(double& v) { getNumeric(v); } - void putDoubleNetwork(double v) { putNumericNetwork(v); } - void putDouble(double v) { putNumeric(v); } - - private: - void throwOutOfBounds(size_t want, size_t has) __attribute__((noinline,noreturn)); - uint8_t peekByte() const { return *getBufferAtPos(); } - -#if defined(__i386__) || defined(__x86_64__) - - template<typename T> - void putDoubleLongNetwork(T val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - putByte(data[i]); - } - } + void getBytes(void *buffer, uint32_t count); +private: template<typename T> - void getDoubleLongNetwork(T &val) { - //TODO: Change this if we move to big-endian hardware - if (__builtin_expect(getRemaining() < (int)sizeof(T), 0)) { - throwOutOfBounds(sizeof(T), getRemaining()); - } + void getDoubleLongNetwork(T &val); - unsigned char* data = reinterpret_cast<unsigned char*>(&val); - for (int i=sizeof(T)-1; i>=0; --i) { - getByte(data[i]); - } - } -#else - #error "getDoubleLongNetwork is undefined for this arcitecture" -#endif + void incPosNoCheck(uint32_t pos) { _pos += pos; } - public: - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - */ - void putInt2_4_8Bytes(int64_t number) { - putInt2_4_8Bytes(number, 0); - } - - /** - * Writes a 62-bit positive integer to the buffer, using 2, 4, or 8 bytes. - * - * @param number the integer to write - * @param len if non-zero, force writing number using len bytes, possibly - * with truncation - */ - void putInt2_4_8Bytes(int64_t number, size_t len); - - /** - * Reads a 62-bit positive integer from the buffer, which was written using - * 2, 4, or 8 bytes. - * - * @param v the integer read - */ - void getInt2_4_8Bytes(int64_t & v); - - /** - * Computes the size used for storing the given integer using 2, 4 or 8 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 2, 4 or 8 - */ - static size_t getSerializedSize2_4_8Bytes(int64_t number); - - /** - * Writes a 30-bit positive integer to the buffer, using 1, 2, or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_2_4Bytes(int32_t number); - - /** - * Reads a 30-bit positive integer from the buffer, which was written using - * 1, 2, or 4 bytes. - * - * @param v the integer read - */ - void getInt1_2_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1, 2 or 4 - * bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1, 2 or 4 - */ - static size_t getSerializedSize1_2_4Bytes(int32_t number); - - /** - * Writes a 31-bit positive integer to the buffer, using 1 or 4 bytes. - * - * @param number the integer to write - */ - void putInt1_4Bytes(int32_t number); - - /** - * Reads a 31-bit positive integer from the buffer, which was written using - * 1 or 4 bytes. - * - * @param v the integer read - */ - void getInt1_4Bytes(int32_t & v); - - /** - * Computes the size used for storing the given integer using 1 or 4 bytes. - * - * @param number the integer to check length of - * @return the number of bytes used to store it; 1 or 4 - */ - static size_t getSerializedSize1_4Bytes(int32_t number); - - /** - * Writes a 8 bit integer to the buffer at the current position, and - * increases the positition accordingly. - * - * @param val the int to store - * @return True if the value could be stored, false if end of buffer is - * reached - */ - void getChar(char & val) { unsigned char t;getByte(t); val=t; } - void putChar(char val) { putByte(static_cast<unsigned char>(val)); } - - /** - * Reads the given number of bytes into the given pointer, and updates the - * positition accordingly - * - * @param buffer where to store the bytes - * @param count number of bytes to read - * @return True if all the bytes could be read, false if end of - * buffer is reached - */ - void getBytes(void *buffer, size_t count); - - /** - * Writes the given number of bytes into the ByteBuffer at the current - * position, and updates the positition accordingly - * - * @param buf the bytes to store - * @param count number of bytes to store - */ - void putBytes(const void *buf, size_t count); - - /** Debug */ - void dump() const; - - class BufferHolder : public vespalib::ReferenceCounter - { - private: - BufferHolder(const BufferHolder &); - BufferHolder& operator=(const BufferHolder &); - - public: - BufferHolder(vespalib::alloc::Alloc buffer); - virtual ~BufferHolder(); - - vespalib::alloc::Alloc _buffer; - }; - - ByteBuffer(BufferHolder* buf, size_t pos, size_t len, size_t limit); - - void set(BufferHolder* buf, size_t pos, size_t len, size_t limit); - -private: - char * _buffer; - size_t _len; - size_t _pos; - size_t _limit; - mutable BufferHolder * _bufHolder; + const char * _buffer; + const uint32_t _len; + uint32_t _pos; vespalib::alloc::Alloc _ownedBuffer; -public: - - std::string toString(); - - void swap(ByteBuffer& other); - - void cleanUp(); }; } // document diff --git a/document/src/vespa/document/util/serializable.cpp b/document/src/vespa/document/util/serializable.cpp deleted file mode 100644 index 32c7bef90f0..00000000000 --- a/document/src/vespa/document/util/serializable.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "serializable.h" -#include "bufferexceptions.h" -#include "serializableexceptions.h" -#include "bytebuffer.h" - -namespace document { - -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Serializable, vespalib::Identifiable); -IMPLEMENT_IDENTIFIABLE_ABSTRACT(Deserializable, Serializable); -VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(SerializeException); - -std::unique_ptr<ByteBuffer> Serializable::serialize() const -{ - size_t len = getSerializedSize(); - std::unique_ptr<ByteBuffer> retVal(new ByteBuffer(len)); - serialize(*retVal.get()); - return retVal; -} - -DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -DeserializeException::DeserializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -SerializeException::SerializeException(const vespalib::string& msg, const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, location, 1) -{ -} - -SerializeException::SerializeException( - const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location) - : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) -{ -} - -void -Serializable::serialize(ByteBuffer& buffer) const { - int pos = buffer.getPos(); - try{ - onSerialize(buffer); - } catch (...) { - buffer.setPos(pos); - throw; - } -} - -void -Deserializable::deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) { - int pos = buffer.getPos(); - try { - onDeserialize(repo, buffer); - } catch (const DeserializeException &) { - buffer.setPos(pos); - throw; - } catch (const BufferOutOfBoundsException &) { - buffer.setPos(pos); - throw; - } -} -} diff --git a/document/src/vespa/document/util/serializable.h b/document/src/vespa/document/util/serializable.h deleted file mode 100644 index 8818feceb9a..00000000000 --- a/document/src/vespa/document/util/serializable.h +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @file serializable.h - * @ingroup document - * - * @brief Interfaces to be used for serializing of objects. - * - * @author Thomas F. Gundersen, H�kon Humberset - * @date 2004-03-15 - * @version $Id$ - */ - -#pragma once - -#include <vespa/vespalib/objects/cloneable.h> -#include <vespa/vespalib/objects/identifiable.h> - -#include <vespa/document/util/bytebuffer.h> -#include <vespa/document/util/identifiableid.h> - -namespace document { -class DocumentTypeRepo; - -/** - * Base class for classes that can be converted into a bytestream, - * normally used later to create a similar instance. - */ - -class Serializable : public vespalib::Identifiable -{ -protected: - virtual void onSerialize(ByteBuffer& buffer) const = 0; -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Serializable); - - virtual ~Serializable() {} - - /** - * @return An upper limit to how many bytes serialization of this instance - * need, providing instance is not altered before serialization. - */ - virtual size_t getSerializedSize() const = 0; - - /** - * Serializes the instance into the buffer given. Use getSerializedSize() - * before calling this method to be sure buffer is big enough. - * On success, the given buffers position will be just past the serialized - * version of this instance, on failure, position will be reset to whatever - * it was prior to calling this function. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - void serialize(ByteBuffer& buffer) const; - - /** - * Creates a bytebuffer with enough space to serialize this instance - * and serialize this instance into it. - * - * @return The created bytebuffer, positioned after the serialization. - * - * @throw SerializeException If for some reason instance cannot be - * serialized. - * @throw BufferOutOfBoundsException If buffer does not have enough space. - */ - std::unique_ptr<ByteBuffer> serialize() const; -}; - -/** - * Base class for instances that can be overwritten from a bytestream, - * given that the bytestream is created from a similar instance. - */ -class Deserializable : public vespalib::Cloneable, public Serializable -{ -protected: - virtual void onDeserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer) = 0; - -public: - DECLARE_IDENTIFIABLE_ABSTRACT(Deserializable); - virtual ~Deserializable() {} - - /** - * Overwrite this object with the object represented by the given - * bytestream. On success, buffer will be positioned after the bytestream - * representing the instance we've just deserialized, on failure, bytebuffer - * will be pointing to where it was pointing before calling this function. - * - * @throw DeserializeException If read data doesn't represent a legal object - * of this type. - * @throw BufferOutOfBoundsException If instance wants to read more data - * than is available in the buffer. - */ - void deserialize(const DocumentTypeRepo &repo, ByteBuffer& buffer); -}; - -} - diff --git a/document/src/vespa/document/util/serializableexceptions.cpp b/document/src/vespa/document/util/serializableexceptions.cpp new file mode 100644 index 00000000000..e80e38015e8 --- /dev/null +++ b/document/src/vespa/document/util/serializableexceptions.cpp @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serializableexceptions.h" + +namespace document { + +VESPA_IMPLEMENT_EXCEPTION_SPINE(DeserializeException); + +DeserializeException::DeserializeException(const vespalib::string& msg, const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, location, 1) +{ +} + +DeserializeException::DeserializeException( + const vespalib::string& msg, const vespalib::Exception& cause, + const vespalib::string& location) + : IoException(msg, IoException::CORRUPT_DATA, cause, location, 1) +{ +} + +} diff --git a/document/src/vespa/document/util/serializableexceptions.h b/document/src/vespa/document/util/serializableexceptions.h index fcfed810bfc..1b692aa27b7 100644 --- a/document/src/vespa/document/util/serializableexceptions.h +++ b/document/src/vespa/document/util/serializableexceptions.h @@ -24,12 +24,4 @@ public: VESPA_DEFINE_EXCEPTION_SPINE(DeserializeException) }; -class SerializeException : public vespalib::IoException { -public: - SerializeException(const vespalib::string& msg, const vespalib::string& location = ""); - SerializeException(const vespalib::string& msg, const vespalib::Exception& cause, - const vespalib::string& location = ""); - VESPA_DEFINE_EXCEPTION_SPINE(SerializeException) -}; - } diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 79f7d7c0ccc..5582c0ea153 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -20,7 +20,6 @@ namespace documentapi { class LoadTypeSet; class RoutingPolicyRepository; class RoutableRepository; -class SystemState; class IRoutingPolicyFactory; class IRoutableFactory; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp index 718611dfc04..6c8394b1b4c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/documentstate.cpp @@ -15,12 +15,11 @@ DocumentState::DocumentState(const DocumentState& o) : _gid(o._gid), _timestamp(o._timestamp), _removeEntry(o._removeEntry) { if (o._docId.get() != 0) { - _docId.reset(new document::DocumentId(*o._docId)); + _docId = std::make_unique<document::DocumentId>(*o._docId); } } -DocumentState::DocumentState(const document::DocumentId& id, - uint64_t timestamp, bool removeEntry) +DocumentState::DocumentState(const document::DocumentId& id, uint64_t timestamp, bool removeEntry) : _docId(new document::DocumentId(id)), _gid(_docId->getGlobalId()), _timestamp(timestamp), @@ -28,8 +27,7 @@ DocumentState::DocumentState(const document::DocumentId& id, { } -DocumentState::DocumentState(const document::GlobalId& gid, - uint64_t timestamp, bool removeEntry) +DocumentState::DocumentState(const document::GlobalId& gid, uint64_t timestamp, bool removeEntry) : _gid(gid), _timestamp(timestamp), _removeEntry(removeEntry) {} DocumentState::DocumentState(document::ByteBuffer& buf) @@ -39,10 +37,10 @@ DocumentState::DocumentState(document::ByteBuffer& buf) buf.getByte(hasDocId); if (hasDocId) { vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); - _docId.reset(new document::DocumentId(stream)); + _docId = std::make_unique<document::DocumentId>(stream); buf.incPos(stream.rp()); } - char* gid = buf.getBufferAtPos(); + const char* gid = buf.getBufferAtPos(); buf.incPos(document::GlobalId::LENGTH); _gid.set(gid); buf.getLongNetwork((int64_t&) _timestamp); @@ -55,8 +53,8 @@ DocumentState& DocumentState::operator=(const DocumentState& other) { _docId.reset(); - if (other._docId.get() != 0) { - _docId.reset(new document::DocumentId(*other._docId)); + if (other._docId) { + _docId = std::make_unique<document::DocumentId>(*other._docId); } _gid = other._gid; _timestamp = other._timestamp; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp index 6753d269ad6..2dcaccbd861 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/putdocumentmessage.cpp @@ -21,12 +21,12 @@ PutDocumentMessage::PutDocumentMessage(document::Document::SP document) : setDocument(std::move(document)); } -PutDocumentMessage::~PutDocumentMessage() {} +PutDocumentMessage::~PutDocumentMessage() = default; DocumentReply::UP PutDocumentMessage::doCreateReply() const { - return DocumentReply::UP(new WriteDocumentReply(DocumentProtocol::REPLY_PUTDOCUMENT)); + return std::make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); } bool diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp index 411a7237cb5..08d631dc44b 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.cpp @@ -4,11 +4,7 @@ namespace documentapi { -QueryResultMessage::QueryResultMessage() : - VisitorMessage(), - _searchResult(), - _summary() -{} +QueryResultMessage::QueryResultMessage() = default; QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, const vdslib::DocumentSummary & summary) : VisitorMessage(), @@ -16,7 +12,7 @@ QueryResultMessage::QueryResultMessage(const vdslib::SearchResult & result, cons _summary(summary) {} -QueryResultMessage::~QueryResultMessage() {} +QueryResultMessage::~QueryResultMessage() = default; DocumentReply::UP QueryResultMessage::doCreateReply() const diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h index 239ce6fefd5..6324e2664e4 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/queryresultmessage.h @@ -25,7 +25,7 @@ public: * Constructs a new search result message for deserialization. */ QueryResultMessage(); - ~QueryResultMessage(); + ~QueryResultMessage() override; /** * Constructs a new search result message for the given search result. diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp index 43ad30ea24f..453e93fd7eb 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.cpp @@ -1,8 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "visitor.h" -#include <climits> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> +#include <climits> using document::FixedBucketSpaces; @@ -56,7 +59,7 @@ CreateVisitorMessage::~CreateVisitorMessage() = default; DocumentReply::UP CreateVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new CreateVisitorReply(DocumentProtocol::REPLY_CREATEVISITOR)); + return std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR); } uint32_t @@ -65,11 +68,7 @@ CreateVisitorMessage::getType() const return DocumentProtocol::MESSAGE_CREATEVISITOR; } -DestroyVisitorMessage::DestroyVisitorMessage() : - DocumentMessage(), - _instanceId() -{ -} +DestroyVisitorMessage::DestroyVisitorMessage() = default; DestroyVisitorMessage::DestroyVisitorMessage(const string& instanceId) : DocumentMessage(), @@ -77,13 +76,12 @@ DestroyVisitorMessage::DestroyVisitorMessage(const string& instanceId) : { } -DestroyVisitorMessage::~DestroyVisitorMessage() { -} +DestroyVisitorMessage::~DestroyVisitorMessage() = default; DocumentReply::UP DestroyVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new DocumentReply(DocumentProtocol::REPLY_DESTROYVISITOR)); + return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_DESTROYVISITOR); } uint32_t @@ -104,20 +102,13 @@ CreateVisitorReply::CreateVisitorReply(uint32_t type) : { } -VisitorInfoMessage::VisitorInfoMessage() : - VisitorMessage(), - _finishedBuckets(), - _errorMessage() -{ -} - -VisitorInfoMessage::~VisitorInfoMessage() { -} +VisitorInfoMessage::VisitorInfoMessage() = default; +VisitorInfoMessage::~VisitorInfoMessage() = default; DocumentReply::UP VisitorInfoMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_VISITORINFO)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO); } uint32_t @@ -126,11 +117,7 @@ VisitorInfoMessage::getType() const return DocumentProtocol::MESSAGE_VISITORINFO; } -MapVisitorMessage::MapVisitorMessage() : - _data() -{ - // empty -} +MapVisitorMessage::MapVisitorMessage() = default; uint32_t MapVisitorMessage::getApproxSize() const @@ -141,7 +128,7 @@ MapVisitorMessage::getApproxSize() const DocumentReply::UP MapVisitorMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_MAPVISITOR)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR); } uint32_t MapVisitorMessage::getType() const @@ -149,60 +136,37 @@ uint32_t MapVisitorMessage::getType() const return DocumentProtocol::MESSAGE_MAPVISITOR; } -DocumentListMessage::Entry::Entry() -{ - // empty -} +DocumentListMessage::Entry::Entry() = default; -DocumentListMessage::Entry::Entry(int64_t timestamp, - document::Document::SP doc, - bool removeEntry) : +DocumentListMessage::Entry::Entry(int64_t timestamp, document::Document::SP doc, bool removeEntry) : _timestamp(timestamp), - _document(doc), + _document(std::move(doc)), _removeEntry(removeEntry) -{ - // empty -} +{ } -DocumentListMessage::Entry::Entry(const Entry& other) : - _timestamp(other._timestamp), - _document(other._document), - _removeEntry(other._removeEntry) -{ - // empty -} +DocumentListMessage::Entry::Entry(const Entry& other) = default; -DocumentListMessage::Entry::Entry(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buf) +DocumentListMessage::Entry::Entry(const document::DocumentTypeRepo &repo, document::ByteBuffer& buf) { buf.getLongNetwork(_timestamp); - _document.reset(new document::Document(repo, buf)); + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + _document = std::make_unique<document::Document>(repo, stream); + buf.incPos(buf.getRemaining() - stream.size()); uint8_t b; buf.getByte(b); _removeEntry = b>0; } void -DocumentListMessage::Entry::serialize(document::ByteBuffer& buf) const +DocumentListMessage::Entry::serialize(vespalib::GrowableByteBuffer& buf) const { - buf.putLongNetwork(_timestamp); - _document->serialize(buf); + buf.putLong(_timestamp); + vespalib::nbostream nbo = _document->serialize(); + buf.putBytes(nbo.data(), nbo.size()); buf.putByte(_removeEntry ? 1 : 0); } -uint32_t -DocumentListMessage::Entry::getSerializedSize() const -{ - return sizeof(int64_t) + sizeof(uint8_t) - + _document->serialize()->getLength(); -} - -DocumentListMessage::DocumentListMessage() : - _bucketId(), - _documents() -{ - // empty -} +DocumentListMessage::DocumentListMessage() = default; DocumentListMessage::DocumentListMessage(document::BucketId bid) : _bucketId(bid), @@ -214,7 +178,7 @@ DocumentListMessage::DocumentListMessage(document::BucketId bid) : DocumentReply::UP DocumentListMessage::doCreateReply() const { - return DocumentReply::UP(new VisitorReply(DocumentProtocol::REPLY_DOCUMENTLIST)); + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST); } uint32_t diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index b18a4e985f3..f47fa48bd80 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -52,7 +52,7 @@ public: const string& instanceId, const string& controlDestination, const string& dataDestination); - ~CreateVisitorMessage(); + ~CreateVisitorMessage() override; const string& getLibraryName() const { return _libName; } void setLibraryName(const string& value) { _libName = value; } @@ -250,7 +250,7 @@ public: const document::Document::SP& getDocument() { return _document; } bool isRemoveEntry() { return _removeEntry; } - void serialize(document::ByteBuffer& buf) const; + void serialize(vespalib::GrowableByteBuffer& buf) const; uint32_t getSerializedSize() const; private: int64_t _timestamp; diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 797f55120fc..7bae7dd7e77 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -20,7 +20,7 @@ namespace documentapi { bool RoutableFactories60::DocumentMessageFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const { - const DocumentMessage &msg = static_cast<const DocumentMessage&>(obj); + const auto &msg = static_cast<const DocumentMessage&>(obj); out.putByte(msg.getPriority()); out.putInt(msg.getLoadType().getId()); return doEncode(msg, out); @@ -93,7 +93,7 @@ RoutableFactories60::CreateVisitorMessageFactory::doDecode(document::ByteBuffer msg->setVisitRemoves(decodeBoolean(buf)); msg->setFieldSet(decodeString(buf)); msg->setVisitInconsistentBuckets(decodeBoolean(buf)); - msg->getParameters().deserialize(_repo, buf); + msg->getParameters().deserialize(buf); msg->setVisitorDispatcherVersion(50); decodeInt(buf); // Unused legacy visitor ordering msg->setMaxBucketsPerVisitor(decodeInt(buf)); @@ -105,7 +105,7 @@ RoutableFactories60::CreateVisitorMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); + const auto &msg = static_cast<const CreateVisitorMessage&>(obj); buf.putString(msg.getLibraryName()); buf.putString(msg.getInstanceId()); @@ -126,10 +126,8 @@ RoutableFactories60::CreateVisitorMessageFactory::doEncode(const DocumentMessage buf.putString(msg.getFieldSet()); buf.putBoolean(msg.visitInconsistentBuckets()); - int len = msg.getParameters().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getParameters().serialize(dbuf); + + msg.getParameters().serialize(buf); buf.putInt(0); // Unused legacy visitor ordering buf.putInt(msg.getMaxBucketsPerVisitor()); @@ -184,7 +182,7 @@ RoutableFactories60::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::CreateVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const CreateVisitorReply &reply = static_cast<const CreateVisitorReply&>(obj); + const auto &reply = static_cast<const CreateVisitorReply&>(obj); buf.putLong(reply.getLastBucket().getRawId()); buf.putInt(reply.getVisitorStatistics().getBucketsVisited()); buf.putLong(reply.getVisitorStatistics().getDocumentsVisited()); @@ -209,19 +207,14 @@ RoutableFactories60::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, } DocumentReply::UP -RoutableFactories60::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& buf) const +RoutableFactories60::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& ) const { - (void) buf; - return DocumentReply::UP(new DocumentIgnoredReply()); + return std::make_unique<DocumentIgnoredReply>(); } bool -RoutableFactories60::DocumentIgnoredReplyFactory::doEncode( - const DocumentReply& obj, - vespalib::GrowableByteBuffer& buf) const +RoutableFactories60::DocumentIgnoredReplyFactory::doEncode(const DocumentReply&, vespalib::GrowableByteBuffer& ) const { - (void) obj; - (void) buf; return true; } @@ -243,15 +236,12 @@ RoutableFactories60::DocumentListMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::DocumentListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const DocumentListMessage &msg = static_cast<const DocumentListMessage&>(obj); + const auto &msg = static_cast<const DocumentListMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); buf.putInt(msg.getDocuments().size()); for (const auto & document : msg.getDocuments()) { - int len = document.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - document.serialize(dbuf); + document.serialize(buf); } return true; @@ -283,11 +273,7 @@ bool RoutableFactories60::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); - - int32_t len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); + msg.serialize(buf); return true; } @@ -322,7 +308,7 @@ RoutableFactories60::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::EmptyBucketsMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); + const auto &msg = static_cast<const EmptyBucketsMessage&>(obj); buf.putInt(msg.getBucketIds().size()); for (const auto & bucketId : msg.getBucketIds()) { @@ -367,7 +353,7 @@ RoutableFactories60::GetBucketListMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::GetBucketListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketListMessage &msg = static_cast<const GetBucketListMessage&>(obj); + const auto &msg = static_cast<const GetBucketListMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); return encodeBucketSpace(msg.getBucketSpace(), buf); } @@ -392,7 +378,7 @@ RoutableFactories60::GetBucketListReplyFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::GetBucketListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketListReply &reply = static_cast<const GetBucketListReply&>(obj); + const auto &reply = static_cast<const GetBucketListReply&>(obj); const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); buf.putInt(buckets.size()); @@ -417,7 +403,7 @@ RoutableFactories60::GetBucketStateMessageFactory::doDecode(document::ByteBuffer bool RoutableFactories60::GetBucketStateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const GetBucketStateMessage &msg = static_cast<const GetBucketStateMessage&>(obj); + const auto &msg = static_cast<const GetBucketStateMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); return true; } @@ -452,14 +438,12 @@ RoutableFactories60::GetBucketStateReplyFactory::doEncode(const DocumentReply &o DocumentMessage::UP RoutableFactories60::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const { - return DocumentMessage::UP( - new GetDocumentMessage(decodeDocumentId(buf), - decodeString(buf))); + document::DocumentId docId = decodeDocumentId(buf); + return std::make_unique<GetDocumentMessage>(docId, decodeString(buf)); } bool -RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, - vespalib::GrowableByteBuffer &buf) const +RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); @@ -468,6 +452,17 @@ RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage & return true; } +namespace { + +std::shared_ptr<document::Document> +decodeDocument(const document::DocumentTypeRepo & repo, document::ByteBuffer & buf) { + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + auto doc = std::make_shared<document::Document>(repo, stream); + buf.incPos(buf.getRemaining() - stream.size()); + return doc; +} + +} DocumentReply::UP RoutableFactories60::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const { @@ -476,7 +471,7 @@ RoutableFactories60::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf bool hasDocument = decodeBoolean(buf); document::Document * document = nullptr; if (hasDocument) { - auto doc = std::make_shared<document::Document>(_repo, buf); + auto doc = decodeDocument(_repo, buf); document = doc.get(); reply->setDocument(std::move(doc)); } @@ -509,7 +504,7 @@ DocumentMessage::UP RoutableFactories60::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<MapVisitorMessage>(); - msg->getData().deserialize(_repo, buf); + msg->getData().deserialize(buf); return msg; } @@ -517,11 +512,7 @@ bool RoutableFactories60::MapVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { const MapVisitorMessage &msg = static_cast<const MapVisitorMessage&>(obj); - - int32_t len = msg.getData().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getData().serialize(dbuf); + msg.getData().serialize(buf); return true; } @@ -540,7 +531,7 @@ RoutableFactories60::MapVisitorReplyFactory::doEncode(const DocumentReply &, ves void RoutableFactories60::PutDocumentMessageFactory::decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocument(make_shared<document::Document>(_repo, buf)); + msg.setDocument(decodeDocument(_repo, buf)); msg.setTimestamp(static_cast<uint64_t>(decodeLong(buf))); decodeTasCondition(msg, buf); } @@ -564,9 +555,6 @@ RoutableFactories60::PutDocumentReplyFactory::doDecode(document::ByteBuffer &buf { auto reply = make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); reply->setHighestModificationTimestamp(decodeLong(buf)); - - // Doing an explicit move here to force converting result to an rvalue. - // This is done automatically in GCC >= 5. return reply; } @@ -656,12 +644,8 @@ RoutableFactories60::SearchResultMessageFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const SearchResultMessage &msg = static_cast<const SearchResultMessage&>(obj); - - int len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); + const auto & msg = static_cast<const SearchResultMessage&>(obj); + msg.serialize(buf); return true; } @@ -679,13 +663,10 @@ RoutableFactories60::QueryResultMessageFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::QueryResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const QueryResultMessage &msg = static_cast<const QueryResultMessage&>(obj); + const auto &msg = static_cast<const QueryResultMessage&>(obj); - int len = msg.getSearchResult().getSerializedSize() + msg.getDocumentSummary().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getSearchResult().serialize(dbuf); - msg.getDocumentSummary().serialize(dbuf); + msg.getSearchResult().serialize(buf); + msg.getDocumentSummary().serialize(buf); return true; } @@ -740,7 +721,7 @@ RoutableFactories60::StatBucketMessageFactory::doDecode(document::ByteBuffer &bu bool RoutableFactories60::StatBucketMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const StatBucketMessage &msg = static_cast<const StatBucketMessage&>(obj); + const auto &msg = static_cast<const StatBucketMessage&>(obj); buf.putLong(msg.getBucketId().getRawId()); buf.putString(msg.getDocumentSelection()); @@ -758,7 +739,7 @@ RoutableFactories60::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) bool RoutableFactories60::StatBucketReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const StatBucketReply &reply = static_cast<const StatBucketReply&>(obj); + const auto &reply = static_cast<const StatBucketReply&>(obj); buf.putString(reply.getResults()); return true; } @@ -789,7 +770,9 @@ RoutableFactories60::StatDocumentReplyFactory::doEncode(const DocumentReply &, v void RoutableFactories60::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf)); + vespalib::nbostream stream(buf.getBufferAtPos(), buf.getRemaining()); + msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, stream)); + buf.incPos(stream.rp()); msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); decodeTasCondition(msg, buf); @@ -798,7 +781,7 @@ RoutableFactories60::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMess bool RoutableFactories60::UpdateDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj); + const auto &msg = static_cast<const UpdateDocumentMessage&>(obj); vespalib::nbostream stream; msg.getDocumentUpdate().serializeHEAD(stream); @@ -822,7 +805,7 @@ RoutableFactories60::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer & bool RoutableFactories60::UpdateDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const UpdateDocumentReply &reply = static_cast<const UpdateDocumentReply&>(obj); + const auto &reply = static_cast<const UpdateDocumentReply&>(obj); buf.putBoolean(reply.getWasFound()); buf.putLong(reply.getHighestModificationTimestamp()); return true; @@ -848,7 +831,7 @@ RoutableFactories60::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &b bool RoutableFactories60::VisitorInfoMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const { - const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); + const auto &msg = static_cast<const VisitorInfoMessage&>(obj); buf.putInt(msg.getFinishedBuckets().size()); for (const auto & bucketId : msg.getFinishedBuckets()) { @@ -883,7 +866,7 @@ RoutableFactories60::WrongDistributionReplyFactory::doDecode(document::ByteBuffe bool RoutableFactories60::WrongDistributionReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const { - const WrongDistributionReply &reply = static_cast<const WrongDistributionReply&>(obj); + const auto &reply = static_cast<const WrongDistributionReply&>(obj); buf.putString(reply.getSystemState()); return true; } diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 937431fb7dd..5c8b5b029d2 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -260,7 +260,7 @@ verifyDocs(const std::vector<DocAndTimestamp>& wanted, << entry.getDocument()->toString(true); } EXPECT_EQ(wanted[wantedIdx].timestamp, entry.getTimestamp()); - size_t serSize = wanted[wantedIdx].doc->serialize()->getLength(); + size_t serSize = wanted[wantedIdx].doc->serialize().size(); EXPECT_EQ(serSize + sizeof(DocEntry), size_t(entry.getSize())); EXPECT_EQ(serSize, size_t(entry.getDocumentSize())); ++wantedIdx; diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp index d482d144d73..f46be6f3a25 100644 --- a/persistence/src/vespa/persistence/spi/docentry.cpp +++ b/persistence/src/vespa/persistence/spi/docentry.cpp @@ -2,6 +2,7 @@ #include "docentry.h" #include <vespa/document/fieldvalue/document.h> +#include <vespa/vespalib/objects/nbostream.h> #include <sstream> #include <cassert> @@ -10,16 +11,13 @@ namespace storage::spi { DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc) : _timestamp(t), _metaFlags(metaFlags), - _persistedDocumentSize(doc->getSerializedSize()), + _persistedDocumentSize(doc->serialize().size()), _size(_persistedDocumentSize + sizeof(DocEntry)), _documentId(), _document(std::move(doc)) { } -DocEntry::DocEntry(Timestamp t, - int metaFlags, - DocumentUP doc, - size_t serializedDocumentSize) +DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc, size_t serializedDocumentSize) : _timestamp(t), _metaFlags(metaFlags), _persistedDocumentSize(serializedDocumentSize), diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index ca8274aab00..54fc072c2b1 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -298,7 +298,7 @@ public: { } virtual RPC::Result receive(const Packet &packet) override { - vespalib::nbostream_longlivedbuf handle(packet.getHandle().c_str(), packet.getHandle().size()); + vespalib::nbostream_longlivedbuf handle(packet.getHandle().data(), packet.getHandle().size()); try { while (handle.size() > 0) { Packet::Entry entry; diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp index e5ac6ed15b0..6d7bd243e71 100644 --- a/searchcore/src/tests/grouping/grouping.cpp +++ b/searchcore/src/tests/grouping/grouping.cpp @@ -168,7 +168,7 @@ TEST_F("testGroupingContextInitialization", DoomFixture()) { nos << (uint32_t)1; baseRequest.serialize(nos); - GroupingContext context(f1.clock, f1.timeOfDoom, os.c_str(), os.size()); + GroupingContext context(f1.clock, f1.timeOfDoom, os.data(), os.size()); ASSERT_TRUE(!context.empty()); GroupingContext::GroupingList list = context.getGroupingList(); ASSERT_TRUE(list.size() == 1); @@ -226,7 +226,7 @@ TEST_F("testGroupingContextSerializing", DoomFixture()) { context.serialize(); vespalib::nbostream & res(context.getResult()); EXPECT_EQUAL(res.size(), os.size()); - ASSERT_TRUE(memcmp(res.c_str(), os.c_str(), res.size()) == 0); + ASSERT_TRUE(memcmp(res.data(), os.data(), res.size()) == 0); } TEST_F("testGroupingManager", DoomFixture()) { diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 8c4bf0d55b0..95ab43dbcba 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -616,7 +616,7 @@ TEST("require that grouping is performed (multi-threaded)") { Grouping grequest; grequest.setRoot(Group().addResult(SumAggregationResult().setExpression(createAttr()))); grequest.serialize(os); - request->groupSpec.assign(buf.c_str(), buf.c_str() + buf.size()); + request->groupSpec.assign(buf.data(), buf.data() + buf.size()); } SearchReply::UP reply = world.performSearch(request, threads); { diff --git a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp index 109a4cc7a25..c3338a973c4 100644 --- a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp +++ b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp @@ -36,7 +36,7 @@ private: void insert_tensor_in_properties(const vespalib::string& tensor_name, const Value& tensor_value) { vespalib::nbostream stream; DefaultTensorEngine::ref().encode(tensor_value, stream); - _props.add(tensor_name, vespalib::stringref(stream.c_str(), stream.size())); + _props.add(tensor_name, vespalib::stringref(stream.data(), stream.size())); } public: diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 96096c0401f..ca48fb773d8 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -105,7 +105,7 @@ RemoveOperationContext::RemoveOperationContext(search::SerialNum serial) str(), packet() { op.serialize(str); - ConstBufferRef buf(str.c_str(), str.wp()); + ConstBufferRef buf(str.data(), str.wp()); packet.reset(new Packet()); packet->add(Packet::Entry(serial, FeedOperation::REMOVE, buf)); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index e658bc0cfa0..1bcbe4e9683 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -31,7 +31,7 @@ DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ss if (removed) { return new DocEntry(timestamp, storage::spi::REMOVE_ENTRY, doc->getId()); } else { - ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->getSerializedSize(); + ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->serialize().size(); return new DocEntry(timestamp, storage::spi::NONE, std::move(doc), serializedSize); } } else { diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index 5a6a990df9b..faee5914a97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -49,7 +49,7 @@ handleProgress(TlsReplayProgress &progress, SerialNum currentSerial) void handlePacket(PacketWrapper::SP wrap, EntryHandler entryHandler) { - vespalib::nbostream_longlivedbuf handle(wrap->packet.getHandle().c_str(), wrap->packet.getHandle().size()); + vespalib::nbostream_longlivedbuf handle(wrap->packet.getHandle().data(), wrap->packet.getHandle().size()); while (handle.size() > 0) { Packet::Entry entry; entry.deserialize(handle); diff --git a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp index bfc59dee35e..bbd02d7efce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/tlcproxy.cpp @@ -14,7 +14,7 @@ namespace proton { void TlcProxy::commit(search::SerialNum serialNum, search::transactionlog::Type type, const vespalib::nbostream &buf, DoneCallback onDone) { - Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.c_str(), buf.size())); + Packet::Entry entry(serialNum, type, vespalib::ConstBufferRef(buf.data(), buf.size())); Packet packet; packet.add(entry); packet.close(); diff --git a/searchlib/src/apps/docstore/create-idx-from-dat.cpp b/searchlib/src/apps/docstore/create-idx-from-dat.cpp index 5990b3ec805..46aca14325f 100644 --- a/searchlib/src/apps/docstore/create-idx-from-dat.cpp +++ b/searchlib/src/apps/docstore/create-idx-from-dat.cpp @@ -78,7 +78,7 @@ generate(uint64_t serialNum, size_t chunks, FastOS_FileInterface & idxFile, size fprintf(stdout, "Failed with lengthError %ld due to '%s'\n", lengthError, e.what()); } } - idxFile.Write2(os.c_str(), os.size()); + idxFile.Write2(os.data(), os.size()); return serialNum; } diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 513e94321e1..610fc58e98f 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -489,9 +489,9 @@ TEST("testResultNodes") { double d(786324.78); nbostream os; os << j << d; - RawResultNode r1(os.c_str(), sizeof(j)); + RawResultNode r1(os.data(), sizeof(j)); EXPECT_EQUAL(r1.getInteger(), 789); - RawResultNode r2(os.c_str() + sizeof(j), sizeof(d)); + RawResultNode r2(os.data() + sizeof(j), sizeof(d)); EXPECT_EQUAL(r2.getFloat(), 786324.78); StringResultNode s1, s2("a"), s3("a"), s4("b"), s5("bb"); @@ -560,7 +560,7 @@ void testStreaming(const Identifiable &v) { EXPECT_EQUAL(os2.size(), os3.size()); ASSERT_TRUE(os2.size() == os3.size()); - EXPECT_EQUAL(0, memcmp(os2.c_str(), os3.c_str(), os3.size())); + EXPECT_EQUAL(0, memcmp(os2.data(), os3.data(), os3.size())); } TEST("testTimeStamp") { diff --git a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp index b7fb3d2b6a1..2f710c5d6e1 100644 --- a/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp +++ b/searchlib/src/tests/features/imported_dot_product/imported_dot_product_test.cpp @@ -111,7 +111,7 @@ struct ArrayFixture : FixtureBase { void check_prepare_state_output(const vespalib::tensor::Tensor & tensor, const ExpectedType & expected) { vespalib::nbostream os; vespalib::tensor::TypedBinaryFormat::serialize(os, tensor); - vespalib::string input_vector(os.c_str(), os.size()); + vespalib::string input_vector(os.data(), os.size()); check_prepare_state_output(".tensor", input_vector, expected); } diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index c4751af5adb..0dced597917 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -63,27 +63,27 @@ class CallBackTest : public TransLogClient::Visitor::Callback private: virtual RPC::Result receive(const Packet & packet) override; virtual void eof() override { _eof = true; } - typedef std::map<SerialNum, ByteBuffer> PacketMap; + typedef std::map<SerialNum, std::unique_ptr<ByteBuffer>> PacketMap; PacketMap _packetMap; public: CallBackTest() : _eof(false) { } size_t size() const { return _packetMap.size(); } bool hasSerial(SerialNum n) const { return (_packetMap.find(n) != _packetMap.end()); } void clear() { _eof = false; _packetMap.clear(); } - const ByteBuffer & packet(SerialNum n) { return (_packetMap.find(n)->second); } + const ByteBuffer & packet(SerialNum n) { return *(_packetMap.find(n)->second); } bool _eof; }; RPC::Result CallBackTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(h.peek(), h.size()).c_str()); while(h.size() > 0) { Packet::Entry e; e.deserialize(h); LOG(info,"CallBackTest::receive (%zu, %zu, %zu)(%s)", h.rp(), h.size(), h.capacity(), myhex(e.data().c_str(), e.data().size()).c_str()); - _packetMap[e.serial()] = ByteBuffer(e.data().c_str(), e.data().size()); + _packetMap[e.serial()] = std::make_unique<ByteBuffer>(e.data().c_str(), e.data().size()); } return RPC::OK; } @@ -103,7 +103,7 @@ public: RPC::Result CallBackManyTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); for(;h.size() > 0; _count++, _value++) { Packet::Entry e; e.deserialize(h); @@ -135,7 +135,7 @@ public: RPC::Result CallBackUpdate::receive(const Packet & packet) { - nbostream_longlivedbuf h(packet.getHandle().c_str(), packet.getHandle().size()); + nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); while (h.size() > 0) { Packet::Entry e; e.deserialize(h); @@ -187,7 +187,7 @@ public: RPC::Result CallBackStatsTest::receive(const Packet & p) { - nbostream_longlivedbuf h(p.getHandle().c_str(), p.getHandle().size()); + nbostream_longlivedbuf h(p.getHandle().data(), p.getHandle().size()); for(;h.size() > 0; ++_count) { Packet::Entry e; e.deserialize(h); @@ -236,13 +236,13 @@ bool Test::partialUpdateTest() nbostream os; os << du; - vespalib::ConstBufferRef bb(os.c_str(), os.size()); + vespalib::ConstBufferRef bb(os.data(), os.size()); LOG(info, "DU : %s", myhex(bb.c_str(), bb.size()).c_str()); Packet::Entry e(7, du.getClass().id(), bb); Packet pa; pa.add(e); pa.close(); - ASSERT_TRUE(session.commit(vespalib::ConstBufferRef(pa.getHandle().c_str(), pa.getHandle().size()))); + ASSERT_TRUE(session.commit(vespalib::ConstBufferRef(pa.getHandle().data(), pa.getHandle().size()))); CallBackUpdate ca; TransLogClient::Visitor::UP visitor = tls.createVisitor("test1", ca); @@ -320,10 +320,10 @@ bool Test::fillDomainTest(TransLogClient::Session * s1, const vespalib::string & ASSERT_TRUE (!b.add(e1)); a.close(); b.close(); - ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(a.getHandle().c_str(), a.getHandle().size()))); - ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(b.getHandle().c_str(), b.getHandle().size()))); + ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(a.getHandle().data(), a.getHandle().size()))); + ASSERT_TRUE (s1->commit(vespalib::ConstBufferRef(b.getHandle().data(), b.getHandle().size()))); try { - s1->commit(vespalib::ConstBufferRef(a.getHandle().c_str(), a.getHandle().size())); + s1->commit(vespalib::ConstBufferRef(a.getHandle().data(), a.getHandle().size())); ASSERT_TRUE(false); } catch (const std::exception & e) { EXPECT_EQUAL(vespalib::string("commit failed with code -2. server says: Exception during commit on " + name + " : Incomming serial number(1) must be bigger than the last one (3)."), e.what()); @@ -340,7 +340,7 @@ bool Test::fillDomainTest(TransLogClient::Session * s1, const vespalib::string & EXPECT_EQUAL(a.range().to(), 3u); Packet::Entry e; - vespalib::nbostream h(a.getHandle().c_str(), a.getHandle().size()); + vespalib::nbostream h(a.getHandle().data(), a.getHandle().size()); e.deserialize(h); e.deserialize(h); e.deserialize(h); @@ -358,13 +358,13 @@ void Test::fillDomainTest(TransLogClient::Session * s1, size_t numPackets, size_ Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&value, sizeof(value))); if ( ! p->add(e) ) { p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); p.reset(new Packet()); ASSERT_TRUE(p->add(e)); } } p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); } } @@ -382,13 +382,13 @@ Test::fillDomainTest(TransLogClient::Session * s1, Packet::Entry e(value+1, j+1, vespalib::ConstBufferRef((const char *)&entryBuffer[0], entryBuffer.size())); if ( ! p->add(e) ) { p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); p.reset(new Packet()); ASSERT_TRUE(p->add(e)); } } p->close(); - ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().c_str(), p->getHandle().size()))); + ASSERT_TRUE(s1->commit(vespalib::ConstBufferRef(p->getHandle().data(), p->getHandle().size()))); } } diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index a0e4b4884a9..74e48081e17 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -17,7 +17,7 @@ LOG_SETUP("translogstress"); -using document::ByteBuffer; +using vespalib::nbostream; using search::Runnable; using vespalib::Monitor; using vespalib::MonitorGuard; @@ -47,10 +47,10 @@ public: BufferGenerator(uint32_t minStrLen, uint32_t maxStrLen) : _rnd(), _minStrLen(minStrLen), _maxStrLen(maxStrLen) {} void setSeed(long seed) { _rnd.srand48(seed); } - ByteBuffer getRandomBuffer(); + nbostream getRandomBuffer(); }; -ByteBuffer +nbostream BufferGenerator::getRandomBuffer() { size_t len = _minStrLen + _rnd.lrand48() % (_maxStrLen - _minStrLen); @@ -59,9 +59,8 @@ BufferGenerator::getRandomBuffer() char c = 'a' + _rnd.lrand48() % ('z' - 'a' + 1); str.push_back(c); } - ByteBuffer buf(str.size() + 1); - buf.putBytes(str.c_str(), str.size() + 1); - buf.flip(); + nbostream buf(str.size() + 1); + buf.write(str.c_str(), str.size() + 1); return buf; } @@ -75,13 +74,13 @@ private: Rand48 _rnd; long _baseSeed; BufferGenerator _bufferGenerator; - const std::vector<document::ByteBuffer> * _buffers; - ByteBuffer _lastGeneratedBuffer; + const std::vector<nbostream> * _buffers; + nbostream _lastGeneratedBuffer; public: EntryGenerator(long baseSeed, const BufferGenerator & bufferGenerator) : - _rnd(), _baseSeed(baseSeed), _bufferGenerator(bufferGenerator), _buffers(NULL), - _lastGeneratedBuffer() {} + _rnd(), _baseSeed(baseSeed), _bufferGenerator(bufferGenerator), _buffers(nullptr), + _lastGeneratedBuffer(0) {} EntryGenerator(const EntryGenerator & rhs) : _rnd(), _baseSeed(rhs._baseSeed), _bufferGenerator(rhs._bufferGenerator), _buffers(rhs._buffers), _lastGeneratedBuffer(rhs._lastGeneratedBuffer) {} @@ -95,7 +94,7 @@ public: SerialNum getRandomSerialNum(SerialNum begin, SerialNum end); Packet::Entry getRandomEntry(SerialNum num); Rand48 & getRnd() { return _rnd; } - void setBuffers(const std::vector<ByteBuffer> & buffers) { + void setBuffers(const std::vector<nbostream> & buffers) { _buffers = &buffers; } }; @@ -118,12 +117,12 @@ EntryGenerator::getRandomEntry(SerialNum num) _rnd.srand48(_baseSeed + num); if (_buffers != NULL) { size_t i = _rnd.lrand48() % _buffers->size(); - const ByteBuffer& buffer = (*_buffers)[i]; - return Packet::Entry(num, 1024, ConstBufferRef(buffer.getBuffer(), buffer.getLength())); + const nbostream& buffer = (*_buffers)[i]; + return Packet::Entry(num, 1024, ConstBufferRef(buffer.data(), buffer.size())); } else { _bufferGenerator.setSeed(_baseSeed + num); _lastGeneratedBuffer = _bufferGenerator.getRandomBuffer(); - return Packet::Entry(num, 1024, ConstBufferRef(_lastGeneratedBuffer.getBuffer(), _lastGeneratedBuffer.getLength())); + return Packet::Entry(num, 1024, ConstBufferRef(_lastGeneratedBuffer.data(), _lastGeneratedBuffer.size())); } } @@ -226,7 +225,7 @@ FeederThread::commitPacket() { _packet.close(); const vespalib::nbostream& stream = _packet.getHandle(); - if (!_session->commit(ConstBufferRef(stream.c_str(), stream.size()))) { + if (!_session->commit(ConstBufferRef(stream.data(), stream.size()))) { throw std::runtime_error(vespalib::make_string ("FeederThread: Failed commiting %s", PacketPrinter::toStr(_packet).c_str())); } else { @@ -708,7 +707,7 @@ TransLogStress::Main() BufferGenerator bufferGenerator(_cfg.minStrLen, _cfg.maxStrLen); bufferGenerator.setSeed(_cfg.baseSeed); - std::vector<ByteBuffer> buffers; + std::vector<nbostream> buffers; for (uint32_t i = 0; i < _cfg.numPreGeneratedBuffers; ++i) { buffers.push_back(bufferGenerator.getRandomBuffer()); } diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp index 4707e8001a8..c7976fd15ae 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp @@ -89,13 +89,13 @@ Chunk::getLid(uint32_t lid) const if (it->getLid() == lid) { #if 1 uint32_t bLid(0), bLen(0); - vespalib::nbostream is(getData().c_str()+it->getOffset(), it->size()); + vespalib::nbostream is(getData().data() + it->getOffset(), it->size()); is >> bLid >> bLen; assert(bLid == lid); assert(bLen == it->netSize()); assert((bLen + 2*sizeof(uint32_t)) == it->size()); #endif - buf = vespalib::ConstBufferRef(getData().c_str() + it->getNetOffset(), it->netSize()); + buf = vespalib::ConstBufferRef(getData().data() + it->getNetOffset(), it->netSize()); } } return buf; diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index 4d76b3fea25..a5bed4c33ce 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -33,7 +33,7 @@ ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const const size_t oldPos(compressed.getDataLen()); compressed.writeInt8(compression.type); compressed.writeInt32(os.size()); - CompressionConfig::Type type(compress(compression, vespalib::ConstBufferRef(os.c_str(), os.size()), compressed, false)); + CompressionConfig::Type type(compress(compression, vespalib::ConstBufferRef(os.data(), os.size()), compressed, false)); if (compression.type != type) { compressed.getData()[oldPos] = type; } diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 994df3237f2..e8504480b7d 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -66,7 +66,7 @@ BlobSet::get(uint32_t lid) const ConstBufferRef buf; for (LidPosition pos : _positions) { if (pos.lid() == lid) { - buf = ConstBufferRef(_buffer.c_str() + pos.offset(), pos.size()); + buf = ConstBufferRef(_buffer.data() + pos.offset(), pos.size()); break; } } diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index eb035ac2a2c..8a06794ee35 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -60,7 +60,7 @@ public: void remove(uint32_t lid); const Positions & getPositions() const { return _positions; } vespalib::ConstBufferRef get(uint32_t lid) const; - vespalib::ConstBufferRef getBuffer() const { return vespalib::ConstBufferRef(_buffer.c_str(), _buffer.size()); } + vespalib::ConstBufferRef getBuffer() const { return vespalib::ConstBufferRef(_buffer.data(), _buffer.size()); } private: Positions _positions; vespalib::nbostream _buffer; diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp index e802ec0a326..70295d81a93 100644 --- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp +++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp @@ -867,13 +867,13 @@ WriteableFileChunk::unconditionallyFlushPendingChunks(const vespalib::LockGuard _pendingDat -= pc.getDataLen(); lastSerial = pc.getLastSerial(); const nbostream &os2(pc.getSerializedIdx()); - os.write(os2.c_str(), os2.size()); + os.write(os2.data(), os2.size()); } } vespalib::system_time timeStamp(vespalib::system_clock::now()); auto idxFile = openIdx(); idxFile->SetPosition(idxFile->GetSize()); - ssize_t wlen = idxFile->Write2(os.c_str(), os.size()); + ssize_t wlen = idxFile->Write2(os.data(), os.size()); updateCurrentDiskFootprint(); if (wlen != static_cast<ssize_t>(os.size())) { diff --git a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp index 0eb85cba4ba..b770b9ffc9d 100644 --- a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp @@ -489,7 +489,7 @@ bool CatFunctionNode::onExecute() const getArg(i).execute(); getArg(i).getResult().serialize(nos); } - static_cast<RawResultNode &>(updateResult()).setBuffer(os.c_str(), os.size()); + static_cast<RawResultNode &>(updateResult()).setBuffer(os.data(), os.size()); return true; } @@ -520,14 +520,14 @@ bool XorBitFunctionNode::internalExecute(const nbostream & os) const { const size_t numBytes(_tmpXor.size()); memset(&_tmpXor[0], 0, numBytes); - const char * s(os.c_str()); + const char * s(os.data()); for (size_t i(0), m(os.size()/numBytes); i < m; i++) { for (size_t j(0), k(numBytes); j < k; j++) { _tmpXor[j] ^= s[j + k*i]; } } for (size_t i((os.size()/numBytes)*numBytes); i < os.size(); i++) { - _tmpXor[i%numBytes] = os.c_str()[i]; + _tmpXor[i%numBytes] = os.data()[i]; } static_cast<RawResultNode &>(updateResult()).setBuffer(&_tmpXor[0], numBytes); return true; @@ -537,7 +537,7 @@ bool MD5BitFunctionNode::internalExecute(const nbostream & os) const { const unsigned int MD5_DIGEST_LENGTH = 16; unsigned char md5ScratchPad[MD5_DIGEST_LENGTH]; - fastc_md5sum(os.c_str(), os.size(), md5ScratchPad); + fastc_md5sum(os.data(), os.size(), md5ScratchPad); static_cast<RawResultNode &>(updateResult()).setBuffer(md5ScratchPad, std::min(sizeof(md5ScratchPad), getNumBytes())); return true; } diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp index 4e522f27ce2..ff41396f66b 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_store.cpp @@ -3,7 +3,6 @@ #include "generic_tensor_store.h" #include <vespa/eval/tensor/tensor.h> #include <vespa/eval/tensor/serialization/typed_binary_format.h> -#include <vespa/document/util/serializable.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/objects/nbostream.h> @@ -15,9 +14,7 @@ using search::datastore::Handle; using vespalib::tensor::Tensor; using vespalib::tensor::TypedBinaryFormat; -namespace search { - -namespace tensor { +namespace search::tensor { constexpr size_t MIN_BUFFER_ARRAYS = 1024; @@ -118,6 +115,4 @@ GenericTensorStore::setTensor(const Tensor &tensor) return raw.ref; } -} // namespace search::tensor - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/transactionlog/common.cpp b/searchlib/src/vespa/searchlib/transactionlog/common.cpp index a84e27b2e53..a5eaa61af12 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/common.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/common.cpp @@ -37,7 +37,7 @@ Packet::Packet(const void * buf, size_t sz) : _limit(sz), _buf(static_cast<const char *>(buf), sz) { - nbostream_longlivedbuf os(_buf.c_str(), sz); + nbostream_longlivedbuf os(_buf.data(), sz); while ( os.size() > 0 ) { Entry e; e.deserialize(os); @@ -55,7 +55,7 @@ bool Packet::merge(const Packet & packet) if (retval) { _count += packet._count; _range.to(packet._range.to()); - _buf.write(packet.getHandle().c_str(), packet.getHandle().size()); + _buf.write(packet.getHandle().data(), packet.getHandle().size()); } return retval; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index fc9518ccf1b..5a64d829183 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -282,7 +282,7 @@ void waitPendingSync(vespalib::Monitor &syncMonitor, bool &pendingSync) void Domain::commit(const Packet & packet) { DomainPart::SP dp(_parts.rbegin()->second); - vespalib::nbostream_longlivedbuf is(packet.getHandle().c_str(), packet.getHandle().size()); + vespalib::nbostream_longlivedbuf is(packet.getHandle().data(), packet.getHandle().size()); Packet::Entry entry; entry.deserialize(is); if (dp->byteSize() > _domainPartSize) { diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index d2838711a51..8a6e833bd1f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -403,7 +403,7 @@ void DomainPart::commit(SerialNum firstSerial, const Packet &packet) { int64_t firstPos(_transLog->GetPosition()); - nbostream_longlivedbuf h(packet.getHandle().c_str(), packet.getHandle().size()); + nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size()); if (_range.from() == 0) { _range.from(firstSerial); } @@ -495,7 +495,7 @@ DomainPart::visit(SerialNumRange &r, Packet &packet) } } else { const nbostream & tmp = start->second.getHandle(); - nbostream_longlivedbuf h(tmp.c_str(), tmp.size()); + nbostream_longlivedbuf h(tmp.data(), tmp.size()); LOG(debug, "Visit partial[%" PRIu64 ", %" PRIu64 "] (%zd, %zd, %zd)", start->second.range().from(), start->second.range().to(), h.rp(), h.size(), h.capacity()); Packet newPacket(h.size()); @@ -585,13 +585,13 @@ DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry) size_t start(os.size()); entry.serialize(os); size_t end(os.size()); - crc = calcCrc(_defaultCrc, os.c_str()+start, end - start); + crc = calcCrc(_defaultCrc, os.data() + start, end - start); os << crc; size_t osSize = os.size(); assert(osSize == len + sizeof(len) + sizeof(uint8_t)); LockGuard guard(_writeLock); - if ( ! file.CheckedWrite(os.c_str(), osSize) ) { + if ( ! file.CheckedWrite(os.data(), osSize) ) { throw runtime_error(handleWriteError("Failed writing the entry.", file, lastKnownGoodPos, entry, end - start)); } _writtenSerial = entry.serial(); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 37903bc21f5..a3528c4f615 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -366,7 +366,7 @@ public: req->SetMethodName("visitCallback"); req->GetParams()->AddString(domain.c_str()); req->GetParams()->AddInt32(id); - req->GetParams()->AddData(packet.getHandle().c_str(), packet.getHandle().size()); + req->GetParams()->AddData(packet.getHandle().data(), packet.getHandle().size()); return send(req); } diff --git a/staging_vespalib/src/tests/objects/identifiable_test.cpp b/staging_vespalib/src/tests/objects/identifiable_test.cpp index b952ad18333..2b4d50cc786 100644 --- a/staging_vespalib/src/tests/objects/identifiable_test.cpp +++ b/staging_vespalib/src/tests/objects/identifiable_test.cpp @@ -161,40 +161,40 @@ void IdentifiableTest::testNboStream() EXPECT_EQUAL(nbostream::ok, s.state()); EXPECT_EQUAL(10u, s.size()); EXPECT_EQUAL(16u, s.capacity()); - EXPECT_EQUAL(0, strncmp(s.c_str()+4, "abcdef", 6)); + EXPECT_EQUAL(0, strncmp(s.data() + 4, "abcdef", 6)); } { nbostream s(8); EXPECT_EQUAL(0u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - const char * prev = s.c_str(); + const char * prev = s.data(); s << "ABCD"; EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); s << "A long string that will cause resizing"; EXPECT_EQUAL(50u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - EXPECT_NOT_EQUAL(prev, s.c_str()); + EXPECT_NOT_EQUAL(prev, s.data()); } { nbostream s(8); EXPECT_EQUAL(0u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - const char * prev = s.c_str(); + const char * prev = s.data(); s << "ABCD"; EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(8u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); s.reserve(50); - EXPECT_NOT_EQUAL(prev, s.c_str()); + EXPECT_NOT_EQUAL(prev, s.data()); EXPECT_EQUAL(8u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - prev = s.c_str(); + prev = s.data(); s << "A long string that will cause resizing"; EXPECT_EQUAL(50u, s.size()); EXPECT_EQUAL(64u, s.capacity()); - EXPECT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(prev, s.data()); } { nbostream s; diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp index 6cc2af1fc90..2465d5f9d9b 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -244,7 +244,7 @@ int Identifiable::onCmp(const Identifiable& b) const nbs << b; size_t minLength(std::min(as.size(), bs.size())); if (minLength > 0) { - diff = memcmp(as.c_str(), bs.c_str(), minLength); + diff = memcmp(as.data(), bs.data(), minLength); } if (diff == 0) { diff = as.size() - bs.size(); diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 57e88873e81..a10cad70579 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -27,7 +27,7 @@ GrowableByteBuffer::allocate(uint32_t len) } void -GrowableByteBuffer::putBytes(const char* buffer, uint32_t length) +GrowableByteBuffer::putBytes(const void * buffer, uint32_t length) { char* buf = allocate(length); memcpy(buf, buffer, length); diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h index d32afeeabee..afbde04fb9b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -45,7 +45,7 @@ public: /** Adds the given buffer to this buffer. */ - void putBytes(const char* buffer, uint32_t length); + void putBytes(const void * buffer, uint32_t length); /** Adds a short to the buffer. diff --git a/storage/src/tests/common/message_sender_stub.cpp b/storage/src/tests/common/message_sender_stub.cpp index c127f9071e5..a82d45b0b99 100644 --- a/storage/src/tests/common/message_sender_stub.cpp +++ b/storage/src/tests/common/message_sender_stub.cpp @@ -22,9 +22,7 @@ MessageSenderStub::getLastCommand(bool verbose) const } std::string -MessageSenderStub::dumpMessage(const api::StorageMessage& msg, - bool includeAddress, - bool verbose) const +MessageSenderStub::dumpMessage(const api::StorageMessage& msg, bool includeAddress, bool verbose) const { std::ostringstream ost; @@ -67,9 +65,7 @@ MessageSenderStub::getLastReply(bool verbose) const throw std::logic_error("Expected reply where there was none"); } - return dumpMessage(*replies.back(), - true, - verbose); + return dumpMessage(*replies.back(),true, verbose); } diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 64306fa7c24..4576f8a08f8 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1501,7 +1501,7 @@ TEST_F(FileStorManagerTest, visiting) { for (uint32_t i=3; i<docCount; ++i) { auto reply = std::dynamic_pointer_cast<api::BucketInfoReply>(top.getReply(i)); ASSERT_TRUE(reply.get()); - ASSERT_TRUE(reply->getResult().success()) << reply->getResult().toString(); + ASSERT_TRUE(reply->getResult().success()) << reply->getResult(); info = reply->getBucketInfo(); } diff --git a/storage/src/tests/visiting/visitormanagertest.cpp b/storage/src/tests/visiting/visitormanagertest.cpp index b7eb7fee3ec..20934d04eaa 100644 --- a/storage/src/tests/visiting/visitormanagertest.cpp +++ b/storage/src/tests/visiting/visitormanagertest.cpp @@ -20,6 +20,7 @@ #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/objects/nbostream.h> #include <gmock/gmock.h> #include <optional> #include <thread> @@ -337,7 +338,7 @@ int getTotalSerializedSize(const std::vector<document::Document::SP>& docs) { int total = 0; for (size_t i = 0; i < docs.size(); ++i) { - total += int(docs[i]->serialize()->getLength()); + total += int(docs[i]->serialize().size()); } return total; } diff --git a/storage/src/vespa/storage/common/bucketmessages.cpp b/storage/src/vespa/storage/common/bucketmessages.cpp index e92e2d4c3bf..1a4dc61a3ce 100644 --- a/storage/src/vespa/storage/common/bucketmessages.cpp +++ b/storage/src/vespa/storage/common/bucketmessages.cpp @@ -14,7 +14,7 @@ ReadBucketList::ReadBucketList(BucketSpace bucketSpace, spi::PartitionId partiti _partition(partition) { } -ReadBucketList::~ReadBucketList() { } +ReadBucketList::~ReadBucketList() = default; document::Bucket ReadBucketList::getBucket() const @@ -38,7 +38,7 @@ ReadBucketListReply::ReadBucketListReply(const ReadBucketList& cmd) _partition(cmd.getPartition()) { } -ReadBucketListReply::~ReadBucketListReply() { } +ReadBucketListReply::~ReadBucketListReply() = default; document::Bucket ReadBucketListReply::getBucket() const @@ -66,7 +66,7 @@ ReadBucketInfo::ReadBucketInfo(const document::Bucket &bucket) _bucket(bucket) { } -ReadBucketInfo::~ReadBucketInfo() { } +ReadBucketInfo::~ReadBucketInfo() = default; void ReadBucketInfo::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -92,7 +92,7 @@ ReadBucketInfoReply::ReadBucketInfoReply(const ReadBucketInfo& cmd) _bucket(cmd.getBucket()) { } -ReadBucketInfoReply::~ReadBucketInfoReply() { } +ReadBucketInfoReply::~ReadBucketInfoReply() = default; void ReadBucketInfoReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "ReadBucketInfoReply()"; @@ -117,7 +117,7 @@ RepairBucketCommand::RepairBucketCommand(const document::Bucket &bucket, uint16_ setPriority(LOW); } -RepairBucketCommand::~RepairBucketCommand() { } +RepairBucketCommand::~RepairBucketCommand() = default; void RepairBucketCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -153,7 +153,7 @@ RepairBucketReply::RepairBucketReply(const RepairBucketCommand& cmd, const api:: _altered(false) { } -RepairBucketReply::~RepairBucketReply() { } +RepairBucketReply::~RepairBucketReply() = default; void RepairBucketReply::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -180,7 +180,7 @@ BucketDiskMoveCommand::BucketDiskMoveCommand(const document::Bucket &bucket, setPriority(LOW); } -BucketDiskMoveCommand::~BucketDiskMoveCommand() { } +BucketDiskMoveCommand::~BucketDiskMoveCommand() = default; void BucketDiskMoveCommand::setBucketId(const document::BucketId& id) @@ -208,14 +208,13 @@ BucketDiskMoveReply::BucketDiskMoveReply(const BucketDiskMoveCommand& cmd, _dstDisk(cmd.getDstDisk()) { } -BucketDiskMoveReply::~BucketDiskMoveReply() { } +BucketDiskMoveReply::~BucketDiskMoveReply() = default; void BucketDiskMoveReply::print(std::ostream& out, bool, const std::string&) const { out << "BucketDiskMoveReply(" << _bucket.getBucketId() << ", source " << _srcDisk - << ", target " << _dstDisk << ", " << _bucketInfo << ", " - << getResult() << ")"; + << ", target " << _dstDisk << ", " << _bucketInfo << ", " << getResult() << ")"; } std::unique_ptr<api::StorageReply> @@ -236,7 +235,7 @@ InternalBucketJoinCommand::InternalBucketJoinCommand(const document::Bucket &buc // them higher than getting more bucket info lists. } -InternalBucketJoinCommand::~InternalBucketJoinCommand() { } +InternalBucketJoinCommand::~InternalBucketJoinCommand() = default; void InternalBucketJoinCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { @@ -255,7 +254,7 @@ InternalBucketJoinReply::InternalBucketJoinReply(const InternalBucketJoinCommand _bucketInfo(info) { } -InternalBucketJoinReply::~InternalBucketJoinReply() { } +InternalBucketJoinReply::~InternalBucketJoinReply() = default; void InternalBucketJoinReply::print(std::ostream& out, bool verbose, const std::string& indent) const diff --git a/storage/src/vespa/storage/common/storagelink.cpp b/storage/src/vespa/storage/common/storagelink.cpp index f73eb3ea36d..431c90b27f2 100644 --- a/storage/src/vespa/storage/common/storagelink.cpp +++ b/storage/src/vespa/storage/common/storagelink.cpp @@ -4,6 +4,7 @@ #include "bucketmessages.h" #include <vespa/vespalib/util/backtrace.h> #include <sstream> +#include <cassert> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".application.link"); @@ -141,7 +142,7 @@ void StorageLink::sendDown(const StorageMessage::SP& msg) sendUp(reply); } } else { - ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); + ost << " Return code: " << static_cast<const StorageReply&>(*msg).getResult(); LOGBP(warning, "%s", ost.str().c_str()); } } else if (!_down->onDown(msg)) { @@ -181,7 +182,7 @@ void StorageLink::sendUp(const shared_ptr<StorageMessage> & msg) sendDown(reply); } } else { - ost << " Return code: " << static_cast<StorageReply&>(*msg).getResult(); + ost << " Return code: " << static_cast<const StorageReply&>(*msg).getResult(); LOGBP(warning, "%s", ost.str().c_str()); } } else if (!_up->onUp(msg)) { diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp index e143f4d8570..15a57c1e7ee 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.cpp @@ -2,6 +2,7 @@ #include "simplemaintenancescanner.h" #include <vespa/storage/distributor/distributor_bucket_space.h> #include <ostream> +#include <cassert> namespace storage::distributor { @@ -16,10 +17,10 @@ SimpleMaintenanceScanner::SimpleMaintenanceScanner(BucketPriorityDatabase& bucke { } -SimpleMaintenanceScanner::~SimpleMaintenanceScanner() {} +SimpleMaintenanceScanner::~SimpleMaintenanceScanner() = default; -SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() {} -SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() {} +SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats() = default; +SimpleMaintenanceScanner::PendingMaintenanceStats::~PendingMaintenanceStats() = default; SimpleMaintenanceScanner::PendingMaintenanceStats::PendingMaintenanceStats(const PendingMaintenanceStats &) = default; SimpleMaintenanceScanner::PendingMaintenanceStats & SimpleMaintenanceScanner::PendingMaintenanceStats::operator = (const PendingMaintenanceStats &) = default; diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp index 8e6494a588d..60c1137bd6d 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp @@ -8,8 +8,7 @@ #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.statbucket"); -namespace storage { -namespace distributor { +namespace storage::distributor { StatBucketOperation::StatBucketOperation( [[maybe_unused]] DistributorComponent& manager, @@ -21,7 +20,7 @@ StatBucketOperation::StatBucketOperation( { } -StatBucketOperation::~StatBucketOperation() {} +StatBucketOperation::~StatBucketOperation() = default; void StatBucketOperation::onClose(DistributorMessageSender& sender) @@ -36,8 +35,7 @@ StatBucketOperation::onStart(DistributorMessageSender& sender) { std::vector<uint16_t> nodes; - BucketDatabase::Entry entry( - _bucketSpace.getBucketDatabase().get(_command->getBucketId())); + BucketDatabase::Entry entry(_bucketSpace.getBucketDatabase().get(_command->getBucketId())); if (entry.valid()) { nodes = entry->getNodes(); @@ -103,5 +101,4 @@ StatBucketOperation::onReceive(DistributorMessageSender& sender, const std::shar } } -} // distributor -} // storage +} diff --git a/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp b/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp index abd9778d72c..9e3230a0f34 100644 --- a/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp +++ b/storage/src/vespa/storage/distributor/throttlingoperationstarter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "throttlingoperationstarter.h" +#include <cassert> namespace storage::distributor { @@ -10,8 +11,7 @@ ThrottlingOperationStarter::ThrottlingOperation::~ThrottlingOperation() } bool -ThrottlingOperationStarter::canStart(uint32_t currentOperationCount, - Priority priority) const +ThrottlingOperationStarter::canStart(uint32_t currentOperationCount, Priority priority) const { uint32_t variablePending(_maxPending - _minPending); uint32_t maxPendingForPri(_minPending + variablePending*((255.0 - priority) / 255.0)); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index bfc89a70a85..65d4035a3dd 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -36,11 +36,8 @@ namespace api { class StorageReply; } -class BucketMergeTest; -class DiskInfo; struct FileStorManagerTest; class ReadBucketList; -class ModifiedBucketCheckerThread; class BucketOwnershipNotifier; class AbortBucketOperationsCommand; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 37e1d818bb8..e5e358cbb60 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -506,18 +506,11 @@ MergeHandler::fetchLocalData( assert(doc != 0); assertContainedInBucket(doc->getId(), bucket, idFactory); e._docName = doc->getId().toString(); - { - vespalib::nbostream stream; - doc->serializeHeader(stream); - e._headerBlob.resize(stream.size()); - memcpy(&e._headerBlob[0], stream.peek(), stream.size()); - } - { - vespalib::nbostream stream; - doc->serializeBody(stream); - e._bodyBlob.resize(stream.size()); - memcpy(&e._bodyBlob[0], stream.peek(), stream.size()); - } + vespalib::nbostream stream; + doc->serialize(stream); + e._headerBlob.resize(stream.size()); + memcpy(&e._headerBlob[0], stream.peek(), stream.size()); + e._bodyBlob.clear(); } else { const DocumentId* docId = docEntry.getDocumentId(); assert(docId != 0); @@ -556,11 +549,11 @@ MergeHandler::deserializeDiffDocument( const api::ApplyBucketDiffCommand::Entry& e, const document::DocumentTypeRepo& repo) const { - Document::UP doc(new Document); - using document::ByteBuffer; - ByteBuffer hbuf(&e._headerBlob[0], e._headerBlob.size()); + auto doc = std::make_unique<Document>(); + vespalib::nbostream hbuf(&e._headerBlob[0], e._headerBlob.size()); if (e._bodyBlob.size() > 0) { - ByteBuffer bbuf(&e._bodyBlob[0], e._bodyBlob.size()); + // TODO Remove this branch and add warning on error. + vespalib::nbostream bbuf(&e._bodyBlob[0], e._bodyBlob.size()); doc->deserialize(repo, hbuf, bbuf); } else { doc->deserialize(repo, hbuf); diff --git a/storage/src/vespa/storage/storageserver/opslogger.cpp b/storage/src/vespa/storage/storageserver/opslogger.cpp index 6fc9795993e..b6bceabf7a1 100644 --- a/storage/src/vespa/storage/storageserver/opslogger.cpp +++ b/storage/src/vespa/storage/storageserver/opslogger.cpp @@ -77,7 +77,7 @@ OpsLogger::onPutReply(const std::shared_ptr<api::PutReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tPUT\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -94,7 +94,7 @@ OpsLogger::onUpdateReply(const std::shared_ptr<api::UpdateReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tUPDATE\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -111,7 +111,7 @@ OpsLogger::onRemoveReply(const std::shared_ptr<api::RemoveReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tREMOVE\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; @@ -128,7 +128,7 @@ OpsLogger::onGetReply(const std::shared_ptr<api::GetReply>& msg) std::ostringstream ost; ost << _component.getClock().getTimeInSeconds().getTime() << "\tGET\t" << msg->getDocumentId() << "\t" - << msg->getResult().toString() << "\n"; + << msg->getResult() << "\n"; { vespalib::LockGuard lock(_lock); if (_targetFile == nullptr) return false; diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp index f0e64f0dfd1..f3240f0663b 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp @@ -16,37 +16,6 @@ MessageMemoryUseMetricSet::MessageMemoryUseMetricSet(metrics::MetricSet* owner) MessageMemoryUseMetricSet::~MessageMemoryUseMetricSet() = default; -DocumentSerializationMetricSet::DocumentSerializationMetricSet(metrics::MetricSet* owner) - : metrics::MetricSet("document_serialization", {{"docserialization"}}, - "Counts of document serialization of various types", owner), - usedCachedSerializationCount( - "cached_serialization_count", {{"docserialization"}}, - "Number of times we didn't need to serialize the document as " - "we already had serialized version cached", this), - compressedDocumentCount( - "compressed_serialization_count", {{"docserialization"}}, - "Number of times we compressed document when serializing", - this), - compressionDidntHelpCount( - "compressed_didnthelp_count", {{"docserialization"}}, - "Number of times we compressed document when serializing, but " - "the compressed version was bigger, so it was dumped", this), - uncompressableCount( - "uncompressable_serialization_count", {{"docserialization"}}, - "Number of times we didn't attempt compression as document " - "had already been tagged uncompressable", this), - serializedUncompressed( - "uncompressed_serialization_count", {{"docserialization"}}, - "Number of times we serialized a document uncompressed", this), - inputWronglySerialized( - "input_wrongly_serialized_count", {{"docserialization"}}, - "Number of times we reserialized a document because the " - "compression it had in cache did not match what was configured", - this) -{} - -DocumentSerializationMetricSet::~DocumentSerializationMetricSet() = default; - StorageMetricSet::StorageMetricSet() : metrics::MetricSet("server", {{"memory"}}, "Metrics for VDS applications"), @@ -54,28 +23,12 @@ StorageMetricSet::StorageMetricSet() memoryUse_messages(this), memoryUse_visiting("memoryusage_visiting", {{"memory"}}, "Message use from visiting", this), - documentSerialization(this), tls_metrics(this) {} StorageMetricSet::~StorageMetricSet() = default; void StorageMetricSet::updateMetrics() { - document::SerializableArray::Statistics stats( - document::SerializableArray::getStatistics()); - - documentSerialization.usedCachedSerializationCount.set( - stats._usedCachedSerializationCount); - documentSerialization.compressedDocumentCount.set( - stats._compressedDocumentCount); - documentSerialization.compressionDidntHelpCount.set( - stats._compressionDidntHelpCount); - documentSerialization.uncompressableCount.set( - stats._uncompressableCount); - documentSerialization.serializedUncompressed.set( - stats._serializedUncompressed); - documentSerialization.inputWronglySerialized.set( - stats._inputWronglySerialized); // Delta snapshotting is destructive, so if an explicit snapshot is triggered // (instead of just regular periodic snapshots), some events will effectively diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h index e9378010540..49795c63324 100644 --- a/storage/src/vespa/storage/storageserver/storagemetricsset.h +++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h @@ -3,7 +3,6 @@ #pragma once #include "tls_statistics_metrics_wrapper.h" - #include <vespa/metrics/metrics.h> namespace storage { @@ -17,21 +16,8 @@ public: metrics::LongValueMetric highpri; metrics::LongValueMetric veryhighpri; - MessageMemoryUseMetricSet(metrics::MetricSet* owner); - ~MessageMemoryUseMetricSet(); -}; - -struct DocumentSerializationMetricSet : public metrics::MetricSet -{ - metrics::LongCountMetric usedCachedSerializationCount; - metrics::LongCountMetric compressedDocumentCount; - metrics::LongCountMetric compressionDidntHelpCount; - metrics::LongCountMetric uncompressableCount; - metrics::LongCountMetric serializedUncompressed; - metrics::LongCountMetric inputWronglySerialized; - - DocumentSerializationMetricSet(metrics::MetricSet* owner); - ~DocumentSerializationMetricSet(); + explicit MessageMemoryUseMetricSet(metrics::MetricSet* owner); + ~MessageMemoryUseMetricSet() override; }; struct StorageMetricSet : public metrics::MetricSet @@ -39,12 +25,11 @@ struct StorageMetricSet : public metrics::MetricSet metrics::LongValueMetric memoryUse; MessageMemoryUseMetricSet memoryUse_messages; metrics::LongValueMetric memoryUse_visiting; - DocumentSerializationMetricSet documentSerialization; TlsStatisticsMetricsWrapper tls_metrics; StorageMetricSet(); - ~StorageMetricSet(); + ~StorageMetricSet() override; void updateMetrics(); }; diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index 7c69a232af0..80e74e890a1 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -2,7 +2,7 @@ #include "recoveryvisitor.h" - +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -36,10 +36,9 @@ RecoveryVisitor::handleDocuments(const document::BucketId& bid, { vespalib::LockGuard guard(_mutex); - LOG(debug, "Visitor %s handling block of %zu documents.", - _id.c_str(), entries.size()); + LOG(debug, "Visitor %s handling block of %zu documents.", _id.c_str(), entries.size()); - documentapi::DocumentListMessage* cmd = NULL; + documentapi::DocumentListMessage* cmd = nullptr; { CommandMap::iterator iter = _activeCommands.find(bid); @@ -71,7 +70,7 @@ RecoveryVisitor::handleDocuments(const document::BucketId& bid, } } - hitCounter.addHit(doc->getId(), doc->serialize()->getLength()); + hitCounter.addHit(doc->getId(), doc->serialize().size()); int64_t timestamp = doc->getLastModified(); cmd->getDocuments().push_back(documentapi::DocumentListMessage::Entry( diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index e68a8fdbc8c..1da2acfed9c 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -12,16 +12,13 @@ #include "visitor.h" #include <vespa/storageapi/message/datagram.h> -namespace documentapi { -class DocumentListMessage; -} +namespace documentapi { class DocumentListMessage; } namespace storage { class RecoveryVisitor : public Visitor { public: - RecoveryVisitor(StorageComponent&, - const vdslib::Parameters& params); + RecoveryVisitor(StorageComponent&, const vdslib::Parameters& params); private: void handleDocuments(const document::BucketId& bucketId, @@ -43,12 +40,11 @@ struct RecoveryVisitorFactory : public VisitorFactory { VisitorEnvironment::UP makeVisitorEnvironment(StorageComponent&) override { - return VisitorEnvironment::UP(new VisitorEnvironment); + return std::make_unique<VisitorEnvironment>(); }; Visitor* - makeVisitor(StorageComponent& c, VisitorEnvironment&, - const vdslib::Parameters& params) override + makeVisitor(StorageComponent& c, VisitorEnvironment&, const vdslib::Parameters& params) override { return new RecoveryVisitor(c, params); } diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index 4b213dff1d5..bdd066e8a4a 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -44,17 +44,12 @@ Visitor::HitCounter::addHit(const document::DocumentId& , uint32_t size) } void -Visitor::HitCounter::updateVisitorStatistics( - vdslib::VisitorStatistics& statistics) +Visitor::HitCounter::updateVisitorStatistics(vdslib::VisitorStatistics& statistics) { - statistics.setDocumentsReturned( - statistics.getDocumentsReturned() + _firstPassHits); - statistics.setBytesReturned( - statistics.getBytesReturned() + _firstPassBytes); - statistics.setSecondPassDocumentsReturned( - statistics.getSecondPassDocumentsReturned() + _secondPassHits); - statistics.setSecondPassBytesReturned( - statistics.getSecondPassBytesReturned() + _secondPassBytes); + statistics.setDocumentsReturned(statistics.getDocumentsReturned() + _firstPassHits); + statistics.setBytesReturned(statistics.getBytesReturned() + _firstPassBytes); + statistics.setSecondPassDocumentsReturned(statistics.getSecondPassDocumentsReturned() + _secondPassHits); + statistics.setSecondPassBytesReturned(statistics.getSecondPassBytesReturned() + _secondPassBytes); } Visitor::VisitorTarget::MessageMeta::MessageMeta( @@ -68,8 +63,7 @@ Visitor::VisitorTarget::MessageMeta::MessageMeta( { } -Visitor::VisitorTarget::MessageMeta::MessageMeta( - Visitor::VisitorTarget::MessageMeta&& rhs) noexcept +Visitor::VisitorTarget::MessageMeta::MessageMeta(Visitor::VisitorTarget::MessageMeta&& rhs) noexcept : messageId(rhs.messageId), retryCount(rhs.retryCount), memoryUsage(rhs.memoryUsage), @@ -78,9 +72,7 @@ Visitor::VisitorTarget::MessageMeta::MessageMeta( { } -Visitor::VisitorTarget::MessageMeta::~MessageMeta() -{ -} +Visitor::VisitorTarget::MessageMeta::~MessageMeta() = default; Visitor::VisitorTarget::MessageMeta& Visitor::VisitorTarget::MessageMeta::operator=( diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index b90153c9517..0cfd2160497 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -24,8 +24,7 @@ ProtocolSerialization4_2::ProtocolSerialization4_2( { } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::GetCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -41,14 +40,12 @@ ProtocolSerialization4_2::onDecodeGetCommand(BBuf& buf) const document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); bool headerOnly(SH::getBoolean(buf)); - api::GetCommand::UP msg( - new api::GetCommand(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp)); + auto msg = std::make_unique<api::GetCommand>(bucket, did, headerOnly ? "[header]" : "[all]", beforeTimestamp); onDecodeCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::RemoveCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -62,13 +59,12 @@ ProtocolSerialization4_2::onDecodeRemoveCommand(BBuf& buf) const document::DocumentId did(SH::getString(buf)); document::Bucket bucket = getBucket(buf); api::Timestamp timestamp(SH::getLong(buf)); - api::RemoveCommand::UP msg(new api::RemoveCommand(bucket, did, timestamp)); + auto msg = std::make_unique<api::RemoveCommand>(bucket, did, timestamp); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::RevertCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RevertCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putInt(msg.getRevertTokens().size()); @@ -86,13 +82,12 @@ ProtocolSerialization4_2::onDecodeRevertCommand(BBuf& buf) const for (uint32_t i=0, n=tokens.size(); i<n; ++i) { tokens[i] = SH::getLong(buf); } - api::RevertCommand::UP msg(new api::RevertCommand(bucket, tokens)); + auto msg = std::make_unique<api::RevertCommand>(bucket, tokens); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::CreateBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { putBucket(msg.getBucket(), buf); onEncodeBucketInfoCommand(buf, msg); @@ -102,13 +97,12 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeCreateBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); + auto msg = std::make_unique<api::CreateBucketCommand>(bucket); onDecodeBucketInfoCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::MergeBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -135,14 +129,12 @@ ProtocolSerialization4_2::onDecodeMergeBucketCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } api::Timestamp timestamp(SH::getLong(buf)); - api::MergeBucketCommand::UP msg( - new api::MergeBucketCommand(bucket, nodes, timestamp)); + auto msg = std::make_unique<api::MergeBucketCommand>(bucket, nodes, timestamp); onDecodeCommand(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::GetBucketDiffCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::GetBucketDiffCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -174,8 +166,7 @@ ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } api::Timestamp timestamp = SH::getLong(buf); - api::GetBucketDiffCommand::UP msg( - new api::GetBucketDiffCommand(bucket, nodes, timestamp)); + auto msg = std::make_unique<api::GetBucketDiffCommand>(bucket, nodes, timestamp); std::vector<api::GetBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -190,8 +181,7 @@ ProtocolSerialization4_2::onDecodeGetBucketDiffCommand(BBuf& buf) const return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { putBucket(msg.getBucket(), buf); const std::vector<api::MergeBucketCommand::Node>& nodes(msg.getNodes()); @@ -201,18 +191,15 @@ void ProtocolSerialization4_2::onEncode( buf.putBoolean(nodes[i].sourceOnly); } buf.putInt(msg.getMaxBufferSize()); - const std::vector<api::ApplyBucketDiffCommand::Entry>& entries( - msg.getDiff()); + const std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg.getDiff()); buf.putInt(entries.size()); for (uint32_t i=0; i<entries.size(); ++i) { onEncodeDiffEntry(buf, entries[i]._entry); buf.putString(entries[i]._docName); buf.putInt(entries[i]._headerBlob.size()); - buf.putBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.putBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); buf.putInt(entries[i]._bodyBlob.size()); - buf.putBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.putBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onEncodeBucketInfoCommand(buf, msg); } @@ -231,8 +218,7 @@ ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const nodes.push_back(Node(index, sourceOnly)); } uint32_t maxBufferSize(SH::getInt(buf)); - api::ApplyBucketDiffCommand::UP msg( - new api::ApplyBucketDiffCommand(bucket, nodes, maxBufferSize)); + auto msg = std::make_unique<api::ApplyBucketDiffCommand>(bucket, nodes, maxBufferSize); std::vector<api::ApplyBucketDiffCommand::Entry>& entries(msg->getDiff()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -248,15 +234,13 @@ ProtocolSerialization4_2::onDecodeApplyBucketDiffCommand(BBuf& buf) const buf.incPos(headerSize); } entries[i]._headerBlob.resize(headerSize); - buf.getBytes(&entries[i]._headerBlob[0], - entries[i]._headerBlob.size()); + buf.getBytes(&entries[i]._headerBlob[0], entries[i]._headerBlob.size()); uint32_t bodySize = SH::getInt(buf); if (bodySize > buf.getRemaining()) { buf.incPos(bodySize); } entries[i]._bodyBlob.resize(bodySize); - buf.getBytes(&entries[i]._bodyBlob[0], - entries[i]._bodyBlob.size()); + buf.getBytes(&entries[i]._bodyBlob[0], entries[i]._bodyBlob.size()); } onDecodeBucketInfoCommand(buf, *msg); return msg; @@ -274,11 +258,9 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RequestBucketInfoReply } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, BBuf& buf) const { - api::RequestBucketInfoReply::UP msg(new api::RequestBucketInfoReply( - static_cast<const api::RequestBucketInfoCommand&>(cmd))); + auto msg = std::make_unique<api::RequestBucketInfoReply>(static_cast<const api::RequestBucketInfoCommand&>(cmd)); api::RequestBucketInfoReply::EntryVector & entries(msg->getBucketInfo()); uint32_t entryCount = SH::getInt(buf); if (entryCount > buf.getRemaining()) { @@ -294,8 +276,7 @@ ProtocolSerialization4_2::onDecodeRequestBucketInfoReply(const SCmd& cmd, return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const { putBucket(msg.getBucket(), buf); putBucketInfo(msg.getBucketInfo(), buf); @@ -307,30 +288,25 @@ ProtocolSerialization4_2::onDecodeNotifyBucketChangeCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); api::BucketInfo info(getBucketInfo(buf)); - api::NotifyBucketChangeCommand::UP msg( - new api::NotifyBucketChangeCommand(bucket, info)); + auto msg = std::make_unique<api::NotifyBucketChangeCommand>(bucket, info); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::NotifyBucketChangeReply& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::NotifyBucketChangeReply& msg) const { onEncodeReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization4_2::onDecodeNotifyBucketChangeReply(const SCmd& cmd,BBuf& buf) const { - api::NotifyBucketChangeReply::UP msg(new api::NotifyBucketChangeReply( - static_cast<const api::NotifyBucketChangeCommand&>(cmd))); + auto msg = std::make_unique<api::NotifyBucketChangeReply>(static_cast<const api::NotifyBucketChangeCommand&>(cmd)); onDecodeReply(buf, *msg); return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::SplitBucketCommand& msg) const +void ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::SplitBucketCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putByte(msg.getMinSplitBits()); @@ -344,7 +320,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); - api::SplitBucketCommand::UP msg(new api::SplitBucketCommand(bucket)); + auto msg = std::make_unique<api::SplitBucketCommand>(bucket); msg->setMinSplitBits(SH::getByte(buf)); msg->setMaxSplitBits(SH::getByte(buf)); msg->setMinByteSize(SH::getInt(buf)); @@ -353,30 +329,24 @@ ProtocolSerialization4_2::onDecodeSplitBucketCommand(BBuf& buf) const return msg; } -void ProtocolSerialization4_2::onEncode( - GBBuf&, const api::SetBucketStateCommand&) const +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateCommand&) const { - throw vespalib::IllegalStateException("Unsupported serialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); } api::StorageCommand::UP ProtocolSerialization4_2::onDecodeSetBucketStateCommand(BBuf&) const { - throw vespalib::IllegalStateException("Unsupported deserialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); } -void ProtocolSerialization4_2::onEncode( - GBBuf&, const api::SetBucketStateReply&) const +void ProtocolSerialization4_2::onEncode(GBBuf&, const api::SetBucketStateReply&) const { - throw vespalib::IllegalStateException("Unsupported serialization", - VESPA_STRLOC); + throw vespalib::IllegalStateException("Unsupported serialization", VESPA_STRLOC); } api::StorageReply::UP -ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, - BBuf&) const +ProtocolSerialization4_2::onDecodeSetBucketStateReply(const SCmd&, BBuf&) const { throw vespalib::IllegalStateException("Unsupported deserialization", VESPA_STRLOC); } @@ -404,11 +374,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::CreateVisitorCommand& buf.putBoolean(msg.getFieldSet() == "[header]"); buf.putBoolean(msg.visitInconsistentBuckets()); buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - - uint32_t size = msg.getParameters().getSerializedSize(); - char* docBuffer = buf.allocate(size); - document::ByteBuffer bbuf(docBuffer, size); - msg.getParameters().serialize(bbuf); + msg.getParameters().serialize(buf); onEncodeCommand(buf, msg); } @@ -420,8 +386,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); - api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); + auto msg = std::make_unique<api::CreateVisitorCommand>(bucketSpace, libraryName, instanceId, selection); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); @@ -450,7 +415,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const msg->setVisitInconsistentBuckets(); } msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(getTypeRepo(), buf); + msg->getParameters().deserialize(buf); onDecodeCommand(buf, *msg); msg->setVisitorDispatcherVersion(42); @@ -458,8 +423,7 @@ ProtocolSerialization4_2::onDecodeCreateVisitorCommand(BBuf& buf) const } void -ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::DestroyVisitorCommand& msg) const +ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorCommand& msg) const { buf.putString(msg.getInstanceId()); onEncodeCommand(buf, msg); @@ -469,7 +433,7 @@ api::StorageCommand::UP ProtocolSerialization4_2::onDecodeDestroyVisitorCommand(BBuf& buf) const { vespalib::stringref instanceId = SH::getString(buf); - api::DestroyVisitorCommand::UP msg(new api::DestroyVisitorCommand(instanceId)); + auto msg = std::make_unique<api::DestroyVisitorCommand>(instanceId); onDecodeCommand(buf, *msg); return msg; } @@ -483,7 +447,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::DestroyVisitorReply& m api::StorageReply::UP ProtocolSerialization4_2::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const { - api::DestroyVisitorReply::UP msg(new api::DestroyVisitorReply(static_cast<const api::DestroyVisitorCommand&>(cmd))); + auto msg = std::make_unique<api::DestroyVisitorReply>(static_cast<const api::DestroyVisitorCommand&>(cmd)); onDecodeReply(buf, *msg); return msg; } @@ -502,8 +466,7 @@ ProtocolSerialization4_2::onDecodeRemoveLocationCommand(BBuf& buf) const vespalib::stringref documentSelection = SH::getString(buf); document::Bucket bucket = getBucket(buf); - api::RemoveLocationCommand::UP msg; - msg.reset(new api::RemoveLocationCommand(documentSelection, bucket)); + auto msg = std::make_unique<api::RemoveLocationCommand>(documentSelection, bucket); onDecodeCommand(buf, *msg); return msg; } @@ -517,7 +480,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::RemoveLocationReply& m api::StorageReply::UP ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { - api::RemoveLocationReply::UP msg(new api::RemoveLocationReply(static_cast<const api::RemoveLocationCommand&>(cmd))); + auto msg = std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); onDecodeBucketInfoReply(buf, *msg); return msg; } @@ -525,15 +488,13 @@ ProtocolSerialization4_2::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf // Utility functions for serialization void -ProtocolSerialization4_2::onEncodeBucketInfoCommand( - GBBuf& buf, const api::BucketInfoCommand& msg) const +ProtocolSerialization4_2::onEncodeBucketInfoCommand(GBBuf& buf, const api::BucketInfoCommand& msg) const { onEncodeCommand(buf, msg); } void -ProtocolSerialization4_2::onDecodeBucketInfoCommand( - BBuf& buf, api::BucketInfoCommand& msg) const +ProtocolSerialization4_2::onDecodeBucketInfoCommand(BBuf& buf, api::BucketInfoCommand& msg) const { onDecodeCommand(buf, msg); } @@ -547,8 +508,7 @@ ProtocolSerialization4_2::onEncode(GBBuf& buf, const api::ReturnCode& rc) const } void -ProtocolSerialization4_2::onEncodeDiffEntry( - GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const +ProtocolSerialization4_2::onEncodeDiffEntry(GBBuf& buf, const api::GetBucketDiffCommand::Entry& entry) const { buf.putLong(entry._timestamp); SH::putGlobalId(entry._gid, buf); @@ -559,8 +519,7 @@ ProtocolSerialization4_2::onEncodeDiffEntry( } void -ProtocolSerialization4_2::onDecodeDiffEntry( - BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const +ProtocolSerialization4_2::onDecodeDiffEntry(BBuf& buf, api::GetBucketDiffCommand::Entry& entry) const { entry._timestamp = SH::getLong(buf); entry._gid = SH::getGlobalId(buf); diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp index b0a1685ed8c..0b1f66127ba 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp @@ -62,32 +62,26 @@ ProtocolSerialization5_1::onDecodeSetBucketStateCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); api::SetBucketStateCommand::BUCKET_STATE state( - static_cast<api::SetBucketStateCommand::BUCKET_STATE>( - SH::getByte(buf))); - api::SetBucketStateCommand::UP msg( - new api::SetBucketStateCommand(bucket, state)); + static_cast<api::SetBucketStateCommand::BUCKET_STATE>(SH::getByte(buf))); + auto msg = std::make_unique<api::SetBucketStateCommand>(bucket, state); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::SetBucketStateReply& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::SetBucketStateReply& msg) const { onEncodeBucketReply(buf, msg); } api::StorageReply::UP -ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, - BBuf& buf) const +ProtocolSerialization5_1::onDecodeSetBucketStateReply(const SCmd& cmd, BBuf& buf) const { - api::SetBucketStateReply::UP msg(new api::SetBucketStateReply( - static_cast<const api::SetBucketStateCommand&>(cmd))); + auto msg = std::make_unique<api::SetBucketStateReply>(static_cast<const api::SetBucketStateCommand&>(cmd)); onDecodeBucketReply(buf, *msg); - return api::StorageReply::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::GetCommand& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::GetCommand& msg) const { buf.putString(msg.getDocumentId().toString()); putBucket(msg.getBucket(), buf); @@ -103,15 +97,13 @@ ProtocolSerialization5_1::onDecodeGetCommand(BBuf& buf) const document::Bucket bucket = getBucket(buf); api::Timestamp beforeTimestamp(SH::getLong(buf)); std::string fieldSet(SH::getString(buf)); - api::GetCommand::UP msg( - new api::GetCommand(bucket, did, fieldSet, beforeTimestamp)); + auto msg = std::make_unique<api::GetCommand>(bucket, did, fieldSet, beforeTimestamp); onDecodeCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } void -ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::CreateVisitorCommand& msg) const +ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const { putBucketSpace(msg.getBucketSpace(), buf); buf.putString(msg.getLibraryName()); @@ -133,11 +125,7 @@ ProtocolSerialization5_1::onEncode( buf.putString(msg.getFieldSet()); buf.putBoolean(msg.visitInconsistentBuckets()); buf.putInt(vespalib::count_ms(msg.getQueueTimeout())); - - uint32_t size = msg.getParameters().getSerializedSize(); - char* docBuffer = buf.allocate(size); - document::ByteBuffer bbuf(docBuffer, size); - msg.getParameters().serialize(bbuf); + msg.getParameters().serialize(buf); onEncodeCommand(buf, msg); @@ -152,8 +140,7 @@ ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const vespalib::stringref libraryName = SH::getString(buf); vespalib::stringref instanceId = SH::getString(buf); vespalib::stringref selection = SH::getString(buf); - api::CreateVisitorCommand::UP msg( - new api::CreateVisitorCommand(bucketSpace, libraryName, instanceId, selection)); + auto msg = std::make_unique<api::CreateVisitorCommand>(bucketSpace, libraryName, instanceId, selection); msg->setVisitorCmdId(SH::getInt(buf)); msg->setControlDestination(SH::getString(buf)); msg->setDataDestination(SH::getString(buf)); @@ -182,17 +169,16 @@ ProtocolSerialization5_1::onDecodeCreateVisitorCommand(BBuf& buf) const msg->setVisitInconsistentBuckets(); } msg->setQueueTimeout(std::chrono::milliseconds(SH::getInt(buf))); - msg->getParameters().deserialize(getTypeRepo(), buf); + msg->getParameters().deserialize(buf); onDecodeCommand(buf, *msg); SH::getInt(buf); // Unused msg->setMaxBucketsPerVisitor(SH::getInt(buf)); msg->setVisitorDispatcherVersion(50); - return api::StorageCommand::UP(msg.release()); + return msg; } -void ProtocolSerialization5_1::onEncode( - GBBuf& buf, const api::CreateBucketCommand& msg) const +void ProtocolSerialization5_1::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { putBucket(msg.getBucket(), buf); buf.putBoolean(msg.getActive()); @@ -204,10 +190,10 @@ ProtocolSerialization5_1::onDecodeCreateBucketCommand(BBuf& buf) const { document::Bucket bucket = getBucket(buf); bool setActive = SH::getBoolean(buf); - api::CreateBucketCommand::UP msg(new api::CreateBucketCommand(bucket)); + auto msg = std::make_unique<api::CreateBucketCommand>(bucket); msg->setActive(setActive); onDecodeBucketInfoCommand(buf, *msg); - return api::StorageCommand::UP(msg.release()); + return msg; } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index ea002ab98ed..9751fd1be98 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -85,7 +85,7 @@ std::shared_ptr<document::Document> get_document(const protobuf::Document& src_d const document::DocumentTypeRepo& type_repo) { if (!src_doc.payload().empty()) { - document::ByteBuffer doc_buf(src_doc.payload().data(), src_doc.payload().size()); + vespalib::nbostream doc_buf(src_doc.payload().data(), src_doc.payload().size()); return std::make_shared<document::Document>(type_repo, doc_buf); } return std::shared_ptr<document::Document>(); diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h index 08cec601cce..24cc3e4371d 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -87,9 +87,9 @@ public: if (size == 0) { return document::Document::UP(); } else { - document::ByteBuffer bbuf(buf.getBufferAtPos(), size); + vespalib::nbostream stream(buf.getBufferAtPos(), size); buf.incPos(size); - return document::Document::UP(new document::Document(repo, bbuf)); + return std::make_unique<document::Document>(repo, stream); } } diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp index 469c6a41bc7..596af6a79d4 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp @@ -3,6 +3,8 @@ #include "storagereply.h" #include "storagecommand.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/objects/nbostream.h> + using vespalib::alloc::Alloc; using vespalib::IllegalStateException; @@ -17,8 +19,8 @@ StorageReply::StorageReply(mbus::BlobRef data, const ProtocolSerialization& seri _reply() { memcpy(_buffer.get(), data.data(), _sz); - document::ByteBuffer buf(data.data(), _sz); - buf.getIntNetwork(reinterpret_cast<int32_t&>(_mbusType)); + vespalib::nbostream nbo(data.data(), _sz); + nbo >> _mbusType; } StorageReply::StorageReply(api::StorageReply::SP reply) diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storageapi/src/vespa/storageapi/message/bucket.cpp index ee8aff31914..de47a52ca9d 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storageapi/src/vespa/storageapi/message/bucket.cpp @@ -3,6 +3,7 @@ #include "bucket.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/array.hpp> #include <ostream> #include <iterator> @@ -319,7 +320,7 @@ ApplyBucketDiffCommand::Entry::print( << "), headerBlob(" << _headerBlob.size() << "), bodyBlob(" << _bodyBlob.size() << ")"; if (_headerBlob.size() > 0) { - document::ByteBuffer buf(&_headerBlob[0], + vespalib::nbostream buf(&_headerBlob[0], _headerBlob.size()); if (_repo) { document::Document doc(*_repo, buf); diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storageapi/src/vespa/storageapi/message/bucket.h index f6185d9f8e7..45af2296e8a 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storageapi/src/vespa/storageapi/message/bucket.h @@ -14,12 +14,12 @@ #include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/document/base/globalid.h> #include <vespa/vdslib/state/clusterstate.h> +#include <vespa/vespalib/util/array.h> #include <vespa/storageapi/defs.h> namespace document { class DocumentTypeRepo; } -namespace storage { -namespace api { +namespace storage::api { /** * @class CreateBucketCommand @@ -254,6 +254,8 @@ public: GetBucketDiffCommand::Entry _entry; vespalib::string _docName; std::vector<char> _headerBlob; + // TODO: In theory the body blob could be removed now as all is in one blob + // That will enable simplification of code in document. std::vector<char> _bodyBlob; const document::DocumentTypeRepo *_repo; @@ -282,7 +284,7 @@ public: ApplyBucketDiffCommand(const document::Bucket &bucket, const std::vector<Node>& nodes, uint32_t maxBufferSize); - ~ApplyBucketDiffCommand(); + ~ApplyBucketDiffCommand() override; const std::vector<Node>& getNodes() const { return _nodes; } const std::vector<Entry>& getDiff() const { return _diff; } @@ -482,5 +484,4 @@ public: DECLARE_STORAGEREPLY(SetBucketStateReply, onSetBucketStateReply) }; -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storageapi/src/vespa/storageapi/message/persistence.cpp index 1463f42abeb..7fd789a8c81 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.cpp +++ b/storageapi/src/vespa/storageapi/message/persistence.cpp @@ -60,7 +60,7 @@ PutCommand::print(std::ostream& out, bool verbose, const std::string& indent) co { out << "Put(" << getBucketId() << ", " << _doc->getId() << ", timestamp " << _timestamp << ", size " - << _doc->serialize()->getLength() << ")"; + << _doc->serialize().size() << ")"; if (verbose) { out << " {\n" << indent << " "; _doc->print(out, verbose, indent + " "); diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storageapi/src/vespa/storageapi/message/visitor.cpp index aeb58f30fb4..e531e73fea5 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storageapi/src/vespa/storageapi/message/visitor.cpp @@ -65,7 +65,7 @@ CreateVisitorCommand::CreateVisitorCommand(const CreateVisitorCommand& o) { } -CreateVisitorCommand::~CreateVisitorCommand() {} +CreateVisitorCommand::~CreateVisitorCommand() = default; document::Bucket CreateVisitorCommand::getBucket() const @@ -141,8 +141,7 @@ DestroyVisitorCommand::DestroyVisitorCommand(vespalib::stringref instanceId) } void -DestroyVisitorCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const +DestroyVisitorCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "DestroyVisitorCommand(" << _instanceId << ")"; if (verbose) { @@ -157,8 +156,7 @@ DestroyVisitorReply::DestroyVisitorReply(const DestroyVisitorCommand& cmd) } void -DestroyVisitorReply::print(std::ostream& out, bool verbose, - const std::string& indent) const +DestroyVisitorReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "DestroyVisitorReply()"; if (verbose) { @@ -175,12 +173,10 @@ VisitorInfoCommand::VisitorInfoCommand() { } -VisitorInfoCommand::~VisitorInfoCommand() { -} +VisitorInfoCommand::~VisitorInfoCommand() = default; void -VisitorInfoCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const +VisitorInfoCommand::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "VisitorInfoCommand("; if (_completed) { out << "completed"; } @@ -205,8 +201,7 @@ VisitorInfoReply::VisitorInfoReply(const VisitorInfoCommand& cmd) } void -VisitorInfoReply::print(std::ostream& out, bool verbose, - const std::string& indent) const +VisitorInfoReply::print(std::ostream& out, bool verbose, const std::string& indent) const { out << "VisitorInfoReply("; if (_completed) { out << "completed"; } diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp index 68fbca75393..1868e53e4a5 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.cpp @@ -3,8 +3,7 @@ #include "returncode.h" #include <ostream> -namespace storage { -namespace api { +namespace storage::api { ReturnCode::ReturnCode() : _result(OK), @@ -14,58 +13,32 @@ ReturnCode::ReturnCode() ReturnCode::ReturnCode(const ReturnCode &) = default; ReturnCode & ReturnCode::operator = (const ReturnCode &) = default; ReturnCode & ReturnCode::operator = (ReturnCode &&) = default; -ReturnCode::~ReturnCode() {} +ReturnCode::~ReturnCode() = default; ReturnCode::ReturnCode(Result result, vespalib::stringref msg) : _result(result), _message(msg) {} -ReturnCode::ReturnCode(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buffer) - : _result(OK), - _message() -{ - deserialize(repo, buffer); -} - -void ReturnCode:: -onDeserialize(const document::DocumentTypeRepo &, document::ByteBuffer& buffer) -{ - int32_t result; - buffer.getInt(result); - _result = static_cast<Result>(result); - int32_t size; - buffer.getInt(size); - const char * p = buffer.getBufferAtPos(); - buffer.incPos(size); - _message.assign(p, size); -} - -void ReturnCode::onSerialize(document::ByteBuffer& buffer) const -{ - buffer.putInt(_result); - buffer.putInt(_message.size()); - buffer.putBytes(_message.c_str(), _message.size()); -} - -size_t ReturnCode::getSerializedSize() const -{ - return 2 * sizeof(int32_t) + _message.size(); +vespalib::string ReturnCode::getResultString(Result result) { + return documentapi::DocumentProtocol::getErrorName(result); } -void -ReturnCode::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "ReturnCode(" << ReturnCode::getResultString(getResult()); - if (getMessage().size() > 0) out << ", " << getMessage(); - out << ")"; +vespalib::string +ReturnCode::toString() const { + vespalib::string ret = "ReturnCode("; + ret += getResultString(_result); + if ( ! _message.empty()) { + ret += ", "; + ret += _message; + } + ret += ")"; + return ret; } -vespalib::string ReturnCode::getResultString(Result result) { - return documentapi::DocumentProtocol::getErrorName(result); +std::ostream & +operator << (std::ostream & os, const ReturnCode & returnCode) { + return os << returnCode.toString(); } bool @@ -173,5 +146,4 @@ ReturnCode::isBucketDisappearance() const } } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storageapi/src/vespa/storageapi/messageapi/returncode.h index ccd95a81aa3..305a998918c 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.h +++ b/storageapi/src/vespa/storageapi/messageapi/returncode.h @@ -10,20 +10,12 @@ #pragma once -#include <vespa/vespalib/util/printable.h> -#include <vespa/document/util/serializable.h> #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <iosfwd> -namespace document { - class ByteBuffer; -} -namespace storage { -namespace api { +namespace storage::api { -class ReturnCode : public document::Deserializable, - public vespalib::Printable { +class ReturnCode { public: typedef documentapi::DocumentProtocol Protocol; @@ -68,31 +60,20 @@ public: private: Result _result; vespalib::string _message; - void onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) override; - void onSerialize(document::ByteBuffer& buffer) const override; - public: ReturnCode(); explicit ReturnCode(Result result, vespalib::stringref msg = ""); - ReturnCode(const document::DocumentTypeRepo &repo, - document::ByteBuffer& buffer); ReturnCode(const ReturnCode &); ReturnCode & operator = (const ReturnCode &); ReturnCode(ReturnCode &&) = default; ReturnCode & operator = (ReturnCode &&); ~ReturnCode(); - ReturnCode* clone() const override { return new ReturnCode(*this); } - - size_t getSerializedSize() const override; - const vespalib::string& getMessage() const { return _message; } void setMessage(vespalib::stringref message) { _message = message; } Result getResult() const { return _result; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - /** * Translate from status code to human-readable string * @param result Status code returned from getResult() @@ -121,8 +102,9 @@ public: bool isShutdownRelated() const; bool isBucketDisappearance() const; bool isNonCriticalForIntegrityChecker() const; + vespalib::string toString() const; }; +std::ostream & operator << (std::ostream & os, const ReturnCode & returnCode); -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp index 86f7cb6e16d..81cdadb3623 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp @@ -4,8 +4,7 @@ #include "storagecommand.h" #include <ostream> -namespace storage { -namespace api { +namespace storage::api { StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) : StorageMessage(cmd.getType().getReplyType(), cmd.getMsgId()), @@ -19,16 +18,14 @@ StorageReply::StorageReply(const StorageCommand& cmd, ReturnCode code) setTransportContext(cmd.getTransportContext()); } -StorageReply::~StorageReply() { } +StorageReply::~StorageReply() = default; void StorageReply::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; (void) indent; - out << "StorageReply(" << _type.getName() << ", " - << _result.toString() << ")"; + out << "StorageReply(" << _type.getName() << ", " << _result << ")"; } -} // api -} // storage +} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storageapi/src/vespa/storageapi/messageapi/storagereply.h index 4219f3e28cd..1a3bbe35eb4 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagereply.h @@ -15,8 +15,7 @@ #include "returncode.h" #include "storagemessage.h" -namespace storage { -namespace api { +namespace storage::api { class StorageCommand; @@ -28,7 +27,7 @@ protected: ReturnCode code = ReturnCode(ReturnCode::OK)); public: - ~StorageReply(); + ~StorageReply() override; DECLARE_POINTER_TYPEDEFS(StorageReply); void setResult(const ReturnCode& r) { _result = r; } @@ -37,6 +36,4 @@ public: void print(std::ostream& out, bool verbose, const std::string& indent) const override; }; -} // api -} // storage - +} diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 94026ffaa19..eb1cc7f0256 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -1108,7 +1108,7 @@ SearchVisitor::generateGroupingResults() vespalib::NBOSerializer nos(os); grouping.serialize(nos); vespalib::MallocPtr blob(os.size()); - memcpy(blob, os.c_str(), os.size()); + memcpy(blob, os.data(), os.size()); searchResult.getGroupingList().add(grouping.getId(), blob); } } diff --git a/vdslib/src/tests/container/parameterstest.cpp b/vdslib/src/tests/container/parameterstest.cpp index c54d8ae66da..95a29fb97be 100644 --- a/vdslib/src/tests/container/parameterstest.cpp +++ b/vdslib/src/tests/container/parameterstest.cpp @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/repo/documenttyperepo.h> #include <vespa/vdslib/container/parameters.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/gtest/gtest.h> -using document::DocumentTypeRepo; +using vespalib::GrowableByteBuffer; +using document::ByteBuffer; using namespace vdslib; TEST(ParametersTest, test_parameters) @@ -15,11 +17,12 @@ TEST(ParametersTest, test_parameters) par.set("number", 6); par.set("int64_t", INT64_C(8589934590)); par.set("double", 0.25); - std::unique_ptr<document::ByteBuffer> buffer(par.serialize()); - buffer->flip(); - DocumentTypeRepo repo; - Parameters par2(repo, *buffer); + GrowableByteBuffer buffer; + par.serialize(buffer); + + ByteBuffer bBuf(buffer.getBuffer(), buffer.position()); + Parameters par2(bBuf); EXPECT_EQ(vespalib::stringref("overture"), par2.get("fast")); EXPECT_EQ(vespalib::stringref("yahoo"), par2.get("overture")); @@ -35,4 +38,5 @@ TEST(ParametersTest, test_parameters) EXPECT_EQ(numberDefault, par2.get("nonexistingnumber", numberDefault)); EXPECT_EQ(int64Default, par2.get("nonexistingint64_t", int64Default)); EXPECT_EQ(doubleDefault, par2.get("nonexistingdouble", doubleDefault)); + } diff --git a/vdslib/src/vespa/vdslib/container/documentsummary.cpp b/vdslib/src/vespa/vdslib/container/documentsummary.cpp index bc8a0473ab3..f948bd64687 100644 --- a/vdslib/src/vespa/vdslib/container/documentsummary.cpp +++ b/vdslib/src/vespa/vdslib/container/documentsummary.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentsummary.h" +#include <vespa/vespalib/util/growablebytebuffer.h> +#include <vespa/document/util/bytebuffer.h> #include <algorithm> namespace vdslib { @@ -21,7 +23,7 @@ DocumentSummary::DocumentSummary(document::ByteBuffer& buf) : deserialize(buf); } -DocumentSummary::~DocumentSummary() {} +DocumentSummary::~DocumentSummary() = default; void DocumentSummary::deserialize(document::ByteBuffer& buf) { @@ -47,18 +49,18 @@ void DocumentSummary::deserialize(document::ByteBuffer& buf) } } -void DocumentSummary::serialize(document::ByteBuffer& buf) const +void DocumentSummary::serialize(vespalib::GrowableByteBuffer& buf) const { - buf.putIntNetwork(0); // Just serialize dummy 4 byte field, to avoid versioning. - buf.putIntNetwork(_summary.size()); + buf.putInt(0); // Just serialize dummy 4 byte field, to avoid versioning. + buf.putInt(_summary.size()); if ( ! _summary.empty() ) { - buf.putIntNetwork(getSummarySize()); + buf.putInt(getSummarySize()); for (size_t i(0), m(_summary.size()); i < m; i++) { Summary s(_summary[i]); buf.putBytes(s.getDocId(_summaryBuffer->c_str()), s.getTotalSize()); } for (size_t i(0), m(_summary.size()); i < m; i++) { - buf.putIntNetwork(_summary[i].getSummarySize()); + buf.putInt(_summary[i].getSummarySize()); } } } diff --git a/vdslib/src/vespa/vdslib/container/documentsummary.h b/vdslib/src/vespa/vdslib/container/documentsummary.h index f04c1fa06bf..375546920ec 100644 --- a/vdslib/src/vespa/vdslib/container/documentsummary.h +++ b/vdslib/src/vespa/vdslib/container/documentsummary.h @@ -2,9 +2,10 @@ #pragma once #include <vespa/vespalib/util/memory.h> -#include <vespa/document/util/bytebuffer.h> #include <vector> +namespace document { class ByteBuffer; } +namespace vespalib { class GrowableByteBuffer; } namespace vdslib { class DocumentSummary { @@ -29,7 +30,7 @@ public: void sort(); void deserialize(document::ByteBuffer& buf); - void serialize(document::ByteBuffer& buf) const; + void serialize(vespalib::GrowableByteBuffer& buf) const; uint32_t getSerializedSize() const; private: class Summary { diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index 4358c42ff29..9c843df1caa 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -1,20 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "parameters.hpp" +#include <vespa/document/util/bytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/xmlstream.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <ostream> using namespace vdslib; Parameters::Parameters() = default; -Parameters::Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) +Parameters::Parameters(document::ByteBuffer& buffer) : _parameters() { - deserialize(repo, buffer); + deserialize(buffer); } Parameters::~Parameters() = default; @@ -28,20 +30,19 @@ size_t Parameters::getSerializedSize() const return mysize; } -void Parameters::onSerialize(document::ByteBuffer& buffer) const +void Parameters::serialize(vespalib::GrowableByteBuffer& buffer) const { - buffer.putIntNetwork(_parameters.size()); + buffer.putInt(_parameters.size()); for (const auto & entry : _parameters) { - buffer.putIntNetwork(entry.first.size()); + buffer.putInt(entry.first.size()); buffer.putBytes(entry.first.c_str(), entry.first.size()); - buffer.putIntNetwork(entry.second.size()); + buffer.putInt(entry.second.size()); buffer.putBytes(entry.second.c_str(), entry.second.size()); } } -void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) +void Parameters::deserialize(document::ByteBuffer& buffer) { - (void) repo; _parameters.clear(); int32_t mysize; buffer.getIntNetwork(mysize); @@ -88,11 +89,6 @@ Parameters::operator==(const Parameters &other) const return true; } -Parameters* Parameters::clone() const -{ - return new Parameters(*this); -} - vespalib::stringref Parameters::get(vespalib::stringref id, vespalib::stringref def) const { ParametersMap::const_iterator it = _parameters.find(id); @@ -130,7 +126,7 @@ void Parameters::print(std::ostream& out, bool verbose, const std::string& inden out << ")"; } -std::string Parameters::toString() const +vespalib::string Parameters::toString() const { vespalib::string ret; for (const auto & entry : _parameters) { diff --git a/vdslib/src/vespa/vdslib/container/parameters.h b/vdslib/src/vespa/vdslib/container/parameters.h index f3ea0543546..61649b29bbe 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.h +++ b/vdslib/src/vespa/vdslib/container/parameters.h @@ -14,18 +14,15 @@ #pragma once -#include <vespa/document/util/serializable.h> #include <vespa/document/util/xmlserializable.h> #include <vespa/vespalib/stllike/hash_map.h> -namespace vespalib { - class asciistream; -} +namespace vespalib { class GrowableByteBuffer; } +namespace document { class ByteBuffer; } namespace vdslib { -class Parameters : public document::Deserializable, - public document::XmlSerializable { +class Parameters : public document::XmlSerializable { public: typedef vespalib::stringref KeyT; class Value : public vespalib::string @@ -42,27 +39,25 @@ public: private: ParametersMap _parameters; - void onSerialize(document::ByteBuffer& buffer) const override; - void onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) override; void printXml(document::XmlOutputStream& xos) const override; public: Parameters(); - Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer); - virtual ~Parameters(); + Parameters(document::ByteBuffer& buffer); + ~Parameters(); bool operator==(const Parameters &other) const; - Parameters* clone() const override; + size_t getSerializedSize() const; - size_t getSerializedSize() const override; - - bool hasValue(KeyT id) const { return (_parameters.find(id) != _parameters.end()); } - unsigned int size() const { return _parameters.size(); } + bool hasValue(KeyT id) const { return (_parameters.find(id) != _parameters.end()); } + unsigned int size() const { return _parameters.size(); } bool lookup(KeyT id, ValueRef & v) const; void set(KeyT id, const void * v, size_t sz) { _parameters[id] = Value(v, sz); } void print(std::ostream& out, bool verbose, const std::string& indent) const; + void serialize(vespalib::GrowableByteBuffer& buffer) const; + void deserialize(document::ByteBuffer& buffer); // Disallow ParametersMap::const_iterator begin() const { return _parameters.begin(); } @@ -92,7 +87,7 @@ public: template<typename T> T get(KeyT id, T def) const; - std::string toString() const; + vespalib::string toString() const; }; } // vdslib diff --git a/vdslib/src/vespa/vdslib/container/searchresult.cpp b/vdslib/src/vespa/vdslib/container/searchresult.cpp index 73b19a2f8a2..20cc53e2de9 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.cpp +++ b/vdslib/src/vespa/vdslib/container/searchresult.cpp @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "searchresult.h" +#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/growablebytebuffer.h> #include <algorithm> namespace vdslib { @@ -25,21 +27,21 @@ void AggregatorList::deserialize(document::ByteBuffer & buf) } } -void AggregatorList::serialize(document::ByteBuffer & buf) const +void AggregatorList::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(size()); - for (const_iterator it(begin()), mt(end()); it != mt; it++) { - buf.putIntNetwork(it->first); - buf.putIntNetwork(it->second.size()); - buf.putBytes(it->second, it->second.size()); + buf.putInt(size()); + for (const auto & entry : *this) { + buf.putInt(entry.first); + buf.putInt(entry.second.size()); + buf.putBytes(entry.second, entry.second.size()); } } uint32_t AggregatorList::getSerializedSize() const { size_t sz(sizeof(uint32_t) * (1 + 2*size())); - for (const_iterator it(begin()), mt(end()); it != mt; it++) { - sz += it->second.size(); + for (const auto & entry : *this) { + sz += entry.second.size(); } return sz; } @@ -51,7 +53,7 @@ BlobContainer::BlobContainer(size_t reserve) : _offsets.push_back(0); } -BlobContainer::~BlobContainer() {} +BlobContainer::~BlobContainer() = default; size_t BlobContainer::append(const void * v, size_t sz) { @@ -84,11 +86,11 @@ void BlobContainer::deserialize(document::ByteBuffer & buf) buf.getBytes(_blob, getSize()); } -void BlobContainer::serialize(document::ByteBuffer & buf) const +void BlobContainer::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(getCount()); + buf.putInt(getCount()); for(size_t i(0), m(getCount()); i < m; i++) { - buf.putIntNetwork(getSize(i)); + buf.putInt(getSize(i)); } buf.putBytes(_blob, getSize()); } @@ -116,7 +118,7 @@ SearchResult::SearchResult(document::ByteBuffer & buf) : deserialize(buf); } -SearchResult::~SearchResult() {} +SearchResult::~SearchResult() = default; void SearchResult::deserialize(document::ByteBuffer & buf) { @@ -143,26 +145,26 @@ void SearchResult::deserialize(document::ByteBuffer & buf) _groupingList.deserialize(buf); } -void SearchResult::serialize(document::ByteBuffer & buf) const +void SearchResult::serialize(vespalib::GrowableByteBuffer & buf) const { - buf.putIntNetwork(_totalHits); + buf.putInt(_totalHits); uint32_t hitCount = std::min(_hits.size(), _wantedHits); - buf.putIntNetwork(hitCount); + buf.putInt(hitCount); if (hitCount > 0) { uint32_t sz = getBufCount(); - buf.putIntNetwork(sz); + buf.putInt(sz); for (size_t i(0), m(hitCount); i < m; i++) { const char * s(_hits[i].getDocId(_docIdBuffer->c_str())); buf.putBytes(s, strlen(s)+1); } for (size_t i(0), m(hitCount); i < m; i++) { - buf.putDoubleNetwork(_hits[i].getRank()); + buf.putDouble(_hits[i].getRank()); } } uint32_t sortCount = std::min(_sortBlob.getCount(), _wantedHits); - buf.putIntNetwork(sortCount); + buf.putInt(sortCount); for (size_t i(0); i < sortCount; i++) { - buf.putIntNetwork(_sortBlob.getSize(_hits[i].getIndex())); + buf.putInt(_sortBlob.getSize(_hits[i].getIndex())); } for (size_t i(0); i < sortCount; i++) { size_t sz; diff --git a/vdslib/src/vespa/vdslib/container/searchresult.h b/vdslib/src/vespa/vdslib/container/searchresult.h index 081873e2989..fc893f6b5be 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.h +++ b/vdslib/src/vespa/vdslib/container/searchresult.h @@ -2,10 +2,11 @@ #pragma once #include <vespa/vespalib/util/memory.h> -#include <vespa/document/util/bytebuffer.h> #include <vector> #include <map> +namespace document { class ByteBuffer; } +namespace vespalib { class GrowableByteBuffer; } namespace vdslib { typedef std::map<size_t, vespalib::MallocPtr> IntBlobMapT; @@ -15,7 +16,7 @@ class AggregatorList : public IntBlobMapT public: void add(size_t id, const vespalib::MallocPtr & aggrBlob); void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const; }; @@ -31,7 +32,7 @@ public: size_t getSize(size_t index) const { return _offsets[index+1] - _offsets[index]; } const void * getBuf(size_t index) const { return _blob.c_str() + _offsets[index]; } void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const { return (1 + getCount()) * sizeof(uint32_t) + getSize(); } private: typedef vespalib::MallocPtr Blob; @@ -76,7 +77,7 @@ public: void sort(); void deserialize(document::ByteBuffer & buf); - void serialize(document::ByteBuffer & buf) const; + void serialize(vespalib::GrowableByteBuffer & buf) const; uint32_t getSerializedSize() const; private: class Hit { diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index 3109295fbdc..daaea981b5a 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -141,7 +141,7 @@ public: size_t size() const { return left(); } size_t capacity() const { return _wbuf.size(); } bool empty() const { return size() == 0; } - const char * c_str() const { return &_rbuf[0]; } + const char * data() const { return &_rbuf[0]; } const char * peek() const { return &_rbuf[_rp]; } size_t rp() const { return _rp; } nbostream & rp(size_t pos) { if (pos > _wp) fail(eof); _rp = pos; return *this; } diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index b52cace45a5..03ebc2807f9 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -59,13 +59,13 @@ public: bool resize_inplace(size_t newSize); Alloc(const Alloc &) = delete; Alloc & operator = (const Alloc &) = delete; - Alloc(Alloc && rhs) : + Alloc(Alloc && rhs) noexcept : _alloc(rhs._alloc), _allocator(rhs._allocator) { rhs.clear(); } - Alloc & operator=(Alloc && rhs) { + Alloc & operator=(Alloc && rhs) noexcept { if (this != & rhs) { if (_alloc.first != nullptr) { _allocator->free(_alloc); |