diff options
Diffstat (limited to 'vespalib')
9 files changed, 232 insertions, 151 deletions
diff --git a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp index 9089ebc3e3a..38c843e170b 100644 --- a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp +++ b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp @@ -2,8 +2,9 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <iostream> +#include <ostream> using vespalib::nbostream; using ExpBuffer = std::vector<uint8_t>; diff --git a/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index d1d713c2987..81caf706304 100644 --- a/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -8,7 +8,8 @@ #include <vespa/vespalib/tensor/tensor_factory.h> #include <vespa/vespalib/tensor/serialization/typed_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> -#include <iostream> +#include <vespa/vespalib/objects/hexdump.h> +#include <ostream> using namespace vespalib::tensor; using vespalib::nbostream; diff --git a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt index 7473092ad8b..f3b6a8e4c0a 100644 --- a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt @@ -2,5 +2,6 @@ vespa_add_library(vespalib_vespalib_objects OBJECT SOURCES nbostream.cpp + hexdump.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/objects/hexdump.cpp b/vespalib/src/vespa/vespalib/objects/hexdump.cpp new file mode 100644 index 00000000000..22ce94298d2 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/hexdump.cpp @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "hexdump.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib { + +namespace { + + const char * hexChar = "0123456789ABCDEF"; + +} + +string +HexDump::toString() const { + asciistream os; + os << *this; + return os.str(); +} + +asciistream & operator << (asciistream & os, const HexDump & hd) +{ + os << hd._sz << ' '; + const uint8_t *c = static_cast<const uint8_t *>(hd._buf); + for (size_t i(0); i < hd._sz; i++) { + os << hexChar[c[i] >> 4] << hexChar[c[i] & 0xf]; + } + return os; +} + +std::ostream & operator << (std::ostream & os, const HexDump & hd) +{ + return os << hd.toString(); +} + +} diff --git a/vespalib/src/vespa/vespalib/objects/hexdump.h b/vespalib/src/vespa/vespalib/objects/hexdump.h new file mode 100644 index 00000000000..c2e5cad78fa --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/hexdump.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +class asciistream; + +/* + * Helper class to provide hex dump of the contents in a buffer. + */ +class HexDump +{ +public: + HexDump(const void * buf, size_t sz) : _buf(buf), _sz(sz) { } + vespalib::string toString() const; + friend std::ostream & operator << (std::ostream & os, const HexDump & hd); + friend asciistream & operator << (asciistream & os, const HexDump & hd); +private: + const void * _buf; + size_t _sz; +}; + +} diff --git a/vespalib/src/vespa/vespalib/objects/nbo.h b/vespalib/src/vespa/vespalib/objects/nbo.h new file mode 100644 index 00000000000..5deb4ea2899 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/nbo.h @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <arpa/inet.h> + +namespace vespalib { +class nbo { +public: + static bool n2h(bool v) { return v; } + static int8_t n2h(int8_t v) { return v; } + static uint8_t n2h(uint8_t v) { return v; } + static char n2h(char v) { return v; } + static int16_t n2h(int16_t v) { return ntohs(v); } + static uint16_t n2h(uint16_t v) { return ntohs(v); } + static int32_t n2h(int32_t v) { return ntohl(v); } + static uint32_t n2h(uint32_t v) { return ntohl(v); } + static int64_t n2h(int64_t v) { return ntohll(v); } + static uint64_t n2h(uint64_t v) { return ntohll(v); } + static float n2h(float v) { + union { uint32_t _u; float _f; } uf; + uf._f = v; + uf._u = ntohl(uf._u); + return uf._f; + } + static double n2h(double v) { + union { uint64_t _u; double _f; } uf; + uf._f = v; + uf._u = ntohll(uf._u); + return uf._f; + } +private: + static uint64_t ntohll(uint64_t v) { + union { uint64_t _ll; uint32_t _l[2]; } w, r; + r._ll = v; + w._l[0] = n2h(r._l[1]); + w._l[1] = n2h(r._l[0]); + return w._ll; + } +}; + +} diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp index e1ec7803aa2..de01b8b60df 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp +++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp @@ -1,48 +1,77 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "nbostream.h" +#include "nbostream.hpp" +#include "hexdump.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/optimized.h> namespace vespalib { -namespace { - - const char * hexChar = "0123456789ABCDEF"; - +nbostream::nbostream(size_t initialSize) : + _wbuf(), + _rbuf(), + _rp(0), + _wp(0), + _state(ok), + _longLivedBuffer(false) +{ + extend(initialSize); } -nbostream::~nbostream() { } - -string -HexDump::toString() const { - asciistream os; - os << *this; - return os.str(); +nbostream::nbostream(const void * buf, size_t sz, bool longLivedBuffer) : + _wbuf(), + _rbuf(buf, sz), + _rp(0), + _wp(sz), + _state(ok), + _longLivedBuffer(longLivedBuffer) +{ } + +nbostream::nbostream(Alloc && buf, size_t sz) : + _wbuf(std::move(buf), sz), + _rbuf(&_wbuf[0], sz), + _rp(0), + _wp(sz), + _state(ok), + _longLivedBuffer(false) +{ + assert(_wbuf.size() >= sz); } -asciistream & operator << (asciistream & os, const HexDump & hd) +nbostream::nbostream(const nbostream & rhs) : + _wbuf(), + _rbuf(), + _rp(0), + _wp(0), + _state(ok), + _longLivedBuffer(false) { - os << hd._sz << ' '; - const uint8_t *c = static_cast<const uint8_t *>(hd._buf); - for (size_t i(0); i < hd._sz; i++) { - os << hexChar[c[i] >> 4] << hexChar[c[i] & 0xf]; - } - return os; + extend(rhs.size()); + _wp = rhs.size(); + memcpy(&_wbuf[0], &rhs._rbuf[rhs._rp], _wp); } -std::ostream & operator << (std::ostream & os, const HexDump & hd) -{ - return os << hd.toString(); +nbostream & +nbostream::operator = (const nbostream & rhs) { + if (this != &rhs) { + nbostream n(rhs); + swap(n); + } + return *this; } +nbostream::~nbostream() { } + void nbostream::fail(State s) { _state = static_cast<State>(_state | s); throw IllegalStateException(make_string("Stream failed bufsize(%zu), readp(%zu), writep(%zu)", _wbuf.size(), _rp, _wp), VESPA_STRLOC); } +std::ostream & operator << (std::ostream & os, const nbostream & s) { + return os << HexDump(&s._rbuf[s._rp], s.left()); +} + void nbostream::reserve(size_t sz) { if (capacity() < sz) { @@ -95,5 +124,13 @@ void nbostream::swap(nbostream & os) std::swap(_rbuf, os._rbuf); } +template nbostream& nbostream::saveVector<int16_t>(const std::vector<int16_t> &); +template nbostream& nbostream::restoreVector<int16_t>(std::vector<int16_t> &); +template nbostream& nbostream::saveVector<int32_t>(const std::vector<int32_t> &); +template nbostream& nbostream::restoreVector<int32_t>(std::vector<int32_t> &); +template nbostream& nbostream::saveVector<uint32_t>(const std::vector<uint32_t> &); +template nbostream& nbostream::restoreVector<uint32_t>(std::vector<uint32_t> &); +template nbostream& nbostream::saveVector<uint64_t>(const std::vector<uint64_t> &); +template nbostream& nbostream::restoreVector<uint64_t>(std::vector<uint64_t> &); } diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index e632704f1fa..6914b533b63 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -5,28 +5,10 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/buffer.h> -#include <arpa/inet.h> +#include "nbo.h" namespace vespalib { -class asciistream; - -/* - * Helper class to provide hex dump of the contents in a buffer. - */ -class HexDump -{ -public: - HexDump(const void * buf, size_t sz) : _buf(buf), _sz(sz) { } - vespalib::string toString() const; - friend std::ostream & operator << (std::ostream & os, const HexDump & hd); - friend asciistream & operator << (asciistream & os, const HexDump & hd); -private: - const void * _buf; - size_t _sz; -}; - - /** * Class for streaming data in network byte order, used to serialize * and deserialize objects. The java code corresponding to the C++ @@ -39,74 +21,29 @@ class nbostream using Buffer = Array<char>; using Alloc = alloc::Alloc; enum State { ok=0, eof=0x01}; - nbostream(size_t initialSize=1024) : - _wbuf(), - _rbuf(), - _rp(0), - _wp(0), - _state(ok), - _longLivedBuffer(false) - { - extend(initialSize); - } - - nbostream(const void * buf, size_t sz, bool longLivedBuffer=false) : - _wbuf(), - _rbuf(buf, sz), - _rp(0), - _wp(sz), - _state(ok), - _longLivedBuffer(longLivedBuffer) - { - } - - nbostream(Alloc && buf, size_t sz) : - _wbuf(std::move(buf), sz), - _rbuf(&_wbuf[0], sz), - _rp(0), - _wp(sz), - _state(ok), - _longLivedBuffer(false) - { - assert(_wbuf.size() >= sz); - } - - nbostream(const nbostream & rhs) : - _wbuf(), - _rbuf(), - _rp(0), - _wp(0), - _state(ok), - _longLivedBuffer(false) - { - extend(rhs.size()); - _wp = rhs.size(); - memcpy(&_wbuf[0], &rhs._rbuf[rhs._rp], _wp); - } + nbostream(size_t initialSize=1024); + nbostream(const void * buf, size_t sz, bool longLivedBuffer=false); + nbostream(Alloc && buf, size_t sz); + nbostream(const nbostream & rhs); ~nbostream(); - nbostream & operator = (const nbostream & rhs) { - if (this != &rhs) { - nbostream n(rhs); - swap(n); - } - return *this; - } - nbostream & operator << (double v) { double n(n2h(v)); write8(&n); return *this; } - nbostream & operator >> (double & v) { double n; read8(&n); v = n2h(n); return *this; } - nbostream & operator << (float v) { float n(n2h(v)); write4(&n); return *this; } - nbostream & operator >> (float & v) { float n; read4(&n); v = n2h(n); return *this; } - nbostream & operator << (int64_t v) { int64_t n(n2h(v)); write8(&n); return *this; } - nbostream & operator >> (int64_t & v) { int64_t n; read8(&n); v = n2h(n); return *this; } - nbostream & operator << (uint64_t v) { uint64_t n(n2h(v)); write8(&n); return *this; } - nbostream & operator >> (uint64_t & v) { uint64_t n; read8(&n); v = n2h(n); return *this; } - nbostream & operator << (int32_t v) { int32_t n(n2h(v)); write4(&n); return *this; } - nbostream & operator >> (int32_t & v) { int32_t n; read4(&n); v = n2h(n); return *this; } - nbostream & operator << (uint32_t v) { uint32_t n(n2h(v)); write4(&n); return *this; } - nbostream & operator >> (uint32_t & v) { uint32_t n; read4(&n); v = n2h(n); return *this; } - nbostream & operator << (int16_t v) { int16_t n(n2h(v)); write2(&n); return *this; } - nbostream & operator >> (int16_t & v) { int16_t n; read2(&n); v = n2h(n); return *this; } - nbostream & operator << (uint16_t v) { uint16_t n(n2h(v)); write2(&n); return *this; } - nbostream & operator >> (uint16_t & v) { uint16_t n; read2(&n); v = n2h(n); return *this; } + nbostream & operator = (const nbostream & rhs); + + nbostream & operator << (double v) { double n(nbo::n2h(v)); write8(&n); return *this; } + nbostream & operator >> (double & v) { double n; read8(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (float v) { float n(nbo::n2h(v)); write4(&n); return *this; } + nbostream & operator >> (float & v) { float n; read4(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (int64_t v) { int64_t n(nbo::n2h(v)); write8(&n); return *this; } + nbostream & operator >> (int64_t & v) { int64_t n; read8(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (uint64_t v) { uint64_t n(nbo::n2h(v)); write8(&n); return *this; } + nbostream & operator >> (uint64_t & v) { uint64_t n; read8(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (int32_t v) { int32_t n(nbo::n2h(v)); write4(&n); return *this; } + nbostream & operator >> (int32_t & v) { int32_t n; read4(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (uint32_t v) { uint32_t n(nbo::n2h(v)); write4(&n); return *this; } + nbostream & operator >> (uint32_t & v) { uint32_t n; read4(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (int16_t v) { int16_t n(nbo::n2h(v)); write2(&n); return *this; } + nbostream & operator >> (int16_t & v) { int16_t n; read2(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (uint16_t v) { uint16_t n(nbo::n2h(v)); write2(&n); return *this; } + nbostream & operator >> (uint16_t & v) { uint16_t n; read2(&n); v = nbo::n2h(n); return *this; } nbostream & operator << (int8_t v) { write1(&v); return *this; } nbostream & operator >> (int8_t & v) { read1(&v); return *this; } nbostream & operator << (uint8_t v) { write1(&v); return *this; } @@ -198,36 +135,12 @@ class nbostream // For checkpointing where capacity should be restored template <typename T> nbostream & - saveVector(const std::vector<T> &val) - { - size_t valCapacity = val.capacity(); - size_t valSize = val.size(); - assert(valCapacity >= valSize); - *this << valCapacity << valSize; - for (const T & v : val) { - *this << v; - } - return *this; - } + saveVector(const std::vector<T> &val); // For checkpointing where capacity should be restored template <typename T> nbostream & - restoreVector(std::vector<T> &val) - { - size_t valCapacity = 0; - size_t valSize = 0; - *this >> valCapacity >> valSize; - assert(valCapacity >= valSize); - val.reserve(valCapacity); - val.clear(); - T i; - for (size_t j = 0; j < valSize; ++j) { - *this >> i; - val.push_back(i); - } - return *this; - } + restoreVector(std::vector<T> &val); size_t size() const { return left(); } size_t capacity() const { return _wbuf.size(); } @@ -241,19 +154,7 @@ class nbostream bool good() const { return _state == ok; } void clear() { _wbuf.clear(); _wp = _rp = 0; _state = ok; } void adjustReadPos(ssize_t adj) { size_t npos = _rp + adj; if (__builtin_expect(npos > _wp, false)) { fail(eof); } _rp = npos; } - friend std::ostream & operator << (std::ostream & os, const nbostream & s) { return os << HexDump(&s._rbuf[s._rp], s.left()); } - static bool n2h(bool v) { return v; } - static int8_t n2h(int8_t v) { return v; } - static uint8_t n2h(uint8_t v) { return v; } - static char n2h(char v) { return v; } - static int16_t n2h(int16_t v) { return ntohs(v); } - static uint16_t n2h(uint16_t v) { return ntohs(v); } - static int32_t n2h(int32_t v) { return ntohl(v); } - static uint32_t n2h(uint32_t v) { return ntohl(v); } - static int64_t n2h(int64_t v) { return ntohll(v); } - static uint64_t n2h(uint64_t v) { return ntohll(v); } - static float n2h(float v) { union { uint32_t _u; float _f; } uf; uf._f = v; uf._u = ntohl(uf._u); return uf._f; } - static double n2h(double v) { union { uint64_t _u; double _f; } uf; uf._f = v; uf._u = ntohll(uf._u); return uf._f; } + friend std::ostream & operator << (std::ostream & os, const nbostream & s); void write(const void *v, size_t sz) { if (__builtin_expect(space() < sz, false)) { extend(sz); @@ -313,7 +214,6 @@ class nbostream adjustReadPos(strSize); } private: - static uint64_t ntohll(uint64_t v) { union { uint64_t _ll; uint32_t _l[2]; } w, r; r._ll = v; w._l[0] = n2h(r._l[1]); w._l[1] = n2h(r._l[0]); return w._ll; } void read1(void *v) { read(v, 1); } void read2(void *v) { read(v, 2); } void read4(void *v) { read(v, 4); } diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.hpp b/vespalib/src/vespa/vespalib/objects/nbostream.hpp new file mode 100644 index 00000000000..767c485550c --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/nbostream.hpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "nbostream.h" + +namespace vespalib { + +template <typename T> +nbostream & +nbostream::saveVector(const std::vector<T> &val) +{ + size_t valCapacity = val.capacity(); + size_t valSize = val.size(); + assert(valCapacity >= valSize); + *this << valCapacity << valSize; + for (const T & v : val) { + *this << v; + } + return *this; +} + +template <typename T> +nbostream & +nbostream::restoreVector(std::vector<T> &val) +{ + size_t valCapacity = 0; + size_t valSize = 0; + *this >> valCapacity >> valSize; + assert(valCapacity >= valSize); + val.reserve(valCapacity); + val.clear(); + T i; + for (size_t j = 0; j < valSize; ++j) { + *this >> i; + val.push_back(i); + } + return *this; +} + +} |