diff options
49 files changed, 330 insertions, 259 deletions
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 1af20de3f9b..1e009498295 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/base/testdocman.h> @@ -24,6 +23,7 @@ #include <vespa/vespalib/tensor/default_tensor.h> #include <vespa/vespalib/tensor/tensor_factory.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/objects/nbostream.h> using namespace document::config_builder; using vespalib::tensor::Tensor; diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp index fc113c79de8..6288d19f3a1 100644 --- a/document/src/tests/fieldpathupdatetestcase.cpp +++ b/document/src/tests/fieldpathupdatetestcase.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/select/node.h> @@ -12,6 +11,7 @@ #include <vespa/vespalib/objects/identifiable.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/serialization/vespadocumentserializer.h> +#include <vespa/vespalib/objects/nbostream.h> #include <fstream> #include <sstream> diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.h b/document/src/vespa/document/fieldvalue/fieldvalue.h index 865a3cc74d1..a6e4b3e868f 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.h +++ b/document/src/vespa/document/fieldvalue/fieldvalue.h @@ -18,9 +18,12 @@ #include <vespa/document/util/xmlserializable.h> #include <vespa/vespalib/util/polymorphicarrays.h> #include <vespa/document/util/bytebuffer.h> -#include <vespa/vespalib/objects/nbostream.h> #include <map> +namespace vespalib { + class nbostream; +} + namespace document { class FieldValue : public vespalib::Identifiable diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp index b7a209d6f78..c37edbfea5f 100644 --- a/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.cpp @@ -6,6 +6,7 @@ #include <vespa/document/annotation/spantree.h> #include <vespa/document/serialization/annotationserializer.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/hexdump.h> #include <vespa/document/serialization/util.h> #include <vespa/document/serialization/annotationdeserializer.h> #include <vespa/document/repo/fixedtyperepo.h> diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.h b/document/src/vespa/document/fieldvalue/stringfieldvalue.h index 6344c0bd5b9..9ec30128599 100644 --- a/document/src/vespa/document/fieldvalue/stringfieldvalue.h +++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.h @@ -11,6 +11,7 @@ #include <vespa/document/annotation/spantree.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/vespalib/util/buffer.h> namespace document { class FixedTypeRepo; @@ -52,7 +53,7 @@ private: class AnnotationData { public: - typedef vespalib::nbostream::Buffer BackingBlob; + typedef std::vector<char> BackingBlob; typedef std::unique_ptr<AnnotationData> UP; VESPA_DLL_LOCAL AnnotationData(const AnnotationData & rhs); VESPA_DLL_LOCAL AnnotationData(vespalib::ConstBufferRef serialized, const FixedTypeRepo &repo, diff --git a/document/src/vespa/document/serialization/annotationdeserializer.cpp b/document/src/vespa/document/serialization/annotationdeserializer.cpp index 1bc88684799..ead83e4d28c 100644 --- a/document/src/vespa/document/serialization/annotationdeserializer.cpp +++ b/document/src/vespa/document/serialization/annotationdeserializer.cpp @@ -1,11 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".annotationdeserializer"); - #include "annotationdeserializer.h" - #include "vespadocumentdeserializer.h" #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/annotation/annotation.h> @@ -14,6 +9,10 @@ LOG_SETUP(".annotationdeserializer"); #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/vespalib/objects/nbostream.h> + +#include <vespa/log/log.h> +LOG_SETUP(".annotationdeserializer"); using std::unique_ptr; diff --git a/document/src/vespa/document/serialization/annotationdeserializer.h b/document/src/vespa/document/serialization/annotationdeserializer.h index 1dd578c6301..b5a769fb5ae 100644 --- a/document/src/vespa/document/serialization/annotationdeserializer.h +++ b/document/src/vespa/document/serialization/annotationdeserializer.h @@ -2,10 +2,13 @@ #pragma once -#include <vespa/vespalib/objects/nbostream.h> #include <vespa/document/annotation/span.h> #include <vespa/document/serialization/util.h> +namespace vespalib { + class nbostream; +} + namespace document { class AlternateSpanList; class Annotation; @@ -16,8 +19,7 @@ class SimpleSpanList; class AnnotationDeserializer { public: - AnnotationDeserializer(const FixedTypeRepo &repo, vespalib::nbostream &stream, - uint16_t version); + AnnotationDeserializer(const FixedTypeRepo &repo, vespalib::nbostream &stream, uint16_t version); std::unique_ptr<SpanTree> readSpanTree(); std::unique_ptr<SpanNode> readSpanNode(); diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index aa4aabb8350..ded79cfe00f 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -31,6 +31,7 @@ #include <vespa/vespalib/tensor/serialization/typed_binary_format.h> #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/base/exceptions.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".vespadocumentdeserializer"); diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h index 0c58cd44cfe..534ee3a6c2d 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentserializer.h @@ -6,7 +6,10 @@ #include <vespa/document/fieldvalue/fieldvaluewriter.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/update/updatevisitor.h> -#include <vespa/vespalib/objects/nbostream.h> + +namespace vespalib { + class nbostream; +} namespace document { diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp index 7cf782376c9..dd9a231536a 100644 --- a/memfilepersistence/src/tests/spi/memfiletestutils.cpp +++ b/memfilepersistence/src/tests/spi/memfiletestutils.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> @@ -9,6 +8,7 @@ #include <vespa/storageframework/defaultimplementation/memory/simplememorylogic.h> #include <vespa/document/update/assignvalueupdate.h> #include <sys/time.h> +#include <vespa/vespalib/objects/nbostream.h> using document::DocumentType; diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h index d305d6a8e25..f5c8914d40d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h @@ -9,11 +9,10 @@ #include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/alloc.h> #include <vespa/memfilepersistence/mapper/versionserializer.h> +#include <vespa/vespalib/objects/nbostream.h> -namespace storage -{ -namespace memfile -{ +namespace storage { +namespace memfile { namespace util { diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp index 89d38d68e9e..33832dc98d0 100644 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/config/config.h> #include <vespa/config/print/fileconfigwriter.h> @@ -18,6 +17,7 @@ #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/slaveproc.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/objects/nbostream.h> #include <iostream> diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index a0048092052..bf123264637 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -15,6 +15,8 @@ #include <vespa/searchcore/proton/common/attrupdate.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchcore/proton/server/commit_and_wait_document_retriever.h> +#include <vespa/vespalib/objects/nbostream.h> + using document::DocumentType; using document::Field; diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h index cd3b69d1bf5..e79408fc6a3 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/searchlib/aggregation/grouping.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vector> #include <memory> @@ -17,9 +18,9 @@ namespace grouping { class GroupingContext { public: - typedef std::unique_ptr<GroupingContext> UP; + typedef std::unique_ptr<GroupingContext> UP; typedef std::shared_ptr<search::aggregation::Grouping> GroupingPtr; - typedef std::vector<GroupingPtr> GroupingList; + typedef std::vector<GroupingPtr> GroupingList; private: GroupingContext &operator=(const GroupingContext &); diff --git a/searchcore/src/vespa/searchcore/grouping/mergingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/mergingmanager.cpp index 3087fcf47d9..b8fffa4a17a 100644 --- a/searchcore/src/vespa/searchcore/grouping/mergingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/mergingmanager.cpp @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".mergingmanager"); #include "mergingmanager.h" #include <map> #include <vespa/searchlib/aggregation/grouping.h> #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/vespalib/objects/objectpredicate.h> #include <vespa/vespalib/objects/objectoperation.h> +#include <vespa/vespalib/objects/nbostream.h> namespace search { namespace grouping { diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 2db27ff0a8e..185bcbd2ca3 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -12,6 +12,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/objects/hexdump.h> #include <vespa/log/log.h> LOG_SETUP("datastore_test"); diff --git a/searchlib/src/tests/sort/sort_test.cpp b/searchlib/src/tests/sort/sort_test.cpp index 1af12c7aecb..1fb938a53cd 100644 --- a/searchlib/src/tests/sort/sort_test.cpp +++ b/searchlib/src/tests/sort/sort_test.cpp @@ -1,12 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/common/sortspec.h> #include <vespa/searchlib/common/converters.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/objects/hexdump.h> #include <vector> #include <fstream> #include <iostream> @@ -14,8 +13,6 @@ #include <stdexcept> #include <unicode/ustring.h> -LOG_SETUP("sort_test"); - using vespalib::Array; using namespace search::common; using namespace search::uca; diff --git a/searchlib/src/tests/sort/sortbenchmark.cpp b/searchlib/src/tests/sort/sortbenchmark.cpp index 1309cf57d5d..f028c81ff7a 100644 --- a/searchlib/src/tests/sort/sortbenchmark.cpp +++ b/searchlib/src/tests/sort/sortbenchmark.cpp @@ -1,12 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/common/sort.h> #include <vespa/vespalib/util/array.h> -#include <vector> - -LOG_SETUP("sort_test"); +#include <vespa/vespalib/util/buffer.h> +#include <vespa/vespalib/stllike/string.h> using vespalib::Array; using vespalib::ConstBufferRef; diff --git a/searchlib/src/vespa/searchlib/aggregation/rawrank.h b/searchlib/src/vespa/searchlib/aggregation/rawrank.h index f9ca265d84d..56c507802c1 100644 --- a/searchlib/src/vespa/searchlib/aggregation/rawrank.h +++ b/searchlib/src/vespa/searchlib/aggregation/rawrank.h @@ -1,34 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/objects/nbostream.h> -#include <vector> - namespace search { namespace aggregation { -#if 0 -/** - * Thin wrapper around a rank value represented as a sequence of - * bytes. - **/ -class RawRank -{ -private: - std::vector<uint8_t> _rank; - -public: - RawRank() : _rank() {} - RawRank(double rank); - RawRank(const char *buf, uint32_t len); - int cmp(const RawRank &rhs) const; - const std::vector<uint8_t> &getRank() const { return _rank; } - friend vespalib::nbostream &operator << (vespalib::nbostream &os, const RawRank &rr); - friend vespalib::nbostream &operator >> (vespalib::nbostream &is, RawRank &rr); -}; -#else typedef double RawRank; -#endif } // namespace search::aggregation } // namespace search diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index 13b536e8a6b..0dcf10bcdbe 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -1468,3 +1468,10 @@ ZcPostingSeqWrite::flushWordNoSkip(void) } // namespace diskindex } // namespace search + +#include <vespa/vespalib/objects/nbostream.hpp> +namespace vespalib { + using UInt32Pair = std::pair<uint32_t, uint32_t>; + template nbostream &nbostream::saveVector<UInt32Pair>(const std::vector<UInt32Pair> &); + template nbostream &nbostream::restoreVector<UInt32Pair>(std::vector<UInt32Pair> &); +}
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.h b/searchlib/src/vespa/searchlib/docstore/visitcache.h index 0602574058d..ccec52cd598 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.h +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.h @@ -7,6 +7,7 @@ #include <vespa/vespalib/stllike/cache.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/alloc.h> +#include <vespa/vespalib/objects/nbostream.h> namespace search { namespace docstore { diff --git a/searchlib/src/vespa/searchlib/expression/catserializer.cpp b/searchlib/src/vespa/searchlib/expression/catserializer.cpp index bbeca330f14..7761bd83f9c 100644 --- a/searchlib/src/vespa/searchlib/expression/catserializer.cpp +++ b/searchlib/src/vespa/searchlib/expression/catserializer.cpp @@ -4,6 +4,7 @@ #include "rawresultnode.h" #include "resultvector.h" #include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/objects/nbostream.h> namespace search { namespace expression { diff --git a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp index e812de071f5..0d0f6ffea2c 100644 --- a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp @@ -1,5 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/searchlib/expression/integerresultnode.h> #include <vespa/searchlib/expression/floatresultnode.h> #include <vespa/searchlib/expression/stringresultnode.h> @@ -15,7 +14,7 @@ namespace search { namespace expression { -using vespalib::nbostream; +using vespalib::nbo; using vespalib::Serializer; using vespalib::Deserializer; using vespalib::make_string; @@ -325,7 +324,7 @@ int64_t RawResultNode::onGetInteger(size_t index) const } nbo; nbo._int64 = 0; memcpy(nbo._bytes, &_value[0], std::min(sizeof(nbo._bytes), _value.size())); - return nbostream::n2h(nbo._int64); + return nbo::n2h(nbo._int64); } double RawResultNode::onGetFloat(size_t index) const @@ -337,7 +336,7 @@ double RawResultNode::onGetFloat(size_t index) const } nbo; nbo._double = 0; memcpy(nbo._bytes, &_value[0], std::min(sizeof(nbo._bytes), _value.size())); - return nbostream::n2h(nbo._double); + return nbo::n2h(nbo._double); } Serializer & RawResultNode::onSerialize(Serializer & os) const diff --git a/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h b/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h index fa0184dc3e5..144bf6ff47e 100644 --- a/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h +++ b/searchlib/src/vespa/searchlib/expression/unarybitfunctionnode.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/searchlib/expression/unaryfunctionnode.h> +#include <vespa/vespalib/objects/nbostream.h> namespace search { namespace expression { diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp b/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp index 442f25b94f0..faf910ea632 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.cpp @@ -103,3 +103,14 @@ operator>>(nbostream &in, DocIdAndFeatures &features) } // namespace index } // namespace search + +#include <vespa/vespalib/objects/nbostream.hpp> + +namespace vespalib { + using search::index::WordDocElementFeatures; + using search::index::WordDocElementWordPosFeatures; + template nbostream& nbostream::saveVector<WordDocElementFeatures>(const std::vector<WordDocElementFeatures> &); + template nbostream& nbostream::restoreVector<WordDocElementFeatures>(std::vector<WordDocElementFeatures> &); + template nbostream& nbostream::saveVector<WordDocElementWordPosFeatures>(const std::vector<WordDocElementWordPosFeatures> &); + template nbostream& nbostream::restoreVector<WordDocElementWordPosFeatures>(std::vector<WordDocElementWordPosFeatures> &); +}
\ No newline at end of file diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.cpp b/searchlib/src/vespa/searchlib/parsequery/parse.cpp index e071b5728e9..ec49a5dd195 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/parse.cpp @@ -1,20 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Creation date: 2000-05-15 - * - * Implementation of ParseItem - * - * Copyright (C) 1997-2003 Fast Search & Transfer ASA - * Copyright (C) 2003 Overture Services Norway AS - * ALL RIGHTS RESERVED - */ - -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(""); #include <vespa/searchlib/parsequery/parse.h> -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/nbo.h> namespace search { @@ -143,9 +130,9 @@ ParseItem::AppendBuffer(RawBuf *buf) const } if (Type() == ITEM_WAND) { buf->appendCompressedPositiveNumber(_arg1); // targetNumHits - double nboVal = vespalib::nbostream::n2h(_arg2); + double nboVal = vespalib::nbo::n2h(_arg2); buf->append(&nboVal, sizeof(nboVal)); // scoreThreshold - nboVal = vespalib::nbostream::n2h(_arg3); + nboVal = vespalib::nbo::n2h(_arg3); buf->append(&nboVal, sizeof(nboVal)); // thresholdBoostFactor } break; diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp index 06b92a429d8..784f85b95c4 100644 --- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp @@ -11,7 +11,7 @@ #include <vespa/log/log.h> #include <vespa/vespalib/util/vstringfmt.h> #include <vespa/vespalib/util/compress.h> -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/nbo.h> #include <vespa/searchlib/parsequery/simplequerystack.h> LOG_SETUP(".search.simplequerystack"); @@ -257,7 +257,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) break; case search::ParseItem::ITEM_PURE_WEIGHTED_LONG: - tmp = vespalib::nbostream::n2h(*reinterpret_cast<const uint64_t *>(p)); + tmp = vespalib::nbo::n2h(*reinterpret_cast<const uint64_t *>(p)); p += sizeof(uint64_t); result.append(make_vespa_string("%c/%lu", _G_ItemName[type], tmp)); break; @@ -275,9 +275,9 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf) if (type == search::ParseItem::ITEM_WAND) { p += vespalib::compress::Integer::decompressPositive(tmp, p); uint32_t targetNumHits = tmp; - double scoreThreshold = vespalib::nbostream::n2h(*reinterpret_cast<const double *>(p)); + double scoreThreshold = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); p += sizeof(double); - double thresholdBoostFactor = vespalib::nbostream::n2h(*reinterpret_cast<const double *>(p)); // thresholdBoostFactor + double thresholdBoostFactor = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); // thresholdBoostFactor p += sizeof(double); result.append(make_vespa_string("%c/%d/%d:%.*s(%u,%f,%f)~", _G_ItemName[type], arity, idxRefLen, idxRefLen, idxRef, targetNumHits, scoreThreshold, thresholdBoostFactor)); @@ -341,7 +341,7 @@ SimpleQueryStack::ReadString(const char *&p) uint64_t SimpleQueryStack::ReadUint64(const char *&p) { - uint64_t l = static_cast<uint64_t>(vespalib::nbostream::n2h(*(const uint64_t *)p)); + uint64_t l = static_cast<uint64_t>(vespalib::nbo::n2h(*(const uint64_t *)p)); p += sizeof(uint64_t); return l; } diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index dac07640bd7..17e1cc16baa 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -10,7 +10,7 @@ #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/util/rawbuf.h> #include <vespa/vespalib/util/compress.h> -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/nbo.h> using search::query::PredicateQueryTerm; @@ -56,7 +56,7 @@ vespalib::string SimpleQueryStackDumpIterator::readString(const char *&p) { uint64_t SimpleQueryStackDumpIterator::readUint64(const char *&p) { if (p + sizeof(uint64_t) > _bufEnd) throw false; - uint64_t l = vespalib::nbostream::n2h(*(const uint64_t *)p); + uint64_t l = vespalib::nbo::n2h(*(const uint64_t *)p); p += sizeof(uint64_t); return l; } @@ -177,7 +177,7 @@ SimpleQueryStackDumpIterator::next() case ParseItem::ITEM_PURE_WEIGHTED_LONG: if (p + 8 > _bufEnd) return false; _generatedTerm.clear(); - _generatedTerm << vespalib::nbostream::n2h(*(const uint64_t *)p); + _generatedTerm << vespalib::nbo::n2h(*(const uint64_t *)p); _currTerm = _generatedTerm.c_str(); _currTermLen = _generatedTerm.size(); p += 8; @@ -269,9 +269,9 @@ SimpleQueryStackDumpIterator::next() if (_currType == ParseItem::ITEM_WAND) { p += vespalib::compress::Integer::decompressPositive(tmp, p); // targetNumHits _currArg1 = tmp; - _currArg2 = vespalib::nbostream::n2h(*reinterpret_cast<const double *>(p)); // scoreThreshold + _currArg2 = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); // scoreThreshold p += sizeof(double); - _currArg3 = vespalib::nbostream::n2h(*reinterpret_cast<const double *>(p)); // thresholdBoostFactor + _currArg3 = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); // thresholdBoostFactor p += sizeof(double); } else { _currArg1 = 0; diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index b5d43176f36..4dc5976b35d 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -1,15 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".stackdumpcreator"); - #include "stackdumpcreator.h" #include "intermediatenodes.h" #include "queryvisitor.h" #include "termnodes.h" -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/nbo.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/util/rawbuf.h> @@ -61,7 +57,7 @@ class QueryNodeConverter : public QueryVisitor { void appendDouble(double i) { _buf.preAlloc(sizeof(double)); - double nboVal = vespalib::nbostream::n2h(i); + double nboVal = vespalib::nbo::n2h(i); _buf.append(&nboVal, sizeof(double)); } void append(const vespalib::string &s) { appendString(s); } diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index 61c1c50d8d7..c00f80719a1 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -7,7 +7,7 @@ #include "term.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/parsequery/simplequerystack.h> -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/hexdump.h> namespace search { namespace query { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 07eaad9807f..df92e0eaf02 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -16,13 +16,12 @@ #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/keywordextractor.h> #include <vespa/searchsummary/docsummary/docsumformat.h> -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/hexdump.h> #include <vespa/juniper/config.h> LOG_SETUP(".searchlib.docsummary.dynamicteaserdfw"); -namespace juniper -{ +namespace juniper { struct ExplicitItemData diff --git a/staging_vespalib/src/vespa/vespalib/util/sort.h b/staging_vespalib/src/vespa/vespalib/util/sort.h index 1710d5739f9..660179301bc 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sort.h +++ b/staging_vespalib/src/vespa/vespalib/util/sort.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/nbo.h> #include <functional> #include <limits> #include <algorithm> @@ -251,7 +251,7 @@ public: template<typename C> uint32_t serializeForSort(typename C::InputType v, void * dst) { - typename C::UIntType nbo(vespalib::nbostream::n2h(C::convert(v))); + typename C::UIntType nbo(vespalib::nbo::n2h(C::convert(v))); memcpy(dst, &nbo, sizeof(nbo)); return sizeof(nbo); } diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 48d2ef2dc61..19fee57ad7e 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/storage/persistence/mergehandler.h> #include <vespa/vdstestlib/cppunit/macros.h> @@ -10,6 +9,7 @@ #include <tests/persistence/common/persistenceproviderwrapper.h> #include <tests/distributor/messagesenderstub.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/vespalib/objects/nbostream.h> #include <cmath> LOG_SETUP(".test.persistence.handler.merge"); diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index 1d5824ed62c..03d73193086 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -5,6 +5,7 @@ #include <tests/persistence/persistencetestutils.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/vespalib/objects/nbostream.h> using document::DocumentType; using storage::framework::defaultimplementation::AllocationLogic; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index c657640158d..a3cec6a8b77 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -6,6 +6,7 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/storage/common/bucketoperationlogger.h> +#include <vespa/vespalib/objects/nbostream.h> #include <vespa/log/log.h> LOG_SETUP(".persistence.mergehandler"); diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index 738fe9ceeee..5a7a2d65da8 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -1,6 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> #include <vespa/document/base/testdocman.h> #include <vespa/document/document.h> #include <vespa/document/repo/documenttyperepo.h> @@ -20,6 +19,7 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/growablebytebuffer.h> #include <vespa/storageapi/message/visitor.h> +#include <vespa/vespalib/objects/nbostream.h> using std::shared_ptr; using document::ByteBuffer; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 9c186412de6..ff54a506468 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -1,7 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> +#include "querytermdata.h" +#include "searchenvironment.h" +#include "searchvisitor.h" #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> @@ -24,9 +25,8 @@ #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vsm/vsm/docsumfilter.h> #include <vespa/vsm/vsm/vsm-adapter.h> -#include "querytermdata.h" -#include "searchenvironment.h" -#include "searchvisitor.h" +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/log/log.h> LOG_SETUP(".visitor.instance.searchvisitor"); diff --git a/vdslib/src/vespa/vdslib/container/operationlist.cpp b/vdslib/src/vespa/vdslib/container/operationlist.cpp index 26c9b6e930d..0355444d25b 100644 --- a/vdslib/src/vespa/vdslib/container/operationlist.cpp +++ b/vdslib/src/vespa/vdslib/container/operationlist.cpp @@ -2,6 +2,7 @@ #include "operationlist.h" #include "documentlist.h" #include <vespa/document/update/documentupdate.h> +#include <vespa/vespalib/objects/nbostream.h> namespace vdslib { diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index 5f239b5ced3..d24dcb6b3a4 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -2,6 +2,7 @@ #include "parameters.hpp" #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/hexdump.h> using namespace vdslib; diff --git a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp index 9089ebc3e3a..38c843e170b 100644 --- a/vespalib/src/tests/objects/nbostream/nbostream_test.cpp +++ b/vespalib/src/tests/objects/nbostream/nbostream_test.cpp @@ -2,8 +2,9 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/objects/hexdump.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <iostream> +#include <ostream> using vespalib::nbostream; using ExpBuffer = std::vector<uint8_t>; diff --git a/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index d1d713c2987..81caf706304 100644 --- a/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/vespalib/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -8,7 +8,8 @@ #include <vespa/vespalib/tensor/tensor_factory.h> #include <vespa/vespalib/tensor/serialization/typed_binary_format.h> #include <vespa/vespalib/objects/nbostream.h> -#include <iostream> +#include <vespa/vespalib/objects/hexdump.h> +#include <ostream> using namespace vespalib::tensor; using vespalib::nbostream; diff --git a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt index 7473092ad8b..f3b6a8e4c0a 100644 --- a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt @@ -2,5 +2,6 @@ vespa_add_library(vespalib_vespalib_objects OBJECT SOURCES nbostream.cpp + hexdump.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/objects/hexdump.cpp b/vespalib/src/vespa/vespalib/objects/hexdump.cpp new file mode 100644 index 00000000000..22ce94298d2 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/hexdump.cpp @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "hexdump.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib { + +namespace { + + const char * hexChar = "0123456789ABCDEF"; + +} + +string +HexDump::toString() const { + asciistream os; + os << *this; + return os.str(); +} + +asciistream & operator << (asciistream & os, const HexDump & hd) +{ + os << hd._sz << ' '; + const uint8_t *c = static_cast<const uint8_t *>(hd._buf); + for (size_t i(0); i < hd._sz; i++) { + os << hexChar[c[i] >> 4] << hexChar[c[i] & 0xf]; + } + return os; +} + +std::ostream & operator << (std::ostream & os, const HexDump & hd) +{ + return os << hd.toString(); +} + +} diff --git a/vespalib/src/vespa/vespalib/objects/hexdump.h b/vespalib/src/vespa/vespalib/objects/hexdump.h new file mode 100644 index 00000000000..c2e5cad78fa --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/hexdump.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +class asciistream; + +/* + * Helper class to provide hex dump of the contents in a buffer. + */ +class HexDump +{ +public: + HexDump(const void * buf, size_t sz) : _buf(buf), _sz(sz) { } + vespalib::string toString() const; + friend std::ostream & operator << (std::ostream & os, const HexDump & hd); + friend asciistream & operator << (asciistream & os, const HexDump & hd); +private: + const void * _buf; + size_t _sz; +}; + +} diff --git a/vespalib/src/vespa/vespalib/objects/nbo.h b/vespalib/src/vespa/vespalib/objects/nbo.h new file mode 100644 index 00000000000..5deb4ea2899 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/nbo.h @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <arpa/inet.h> + +namespace vespalib { +class nbo { +public: + static bool n2h(bool v) { return v; } + static int8_t n2h(int8_t v) { return v; } + static uint8_t n2h(uint8_t v) { return v; } + static char n2h(char v) { return v; } + static int16_t n2h(int16_t v) { return ntohs(v); } + static uint16_t n2h(uint16_t v) { return ntohs(v); } + static int32_t n2h(int32_t v) { return ntohl(v); } + static uint32_t n2h(uint32_t v) { return ntohl(v); } + static int64_t n2h(int64_t v) { return ntohll(v); } + static uint64_t n2h(uint64_t v) { return ntohll(v); } + static float n2h(float v) { + union { uint32_t _u; float _f; } uf; + uf._f = v; + uf._u = ntohl(uf._u); + return uf._f; + } + static double n2h(double v) { + union { uint64_t _u; double _f; } uf; + uf._f = v; + uf._u = ntohll(uf._u); + return uf._f; + } +private: + static uint64_t ntohll(uint64_t v) { + union { uint64_t _ll; uint32_t _l[2]; } w, r; + r._ll = v; + w._l[0] = n2h(r._l[1]); + w._l[1] = n2h(r._l[0]); + return w._ll; + } +}; + +} diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.cpp b/vespalib/src/vespa/vespalib/objects/nbostream.cpp index e1ec7803aa2..de01b8b60df 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.cpp +++ b/vespalib/src/vespa/vespalib/objects/nbostream.cpp @@ -1,48 +1,77 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "nbostream.h" +#include "nbostream.hpp" +#include "hexdump.h" #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/optimized.h> namespace vespalib { -namespace { - - const char * hexChar = "0123456789ABCDEF"; - +nbostream::nbostream(size_t initialSize) : + _wbuf(), + _rbuf(), + _rp(0), + _wp(0), + _state(ok), + _longLivedBuffer(false) +{ + extend(initialSize); } -nbostream::~nbostream() { } - -string -HexDump::toString() const { - asciistream os; - os << *this; - return os.str(); +nbostream::nbostream(const void * buf, size_t sz, bool longLivedBuffer) : + _wbuf(), + _rbuf(buf, sz), + _rp(0), + _wp(sz), + _state(ok), + _longLivedBuffer(longLivedBuffer) +{ } + +nbostream::nbostream(Alloc && buf, size_t sz) : + _wbuf(std::move(buf), sz), + _rbuf(&_wbuf[0], sz), + _rp(0), + _wp(sz), + _state(ok), + _longLivedBuffer(false) +{ + assert(_wbuf.size() >= sz); } -asciistream & operator << (asciistream & os, const HexDump & hd) +nbostream::nbostream(const nbostream & rhs) : + _wbuf(), + _rbuf(), + _rp(0), + _wp(0), + _state(ok), + _longLivedBuffer(false) { - os << hd._sz << ' '; - const uint8_t *c = static_cast<const uint8_t *>(hd._buf); - for (size_t i(0); i < hd._sz; i++) { - os << hexChar[c[i] >> 4] << hexChar[c[i] & 0xf]; - } - return os; + extend(rhs.size()); + _wp = rhs.size(); + memcpy(&_wbuf[0], &rhs._rbuf[rhs._rp], _wp); } -std::ostream & operator << (std::ostream & os, const HexDump & hd) -{ - return os << hd.toString(); +nbostream & +nbostream::operator = (const nbostream & rhs) { + if (this != &rhs) { + nbostream n(rhs); + swap(n); + } + return *this; } +nbostream::~nbostream() { } + void nbostream::fail(State s) { _state = static_cast<State>(_state | s); throw IllegalStateException(make_string("Stream failed bufsize(%zu), readp(%zu), writep(%zu)", _wbuf.size(), _rp, _wp), VESPA_STRLOC); } +std::ostream & operator << (std::ostream & os, const nbostream & s) { + return os << HexDump(&s._rbuf[s._rp], s.left()); +} + void nbostream::reserve(size_t sz) { if (capacity() < sz) { @@ -95,5 +124,13 @@ void nbostream::swap(nbostream & os) std::swap(_rbuf, os._rbuf); } +template nbostream& nbostream::saveVector<int16_t>(const std::vector<int16_t> &); +template nbostream& nbostream::restoreVector<int16_t>(std::vector<int16_t> &); +template nbostream& nbostream::saveVector<int32_t>(const std::vector<int32_t> &); +template nbostream& nbostream::restoreVector<int32_t>(std::vector<int32_t> &); +template nbostream& nbostream::saveVector<uint32_t>(const std::vector<uint32_t> &); +template nbostream& nbostream::restoreVector<uint32_t>(std::vector<uint32_t> &); +template nbostream& nbostream::saveVector<uint64_t>(const std::vector<uint64_t> &); +template nbostream& nbostream::restoreVector<uint64_t>(std::vector<uint64_t> &); } diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index e632704f1fa..6914b533b63 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -5,28 +5,10 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/array.h> #include <vespa/vespalib/util/buffer.h> -#include <arpa/inet.h> +#include "nbo.h" namespace vespalib { -class asciistream; - -/* - * Helper class to provide hex dump of the contents in a buffer. - */ -class HexDump -{ -public: - HexDump(const void * buf, size_t sz) : _buf(buf), _sz(sz) { } - vespalib::string toString() const; - friend std::ostream & operator << (std::ostream & os, const HexDump & hd); - friend asciistream & operator << (asciistream & os, const HexDump & hd); -private: - const void * _buf; - size_t _sz; -}; - - /** * Class for streaming data in network byte order, used to serialize * and deserialize objects. The java code corresponding to the C++ @@ -39,74 +21,29 @@ class nbostream using Buffer = Array<char>; using Alloc = alloc::Alloc; enum State { ok=0, eof=0x01}; - nbostream(size_t initialSize=1024) : - _wbuf(), - _rbuf(), - _rp(0), - _wp(0), - _state(ok), - _longLivedBuffer(false) - { - extend(initialSize); - } - - nbostream(const void * buf, size_t sz, bool longLivedBuffer=false) : - _wbuf(), - _rbuf(buf, sz), - _rp(0), - _wp(sz), - _state(ok), - _longLivedBuffer(longLivedBuffer) - { - } - - nbostream(Alloc && buf, size_t sz) : - _wbuf(std::move(buf), sz), - _rbuf(&_wbuf[0], sz), - _rp(0), - _wp(sz), - _state(ok), - _longLivedBuffer(false) - { - assert(_wbuf.size() >= sz); - } - - nbostream(const nbostream & rhs) : - _wbuf(), - _rbuf(), - _rp(0), - _wp(0), - _state(ok), - _longLivedBuffer(false) - { - extend(rhs.size()); - _wp = rhs.size(); - memcpy(&_wbuf[0], &rhs._rbuf[rhs._rp], _wp); - } + nbostream(size_t initialSize=1024); + nbostream(const void * buf, size_t sz, bool longLivedBuffer=false); + nbostream(Alloc && buf, size_t sz); + nbostream(const nbostream & rhs); ~nbostream(); - nbostream & operator = (const nbostream & rhs) { - if (this != &rhs) { - nbostream n(rhs); - swap(n); - } - return *this; - } - nbostream & operator << (double v) { double n(n2h(v)); write8(&n); return *this; } - nbostream & operator >> (double & v) { double n; read8(&n); v = n2h(n); return *this; } - nbostream & operator << (float v) { float n(n2h(v)); write4(&n); return *this; } - nbostream & operator >> (float & v) { float n; read4(&n); v = n2h(n); return *this; } - nbostream & operator << (int64_t v) { int64_t n(n2h(v)); write8(&n); return *this; } - nbostream & operator >> (int64_t & v) { int64_t n; read8(&n); v = n2h(n); return *this; } - nbostream & operator << (uint64_t v) { uint64_t n(n2h(v)); write8(&n); return *this; } - nbostream & operator >> (uint64_t & v) { uint64_t n; read8(&n); v = n2h(n); return *this; } - nbostream & operator << (int32_t v) { int32_t n(n2h(v)); write4(&n); return *this; } - nbostream & operator >> (int32_t & v) { int32_t n; read4(&n); v = n2h(n); return *this; } - nbostream & operator << (uint32_t v) { uint32_t n(n2h(v)); write4(&n); return *this; } - nbostream & operator >> (uint32_t & v) { uint32_t n; read4(&n); v = n2h(n); return *this; } - nbostream & operator << (int16_t v) { int16_t n(n2h(v)); write2(&n); return *this; } - nbostream & operator >> (int16_t & v) { int16_t n; read2(&n); v = n2h(n); return *this; } - nbostream & operator << (uint16_t v) { uint16_t n(n2h(v)); write2(&n); return *this; } - nbostream & operator >> (uint16_t & v) { uint16_t n; read2(&n); v = n2h(n); return *this; } + nbostream & operator = (const nbostream & rhs); + + nbostream & operator << (double v) { double n(nbo::n2h(v)); write8(&n); return *this; } + nbostream & operator >> (double & v) { double n; read8(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (float v) { float n(nbo::n2h(v)); write4(&n); return *this; } + nbostream & operator >> (float & v) { float n; read4(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (int64_t v) { int64_t n(nbo::n2h(v)); write8(&n); return *this; } + nbostream & operator >> (int64_t & v) { int64_t n; read8(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (uint64_t v) { uint64_t n(nbo::n2h(v)); write8(&n); return *this; } + nbostream & operator >> (uint64_t & v) { uint64_t n; read8(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (int32_t v) { int32_t n(nbo::n2h(v)); write4(&n); return *this; } + nbostream & operator >> (int32_t & v) { int32_t n; read4(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (uint32_t v) { uint32_t n(nbo::n2h(v)); write4(&n); return *this; } + nbostream & operator >> (uint32_t & v) { uint32_t n; read4(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (int16_t v) { int16_t n(nbo::n2h(v)); write2(&n); return *this; } + nbostream & operator >> (int16_t & v) { int16_t n; read2(&n); v = nbo::n2h(n); return *this; } + nbostream & operator << (uint16_t v) { uint16_t n(nbo::n2h(v)); write2(&n); return *this; } + nbostream & operator >> (uint16_t & v) { uint16_t n; read2(&n); v = nbo::n2h(n); return *this; } nbostream & operator << (int8_t v) { write1(&v); return *this; } nbostream & operator >> (int8_t & v) { read1(&v); return *this; } nbostream & operator << (uint8_t v) { write1(&v); return *this; } @@ -198,36 +135,12 @@ class nbostream // For checkpointing where capacity should be restored template <typename T> nbostream & - saveVector(const std::vector<T> &val) - { - size_t valCapacity = val.capacity(); - size_t valSize = val.size(); - assert(valCapacity >= valSize); - *this << valCapacity << valSize; - for (const T & v : val) { - *this << v; - } - return *this; - } + saveVector(const std::vector<T> &val); // For checkpointing where capacity should be restored template <typename T> nbostream & - restoreVector(std::vector<T> &val) - { - size_t valCapacity = 0; - size_t valSize = 0; - *this >> valCapacity >> valSize; - assert(valCapacity >= valSize); - val.reserve(valCapacity); - val.clear(); - T i; - for (size_t j = 0; j < valSize; ++j) { - *this >> i; - val.push_back(i); - } - return *this; - } + restoreVector(std::vector<T> &val); size_t size() const { return left(); } size_t capacity() const { return _wbuf.size(); } @@ -241,19 +154,7 @@ class nbostream bool good() const { return _state == ok; } void clear() { _wbuf.clear(); _wp = _rp = 0; _state = ok; } void adjustReadPos(ssize_t adj) { size_t npos = _rp + adj; if (__builtin_expect(npos > _wp, false)) { fail(eof); } _rp = npos; } - friend std::ostream & operator << (std::ostream & os, const nbostream & s) { return os << HexDump(&s._rbuf[s._rp], s.left()); } - static bool n2h(bool v) { return v; } - static int8_t n2h(int8_t v) { return v; } - static uint8_t n2h(uint8_t v) { return v; } - static char n2h(char v) { return v; } - static int16_t n2h(int16_t v) { return ntohs(v); } - static uint16_t n2h(uint16_t v) { return ntohs(v); } - static int32_t n2h(int32_t v) { return ntohl(v); } - static uint32_t n2h(uint32_t v) { return ntohl(v); } - static int64_t n2h(int64_t v) { return ntohll(v); } - static uint64_t n2h(uint64_t v) { return ntohll(v); } - static float n2h(float v) { union { uint32_t _u; float _f; } uf; uf._f = v; uf._u = ntohl(uf._u); return uf._f; } - static double n2h(double v) { union { uint64_t _u; double _f; } uf; uf._f = v; uf._u = ntohll(uf._u); return uf._f; } + friend std::ostream & operator << (std::ostream & os, const nbostream & s); void write(const void *v, size_t sz) { if (__builtin_expect(space() < sz, false)) { extend(sz); @@ -313,7 +214,6 @@ class nbostream adjustReadPos(strSize); } private: - static uint64_t ntohll(uint64_t v) { union { uint64_t _ll; uint32_t _l[2]; } w, r; r._ll = v; w._l[0] = n2h(r._l[1]); w._l[1] = n2h(r._l[0]); return w._ll; } void read1(void *v) { read(v, 1); } void read2(void *v) { read(v, 2); } void read4(void *v) { read(v, 4); } diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.hpp b/vespalib/src/vespa/vespalib/objects/nbostream.hpp new file mode 100644 index 00000000000..767c485550c --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/nbostream.hpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "nbostream.h" + +namespace vespalib { + +template <typename T> +nbostream & +nbostream::saveVector(const std::vector<T> &val) +{ + size_t valCapacity = val.capacity(); + size_t valSize = val.size(); + assert(valCapacity >= valSize); + *this << valCapacity << valSize; + for (const T & v : val) { + *this << v; + } + return *this; +} + +template <typename T> +nbostream & +nbostream::restoreVector(std::vector<T> &val) +{ + size_t valCapacity = 0; + size_t valSize = 0; + *this >> valCapacity >> valSize; + assert(valCapacity >= valSize); + val.reserve(valCapacity); + val.clear(); + T i; + for (size_t j = 0; j < valSize; ++j) { + *this >> i; + val.push_back(i); + } + return *this; +} + +} diff --git a/vsm/src/vespa/vsm/common/document.h b/vsm/src/vespa/vsm/common/document.h index a1c12dafbb6..b4fc7c86d8e 100644 --- a/vsm/src/vespa/vsm/common/document.h +++ b/vsm/src/vespa/vsm/common/document.h @@ -5,8 +5,11 @@ #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/vespalib/stllike/hash_map.h> -namespace vsm -{ +namespace vespalib { + class asciistream; +} + +namespace vsm { /// Type to identify fields in documents. typedef unsigned int FieldIdT; |