diff options
author | Geir Storli <geirstorli@yahoo.no> | 2017-02-06 14:49:53 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-02-06 14:49:53 +0100 |
commit | e2ee26a6524b664f7ef057dff345652b67cf9307 (patch) | |
tree | 41b4f64bdff8c86c71acaeeff1e8c99c2d52de9f | |
parent | bbf3e05f20e7b40e969fc0b9b090bd38e6d0edb3 (diff) | |
parent | f5fd8418f1d1c48cb28dc308821ce189cca4fa52 (diff) |
Merge pull request #1682 from yahoo/havardpe/wire-in-new-input-output-model
Havardpe/wire in new input output model
139 files changed, 509 insertions, 1559 deletions
diff --git a/config/src/vespa/config/common/configvalue.cpp b/config/src/vespa/config/common/configvalue.cpp index ce6f0a47cbd..de9f3500d5b 100644 --- a/config/src/vespa/config/common/configvalue.cpp +++ b/config/src/vespa/config/common/configvalue.cpp @@ -64,7 +64,7 @@ ConfigValue::serializeV1(vespalib::slime::Cursor & cursor) const // TODO: Remove v1 when we can bump disk format. std::vector<vespalib::string> lines(getLegacyFormat()); for (size_t i = 0; i < lines.size(); i++) { - cursor.addString(vespalib::slime::Memory(lines[i])); + cursor.addString(vespalib::Memory(lines[i])); } } diff --git a/config/src/vespa/config/common/misc.cpp b/config/src/vespa/config/common/misc.cpp index 51a6051e20b..071411f170a 100644 --- a/config/src/vespa/config/common/misc.cpp +++ b/config/src/vespa/config/common/misc.cpp @@ -5,8 +5,11 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/memory.h> #include "exceptions.h" +using vespalib::Memory; + namespace config { vespalib::string diff --git a/config/src/vespa/config/common/payload_converter.cpp b/config/src/vespa/config/common/payload_converter.cpp index 2f7feb6a1b0..5dc45c12fa3 100644 --- a/config/src/vespa/config/common/payload_converter.cpp +++ b/config/src/vespa/config/common/payload_converter.cpp @@ -1,7 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "payload_converter.h" +#include <vespa/vespalib/data/memory.h> using namespace vespalib::slime; +using vespalib::Memory; namespace config { diff --git a/config/src/vespa/config/common/payload_converter.h b/config/src/vespa/config/common/payload_converter.h index e53705fc633..3a9d53f4c77 100644 --- a/config/src/vespa/config/common/payload_converter.h +++ b/config/src/vespa/config/common/payload_converter.h @@ -15,14 +15,14 @@ class PayloadConverter : public vespalib::slime::ObjectTraverser, public vespali public: PayloadConverter(const vespalib::slime::Inspector & inspector); const std::vector<vespalib::string> & convert(); - void field(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & inspector); + void field(const vespalib::Memory & symbol, const vespalib::slime::Inspector & inspector); void entry(size_t idx, const vespalib::slime::Inspector & inspector); private: void printPrefix(); void encode(const vespalib::slime::Inspector & inspector); - void encode(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & inspector); - void encodeObject(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & object); - void encodeArray(const vespalib::slime::Memory & symbol, const vespalib::slime::Inspector & object); + void encode(const vespalib::Memory & symbol, const vespalib::slime::Inspector & inspector); + void encodeObject(const vespalib::Memory & symbol, const vespalib::slime::Inspector & object); + void encodeArray(const vespalib::Memory & symbol, const vespalib::slime::Inspector & object); void encodeValue(const vespalib::slime::Inspector & value); void encodeString(const vespalib::string & value); void encodeQuotedString(const vespalib::string & value); diff --git a/config/src/vespa/config/common/trace.h b/config/src/vespa/config/common/trace.h index 268d46d0de6..abbd8c3838a 100644 --- a/config/src/vespa/config/common/trace.h +++ b/config/src/vespa/config/common/trace.h @@ -39,8 +39,8 @@ public: private: void serializeTraceLog(vespalib::slime::Cursor & array) const; void deserializeTraceLog(const vespalib::slime::Inspector & inspector); - static const vespalib::slime::Memory TRACELOG; - static const vespalib::slime::Memory TRACELEVEL; + static const vespalib::Memory TRACELOG; + static const vespalib::Memory TRACELEVEL; vespalib::TraceNode _root; uint32_t _traceLevel; const Clock & _clock; diff --git a/config/src/vespa/config/configgen/map_inserter.h b/config/src/vespa/config/configgen/map_inserter.h index d03cf4eb6cd..75238df912a 100644 --- a/config/src/vespa/config/configgen/map_inserter.h +++ b/config/src/vespa/config/configgen/map_inserter.h @@ -15,7 +15,7 @@ template<typename T, typename Converter = config::internal::ValueConverter<T> > class MapInserter : public ::vespalib::slime::ObjectTraverser { public: MapInserter(std::map<vespalib::string, T> & map); - void field(const ::vespalib::slime::Memory & symbol, const ::vespalib::slime::Inspector & inspector); + void field(const ::vespalib::Memory & symbol, const ::vespalib::slime::Inspector & inspector); private: std::map<vespalib::string, T> & _map; }; diff --git a/config/src/vespa/config/configgen/map_inserter.hpp b/config/src/vespa/config/configgen/map_inserter.hpp index cf1f7150041..543ae3d7b45 100644 --- a/config/src/vespa/config/configgen/map_inserter.hpp +++ b/config/src/vespa/config/configgen/map_inserter.hpp @@ -10,7 +10,7 @@ MapInserter<T, Converter>::MapInserter(std::map<vespalib::string, T> & map) template<typename T, typename Converter> void -MapInserter<T, Converter>::field(const ::vespalib::slime::Memory & symbol, const ::vespalib::slime::Inspector & inspector) +MapInserter<T, Converter>::field(const ::vespalib::Memory & symbol, const ::vespalib::slime::Inspector & inspector) { Converter converter; _map[symbol.make_string()] = converter(inspector); diff --git a/config/src/vespa/config/frt/frtconfigresponsev3.cpp b/config/src/vespa/config/frt/frtconfigresponsev3.cpp index b9550ba0ec4..191b1932bf0 100644 --- a/config/src/vespa/config/frt/frtconfigresponsev3.cpp +++ b/config/src/vespa/config/frt/frtconfigresponsev3.cpp @@ -17,7 +17,7 @@ using namespace config::protocol::v3; namespace config { std::string make_json(const Slime &slime, bool compact) { - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(slime, buf, compact); return buf.get().make_string(); } diff --git a/config/src/vespa/config/frt/protocol.h b/config/src/vespa/config/frt/protocol.h index 6334d78ceea..45d8e15b77f 100644 --- a/config/src/vespa/config/frt/protocol.h +++ b/config/src/vespa/config/frt/protocol.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/data/slime/memory.h> +#include <vespa/vespalib/data/memory.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/config/common/compressiontype.h> #include <vespa/vespalib/util/alloc.h> @@ -29,30 +29,30 @@ struct Payload { namespace v2 { -extern const vespalib::slime::Memory REQUEST_VERSION; -extern const vespalib::slime::Memory REQUEST_DEF_NAME; -extern const vespalib::slime::Memory REQUEST_DEF_NAMESPACE; -extern const vespalib::slime::Memory REQUEST_DEF_MD5; -extern const vespalib::slime::Memory REQUEST_DEF_CONTENT; -extern const vespalib::slime::Memory REQUEST_CLIENT_CONFIGID; -extern const vespalib::slime::Memory REQUEST_CLIENT_HOSTNAME; -extern const vespalib::slime::Memory REQUEST_CONFIG_MD5; -extern const vespalib::slime::Memory REQUEST_CURRENT_GENERATION; -extern const vespalib::slime::Memory REQUEST_WANTED_GENERATION; -extern const vespalib::slime::Memory REQUEST_TIMEOUT; -extern const vespalib::slime::Memory REQUEST_TRACE; -extern const vespalib::slime::Memory REQUEST_VESPA_VERSION; - -extern const vespalib::slime::Memory RESPONSE_VERSION; -extern const vespalib::slime::Memory RESPONSE_DEF_NAME; -extern const vespalib::slime::Memory RESPONSE_DEF_NAMESPACE; -extern const vespalib::slime::Memory RESPONSE_DEF_MD5; -extern const vespalib::slime::Memory RESPONSE_CONFIGID; -extern const vespalib::slime::Memory RESPONSE_CLIENT_HOSTNAME; -extern const vespalib::slime::Memory RESPONSE_CONFIG_MD5; -extern const vespalib::slime::Memory RESPONSE_CONFIG_GENERATION; -extern const vespalib::slime::Memory RESPONSE_PAYLOAD; -extern const vespalib::slime::Memory RESPONSE_TRACE; +extern const vespalib::Memory REQUEST_VERSION; +extern const vespalib::Memory REQUEST_DEF_NAME; +extern const vespalib::Memory REQUEST_DEF_NAMESPACE; +extern const vespalib::Memory REQUEST_DEF_MD5; +extern const vespalib::Memory REQUEST_DEF_CONTENT; +extern const vespalib::Memory REQUEST_CLIENT_CONFIGID; +extern const vespalib::Memory REQUEST_CLIENT_HOSTNAME; +extern const vespalib::Memory REQUEST_CONFIG_MD5; +extern const vespalib::Memory REQUEST_CURRENT_GENERATION; +extern const vespalib::Memory REQUEST_WANTED_GENERATION; +extern const vespalib::Memory REQUEST_TIMEOUT; +extern const vespalib::Memory REQUEST_TRACE; +extern const vespalib::Memory REQUEST_VESPA_VERSION; + +extern const vespalib::Memory RESPONSE_VERSION; +extern const vespalib::Memory RESPONSE_DEF_NAME; +extern const vespalib::Memory RESPONSE_DEF_NAMESPACE; +extern const vespalib::Memory RESPONSE_DEF_MD5; +extern const vespalib::Memory RESPONSE_CONFIGID; +extern const vespalib::Memory RESPONSE_CLIENT_HOSTNAME; +extern const vespalib::Memory RESPONSE_CONFIG_MD5; +extern const vespalib::Memory RESPONSE_CONFIG_GENERATION; +extern const vespalib::Memory RESPONSE_PAYLOAD; +extern const vespalib::Memory RESPONSE_TRACE; const vespalib::slime::Inspector & extractPayload(const vespalib::Slime & data); @@ -60,10 +60,10 @@ const vespalib::slime::Inspector & extractPayload(const vespalib::Slime & data); namespace v3 { -extern const vespalib::slime::Memory REQUEST_COMPRESSION_TYPE; -extern const vespalib::slime::Memory RESPONSE_COMPRESSION_INFO; -extern const vespalib::slime::Memory RESPONSE_COMPRESSION_INFO_TYPE; -extern const vespalib::slime::Memory RESPONSE_COMPRESSION_INFO_UNCOMPRESSED_SIZE; +extern const vespalib::Memory REQUEST_COMPRESSION_TYPE; +extern const vespalib::Memory RESPONSE_COMPRESSION_INFO; +extern const vespalib::Memory RESPONSE_COMPRESSION_INFO_TYPE; +extern const vespalib::Memory RESPONSE_COMPRESSION_INFO_UNCOMPRESSED_SIZE; struct DecompressedData { DecompressedData(vespalib::alloc::Alloc mem, uint32_t sz) @@ -71,14 +71,14 @@ struct DecompressedData { memRef(static_cast<const char *>(memory.get()), sz), size(sz) { } - DecompressedData(const vespalib::slime::Memory & mem, uint32_t sz) + DecompressedData(const vespalib::Memory & mem, uint32_t sz) : memory(), memRef(mem), size(sz) {} vespalib::alloc::Alloc memory; - vespalib::slime::Memory memRef; + vespalib::Memory memRef; uint32_t size; }; diff --git a/config/src/vespa/config/print/fileconfigformatter.cpp b/config/src/vespa/config/print/fileconfigformatter.cpp index 64421e2bbc8..c983572e04a 100644 --- a/config/src/vespa/config/print/fileconfigformatter.cpp +++ b/config/src/vespa/config/print/fileconfigformatter.cpp @@ -11,9 +11,9 @@ using namespace vespalib::slime::convenience; using vespalib::slime::ArrayTraverser; using vespalib::slime::ObjectTraverser; -using vespalib::slime::BufferedOutput; -using vespalib::slime::SimpleBuffer; -using vespalib::slime::Output; +using vespalib::OutputWriter; +using vespalib::SimpleBuffer; +using vespalib::Output; namespace config { void doEncode(ConfigDataBuffer & buffer, Output & output); @@ -24,12 +24,12 @@ namespace { struct ConfigEncoder : public ArrayTraverser, public ObjectTraverser { - BufferedOutput &out; + OutputWriter &out; int level; bool head; std::vector<std::string> prefixList; - ConfigEncoder(BufferedOutput &out_in) + ConfigEncoder(OutputWriter &out_in) : out(out_in), level(0), head(true) {} void printPrefix() { @@ -80,9 +80,9 @@ struct ConfigEncoder : public ArrayTraverser, out.commit(len); } void encodeSTRING(const Memory &memory) { - out.writeByte('\"'); + out.write('\"'); encodeSTRINGNOQUOTE(memory); - out.writeByte('\"'); + out.write('\"'); } void encodeARRAY(const Inspector &inspector) { ArrayTraverser &array_traverser = *this; @@ -107,10 +107,10 @@ struct ConfigEncoder : public ArrayTraverser, prefixList.pop_back(); } else { printPrefix(); - out.writeByte(' '); + out.write(' '); if (type.compare("enum") == 0) encodeSTRINGNOQUOTE(inspector["value"].asString()); else encodeValue(inspector["value"]); - out.writeByte('\n'); + out.write('\n'); } } } @@ -133,7 +133,7 @@ struct ConfigEncoder : public ArrayTraverser, virtual void entry(size_t idx, const Inspector &inspector); virtual void field(const Memory &symbol_name, const Inspector &inspector); - static void encode(Inspector & root, BufferedOutput &out) { + static void encode(Inspector & root, OutputWriter &out) { ConfigEncoder encoder(out); encoder.encodeValue(root); } @@ -158,14 +158,14 @@ ConfigEncoder::entry(size_t index, const Inspector &inspector) prefixList.pop_back(); } else { printPrefix(); - out.writeByte('['); + out.write('['); encodeLONG(index); - out.writeByte(']'); - out.writeByte(' '); + out.write(']'); + out.write(' '); if (type.compare("enum") == 0) encodeSTRINGNOQUOTE(inspector["value"].asString()); else encodeValue(inspector["value"]); - out.writeByte('\n'); + out.write('\n'); } } } @@ -196,11 +196,11 @@ ConfigEncoder::field(const Memory &symbol_name, const Inspector &inspector) } else { printPrefix(); encodeSTRINGNOQUOTE(symbol_name); - out.writeByte(' '); + out.write(' '); if (type.compare("enum") == 0) encodeSTRINGNOQUOTE(inspector["value"].asString()); else encodeValue(inspector["value"]); - out.writeByte('\n'); + out.write('\n'); } } } @@ -212,7 +212,7 @@ namespace config { void doEncode(ConfigDataBuffer & buffer, Output & output) { - BufferedOutput out(output); + OutputWriter out(output, 8000); ConfigEncoder::encode(buffer.slimeObject().get()["configPayload"], out); } diff --git a/config/src/vespa/config/print/jsonconfigformatter.cpp b/config/src/vespa/config/print/jsonconfigformatter.cpp index ad6e41be756..9ca35b42f35 100644 --- a/config/src/vespa/config/print/jsonconfigformatter.cpp +++ b/config/src/vespa/config/print/jsonconfigformatter.cpp @@ -3,10 +3,10 @@ #include "jsonconfigformatter.h" #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/slime/json_format.h> -#include <vespa/vespalib/data/slime/simple_buffer.h> +#include <vespa/vespalib/data/simple_buffer.h> -using vespalib::slime::SimpleBuffer; -using vespalib::slime::Output; +using vespalib::SimpleBuffer; +using vespalib::Output; using vespalib::slime::JsonFormat; namespace config { diff --git a/config/src/vespa/config/retriever/configsnapshot.cpp b/config/src/vespa/config/retriever/configsnapshot.cpp index 02331671245..b46b3f34898 100644 --- a/config/src/vespa/config/retriever/configsnapshot.cpp +++ b/config/src/vespa/config/retriever/configsnapshot.cpp @@ -4,11 +4,12 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/config/common/misc.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/memory.h> using vespalib::Slime; using vespalib::slime::Cursor; using vespalib::slime::Inspector; -using vespalib::slime::Memory; +using vespalib::Memory; namespace config { @@ -134,7 +135,7 @@ ConfigSnapshot::serializeKeyV1(Cursor & cursor, const ConfigKey & key) const Cursor & defSchema(cursor.setArray("defSchema")); const SchemaVector & vec(key.getDefSchema()); for (SchemaVector::const_iterator it(vec.begin()), mt(vec.end()); it != mt; it++) { - defSchema.addString(vespalib::slime::Memory(*it)); + defSchema.addString(vespalib::Memory(*it)); } } diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java index e6d7f29ad36..c415b8732a9 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -859,12 +859,12 @@ public class CppClassBuilder implements ClassBuilder { w.write(indent + "vespalib::slime::Cursor & __croot = __slime.setObject();\n"); w.write(indent + "__croot.setDouble(\"version\", CONFIG_DEF_SERIALIZE_VERSION);\n"); w.write(indent + "vespalib::slime::Cursor & __key = __croot.setObject(\"configKey\");\n"); - w.write(indent + "__key.setString(\"defName\", vespalib::slime::Memory(CONFIG_DEF_NAME));\n"); - w.write(indent + "__key.setString(\"defNamespace\", vespalib::slime::Memory(CONFIG_DEF_NAMESPACE));\n"); - w.write(indent + "__key.setString(\"defMd5\", vespalib::slime::Memory(CONFIG_DEF_MD5));\n"); + w.write(indent + "__key.setString(\"defName\", vespalib::Memory(CONFIG_DEF_NAME));\n"); + w.write(indent + "__key.setString(\"defNamespace\", vespalib::Memory(CONFIG_DEF_NAMESPACE));\n"); + w.write(indent + "__key.setString(\"defMd5\", vespalib::Memory(CONFIG_DEF_MD5));\n"); w.write(indent + "vespalib::slime::Cursor & __keySchema =__key.setArray(\"defSchema\");\n"); w.write(indent + "for (size_t i = 0; i < CONFIG_DEF_SCHEMA.size(); i++) {\n"); - w.write(indent + " __keySchema.addString(vespalib::slime::Memory(CONFIG_DEF_SCHEMA[i]));\n"); + w.write(indent + " __keySchema.addString(vespalib::Memory(CONFIG_DEF_SCHEMA[i]));\n"); w.write(indent + "}\n"); w.write(indent + "vespalib::slime::Cursor & __cursor = __croot.setObject(\"configPayload\");\n"); } else { @@ -887,14 +887,14 @@ public class CppClassBuilder implements ClassBuilder { String repType = slimeTypeMap.get("enum"); w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); w.write(indent + " __c3.set" + repType); - w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + "[__i])));\n"); + w.write("(\"value\", vespalib::Memory(get" + childType + "Name(" + childName + "[__i])));\n"); } else if (child instanceof LeafCNode) { String type = ((LeafCNode) child).getType(); String repType = slimeTypeMap.get(type); w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); w.write(indent + " __c3.set" + repType); if ("String".equals(repType)) { - w.write("(\"value\", vespalib::slime::Memory(" + childName + "[__i]));\n"); + w.write("(\"value\", vespalib::Memory(" + childName + "[__i]));\n"); } else { w.write("(\"value\", " + childName + "[__i]);\n"); } @@ -910,19 +910,19 @@ public class CppClassBuilder implements ClassBuilder { String childMapType = getTypeName(child, true); w.write(indent + "for (" + childMapType + "::const_iterator it(" + childName + ".begin()), mt(" + childName + ".end()); it != mt; it++) {\n"); w.write(indent + " vespalib::slime::Cursor & __c3 = __c2.addObject();\n"); - w.write(indent + " __c3.setString(\"key\", vespalib::slime::Memory(it->first));\n"); + w.write(indent + " __c3.setString(\"key\", vespalib::Memory(it->first));\n"); if (child instanceof LeafCNode.EnumLeaf) { String repType = slimeTypeMap.get("enum"); w.write(indent + " __c3.setString(\"type\", \"enum\");\n"); w.write(indent + " __c3.set" + repType); - w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(it->second)));\n"); + w.write("(\"value\", vespalib::Memory(get" + childType + "Name(it->second)));\n"); } else if (child instanceof LeafCNode) { String type = ((LeafCNode) child).getType(); String repType = slimeTypeMap.get(type); w.write(indent + " __c3.setString(\"type\", \"" + type + "\");\n"); w.write(indent + " __c3.set" + repType); if ("String".equals(repType)) { - w.write("(\"value\", vespalib::slime::Memory(it->second));\n"); + w.write("(\"value\", vespalib::Memory(it->second));\n"); } else { w.write("(\"value\", it->second);\n"); } @@ -937,14 +937,14 @@ public class CppClassBuilder implements ClassBuilder { String repType = slimeTypeMap.get("enum"); w.write(indent + "__c.setString(\"type\", \"enum\");\n"); w.write(indent + "__c.set" + repType); - w.write("(\"value\", vespalib::slime::Memory(get" + childType + "Name(" + childName + ")));\n"); + w.write("(\"value\", vespalib::Memory(get" + childType + "Name(" + childName + ")));\n"); } else if (child instanceof LeafCNode) { String type = ((LeafCNode) child).getType(); String repType = slimeTypeMap.get(type); w.write(indent + "__c.setString(\"type\", \"" + type + "\");\n"); w.write(indent + "__c.set" + repType); if ("String".equals(repType)) { - w.write("(\"value\", vespalib::slime::Memory(" + childName + "));\n"); + w.write("(\"value\", vespalib::Memory(" + childName + "));\n"); } else { w.write("(\"value\", " + childName + ");\n"); } diff --git a/configutil/src/lib/configstatus.cpp b/configutil/src/lib/configstatus.cpp index 76a1faec625..5063a6d09cc 100644 --- a/configutil/src/lib/configstatus.cpp +++ b/configutil/src/lib/configstatus.cpp @@ -43,7 +43,7 @@ struct ComponentTraverser : public vespalib::slime::ObjectTraverser } virtual void - field(const vespalib::slime::Memory &symbol_name, const vespalib::slime::Inspector &inspector) { + field(const vespalib::Memory &symbol_name, const vespalib::slime::Inspector &inspector) { switch (_state) { case ROOT: _component = symbol_name.make_string(); @@ -210,7 +210,7 @@ ConfigStatus::checkServiceGeneration(std::string configId, std::string host, int return false; } - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(slime, buf, false); if (slime.get()["config"].valid()) { diff --git a/document/src/vespa/document/serialization/slime_output_to_vector.h b/document/src/vespa/document/serialization/slime_output_to_vector.h index 4dcd6524a96..ae95accdd79 100644 --- a/document/src/vespa/document/serialization/slime_output_to_vector.h +++ b/document/src/vespa/document/serialization/slime_output_to_vector.h @@ -2,26 +2,29 @@ #pragma once -#include <vespa/vespalib/data/slime/output.h> +#include <vespa/vespalib/data/output.h> +#include <vespa/vespalib/data/writable_memory.h> #include <vector> namespace document { -class SlimeOutputToVector : public vespalib::slime::Output { +class SlimeOutputToVector : public vespalib::Output { std::vector<char> _buf; size_t _size; public: SlimeOutputToVector() : _buf(), _size(0) {} - virtual char *exchange(char *p, size_t commit, size_t reserve) { - assert(!commit || p == &_buf[_size]); - (void) p; - _size += commit; + vespalib::WritableMemory reserve(size_t reserve) { if (_size + reserve > _buf.size()) { _buf.resize(_size + reserve); } - return &_buf[_size]; + return vespalib::WritableMemory(&_buf[_size], _buf.size() - _size); + } + + Output &commit(size_t commit) override { + _size += commit; + return *this; } const char *data() const { return &_buf[0]; } diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index fc0cf593334..93856b376aa 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -35,7 +35,7 @@ using std::vector; using vespalib::Slime; using vespalib::asciistream; using vespalib::nbostream; -using vespalib::slime::Memory; +using vespalib::Memory; using vespalib::stringref; namespace document { diff --git a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp b/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp index 0a916ad871a..a6ef2f3fe5f 100644 --- a/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_slime_serialization/tensor_slime_serialization_test.cpp @@ -32,7 +32,7 @@ struct Fixture const TensorDimensions &rhsDimensions) { Tensor::UP rhsTensor(createTensor(rhs, rhsDimensions)); auto slime = SlimeBinaryFormat::serialize(*rhsTensor); - vespalib::slime::Memory memory_exp(exp); + vespalib::Memory memory_exp(exp); vespalib::Slime expSlime; size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime); EXPECT_EQUAL(used, memory_exp.size); @@ -132,7 +132,7 @@ struct DenseFixture const DenseTensorCells &rhs) { Tensor::UP rhsTensor(createTensor(rhs)); auto slime = SlimeBinaryFormat::serialize(*rhsTensor); - vespalib::slime::Memory memory_exp(exp); + vespalib::Memory memory_exp(exp); vespalib::Slime expSlime; size_t used = vespalib::slime::JsonFormat::decode(memory_exp, expSlime); EXPECT_EQUAL(used, memory_exp.size); diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp index d04b7ae3c2b..37355406952 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp @@ -15,7 +15,6 @@ LOG_SETUP(".vespalib.eval.value_cache.constant_tensor_loader"); namespace vespalib { namespace eval { -using Memory = slime::Memory; using Inspector = slime::Inspector; using ObjectTraverser = slime::ObjectTraverser; diff --git a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp index 16726038cf8..7bb1477e7a3 100644 --- a/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/slime_binary_format.cpp @@ -9,7 +9,7 @@ #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/slime.h> - +#include <vespa/vespalib/data/memory.h> namespace vespalib { namespace tensor { @@ -19,7 +19,6 @@ using slime::Inserter; using slime::SlimeInserter; using slime::Cursor; using slime::ObjectInserter; -using slime::Memory; namespace { diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 8a36ed16cbf..72a4a287f2c 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -770,9 +770,9 @@ void MetricManagerTest::testJsonOutput() // Parse it back using namespace vespalib::slime; vespalib::Slime slime; - size_t parsed = JsonFormat::decode(Memory(jsonData), slime); + size_t parsed = JsonFormat::decode(vespalib::Memory(jsonData), slime); if (jsonData.size() != parsed) { - SimpleBuffer buffer; + vespalib::SimpleBuffer buffer; JsonFormat::encode(slime, buffer, false); std::ostringstream ost; ost << "Failed to parse JSON: '\n" @@ -902,7 +902,7 @@ public: : _jsonText(jsonText) { vespalib::slime::JsonFormat::decode( - vespalib::slime::Memory(jsonText), _tree); + vespalib::Memory(jsonText), _tree); } // XXX ideally all these should be const, but is not clear how/if it's diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 1450ef5fc37..9b8e7240376 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -434,13 +434,13 @@ Test::assertSlime(const std::string &exp, const DocsumReply &reply, uint32_t id, memcpy(&classId, docsum.data.c_str(), sizeof(classId)); ASSERT_EQUAL(::search::fs4transport::SLIME_MAGIC_ID, classId); vespalib::Slime slime; - vespalib::slime::Memory serialized(docsum.data.c_str() + sizeof(classId), + vespalib::Memory serialized(docsum.data.c_str() + sizeof(classId), docsum.data.size() - sizeof(classId)); size_t decodeRes = vespalib::slime::BinaryFormat::decode(serialized, slime); ASSERT_EQUAL(decodeRes, serialized.size); if (relaxed) { - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(slime, buf, false); vespalib::Slime tmpSlime; size_t used = vespalib::slime::JsonFormat::decode(buf.get(), tmpSlime); diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index c2ece54ff1a..8cd98da51eb 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -21,6 +21,7 @@ using namespace vespalib::slime; using vespalib::stringref; using vespalib::ConstBufferRef; using vespalib::DataBuffer; +using vespalib::Memory; namespace proton { @@ -226,7 +227,7 @@ verify(vespalib::stringref exp, const Slime & slime) vespalib::Slime expSlime; size_t used = vespalib::slime::JsonFormat::decode(expMemory, expSlime); EXPECT_EQUAL(used, expMemory.size); - SimpleBuffer output; + vespalib::SimpleBuffer output; vespalib::slime::JsonFormat::encode(slime, output, true); Slime reSlimed; used = vespalib::slime::JsonFormat::decode(output.get(), reSlimed); @@ -386,7 +387,7 @@ verifyRPC(size_t count, { Server server; vespalib::Slime slimeRequest = createSlimeRequestLarger(count); - SimpleBuffer buf; + vespalib::SimpleBuffer buf; BinaryFormat::encode(slimeRequest, buf); EXPECT_EQUAL(requestSize, buf.get().size); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index e1ba72ac710..508dafe4d0a 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -13,7 +13,7 @@ using search::common::Location; using vespalib::string; using vespalib::slime::SymbolTable; using vespalib::slime::NIX; -using vespalib::slime::Memory; +using vespalib::Memory; using vespalib::slime::Cursor; using vespalib::slime::Symbol; using vespalib::slime::Inserter; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.cpp index 03ae8055c1c..e4cb28b56e8 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryfieldconverter.cpp @@ -369,7 +369,7 @@ class JsonFiller : public ConstFieldValueVisitor { if (tensor) { auto slime = vespalib::tensor::SlimeBinaryFormat::serialize(*tensor); - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(*slime, buf, true); _json.appendJSON(buf.get().make_string()); } else { @@ -652,7 +652,7 @@ class SlimeFiller : public ConstFieldValueVisitor { if (tensor) { vespalib::tensor::TypedBinaryFormat::serialize(s, *tensor); } - _inserter.insertData(vespalib::slime::Memory(s.peek(), s.size())); + _inserter.insertData(vespalib::Memory(s.peek(), s.size())); } void visit(const ReferenceFieldValue& value) override { diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index 163c8e7efd2..132856ef755 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -15,11 +15,11 @@ using search::engine::DocsumReply; using vespalib::slime::Inspector; using vespalib::slime::Cursor; using vespalib::slime::ObjectSymbolInserter; -using vespalib::slime::Memory; +using vespalib::Memory; using vespalib::slime::Symbol; using vespalib::slime::BinaryFormat; using vespalib::slime::ArrayTraverser; -using vespalib::slime::SimpleBuffer; +using vespalib::SimpleBuffer; using vespalib::DataBuffer; using vespalib::ConstBufferRef; using document::CompressionConfig; diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 6ce62f228b8..e2025f77e43 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -612,9 +612,9 @@ FS4Packet_DOCSUM::toString(uint32_t indent) const if (_buf.size() >= sizeof(magic) && memcmp(_buf.c_str(), &magic, sizeof(magic)) == 0) { vespalib::Slime slime; - vespalib::slime::Memory input(_buf.c_str() + sizeof(magic), + vespalib::Memory input(_buf.c_str() + sizeof(magic), _buf.size() - sizeof(magic)); - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::BinaryFormat::decode(input, slime); vespalib::slime::JsonFormat::encode(slime, buf, false); s += make_string("%*s json dump : ", indent, ""); diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp index d51de9b09f1..543d972e88b 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_tree_analyzer.cpp @@ -16,7 +16,7 @@ using std::map; using std::min; using std::string; using vespalib::slime::Inspector; -using vespalib::slime::Memory; +using vespalib::Memory; namespace search { namespace predicate { diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.cpp index 84df5adc9d4..8c45b18c1e5 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.cpp @@ -21,7 +21,7 @@ using document::Predicate; using std::map; using std::string; using vespalib::slime::Inspector; -using vespalib::slime::Memory; +using vespalib::Memory; namespace search { namespace predicate { diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h b/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h index e6dd9dbaeb4..a955d90411b 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_tree_annotator.h @@ -4,7 +4,7 @@ #include "predicate_interval.h" #include <unordered_map> -#include <vespa/vespalib/data/slime/memory.h> +#include <vespa/vespalib/data/memory.h> #include <vespa/vespalib/stllike/string.h> #include <climits> #include <vector> @@ -15,7 +15,7 @@ namespace search { namespace predicate { struct RangeFeature { - vespalib::slime::Memory label; + vespalib::Memory label; int64_t from; int64_t to; }; diff --git a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h index 9ab88d07c28..771573a30a1 100644 --- a/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h +++ b/searchlib/src/vespa/searchlib/util/slime_output_raw_buf_adapter.h @@ -2,21 +2,24 @@ #pragma once -#include <vespa/vespalib/data/slime/output.h> +#include <vespa/vespalib/data/output.h> #include "rawbuf.h" namespace search { -class SlimeOutputRawBufAdapter : public ::vespalib::slime::Output +class SlimeOutputRawBufAdapter : public ::vespalib::Output { private: RawBuf &_buf; public: SlimeOutputRawBufAdapter(RawBuf &buf) : _buf(buf) {} - virtual char *exchange(char *, size_t commit, size_t reserve) { + vespalib::WritableMemory reserve(size_t reserve) override { + return vespalib::WritableMemory(_buf.GetWritableFillPos(reserve), reserve); + } + Output &commit(size_t commit) override { _buf.Fill(commit); - return _buf.GetWritableFillPos(reserve); + return *this; } }; diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 45e4fb009ac..f5f37468b79 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/util/rawbuf.h> #include <vespa/searchlib/common/transport.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/slime/simple_buffer.h> +#include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index 6af943c516b..eb5219153c1 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -232,7 +232,7 @@ SingleAttrDFW::insertField(uint32_t docid, if (tensor) { vespalib::nbostream str; vespalib::tensor::TypedBinaryFormat::serialize(str, *tensor); - target.insertData(vespalib::slime::Memory(str.peek(), str.size())); + target.insertData(vespalib::Memory(str.peek(), str.size())); } } default: @@ -246,13 +246,13 @@ SingleAttrDFW::insertField(uint32_t docid, case RES_LONG_STRING: case RES_STRING: { s = v.getString(docid, NULL, 0); // no need to pass in a buffer, this attribute has a string storage. - target.insertString(vespalib::slime::Memory(s)); + target.insertString(vespalib::Memory(s)); break; } case RES_LONG_DATA: case RES_DATA: { s = v.getString(docid, NULL, 0); // no need to pass in a buffer, this attribute has a string storage. - target.insertData(vespalib::slime::Memory(s)); + target.insertData(vespalib::Memory(s)); break; } default: @@ -355,7 +355,7 @@ MultiAttrDFW::insertField(uint32_t docid, vespalib::slime::Inserter &target) { using vespalib::slime::Cursor; - using vespalib::slime::Memory; + using vespalib::Memory; const IAttributeVector & v = vec(*state); uint32_t entries = v.getValueCount(docid); bool isWeightedSet = v.hasWeightedSetType(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp index 446ca347bff..ba30940f5e0 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumfieldwriter.cpp @@ -160,7 +160,7 @@ CopyDFW::insertField(uint32_t /*docid*/, // resolve field entry->_resolve_field(&spt, &len, &state->_docSumFieldSpace); - vespalib::slime::Memory value(spt, len); + vespalib::Memory value(spt, len); target.insertString(value); break; } @@ -172,7 +172,7 @@ CopyDFW::insertField(uint32_t /*docid*/, // resolve field entry->_resolve_field(&dpt, &len, &state->_docSumFieldSpace); - vespalib::slime::Memory value(dpt, len); + vespalib::Memory value(dpt, len); target.insertData(value); break; } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index df92e0eaf02..f18cf1a4da4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -480,7 +480,7 @@ DynamicTeaserDFW::insertField(uint32_t docid, vespalib::slime::Inserter &target) { vespalib::string teaser = makeDynamicTeaser(docid, gres, state); - vespalib::slime::Memory value(teaser.c_str(), teaser.size()); + vespalib::Memory value(teaser.c_str(), teaser.size()); target.insertString(value); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 250ad272355..723238b1b8b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -89,7 +89,7 @@ AbsDistanceDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *st target.insertLong(absdist); } else { vespalib::string value = vespalib::stringify(absdist); - vespalib::slime::Memory data(value.c_str(), value.size()); + vespalib::Memory data(value.c_str(), value.size()); if (type == RES_STRING || type == RES_LONG_STRING || @@ -272,7 +272,7 @@ PositionsDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState * dsS ResType type, vespalib::slime::Inserter &target) { vespalib::asciistream val(formatField(vec(*dsState), docid, type)); - target.insertString(vespalib::slime::Memory(val.c_str(), val.size())); + target.insertString(vespalib::Memory(val.c_str(), val.size())); } //-------------------------------------------------------------------------- diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index b81b1cc231d..07c82382233 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -79,7 +79,7 @@ RankFeaturesDFW::insertField(uint32_t docid, if (type == RES_FEATUREDATA && values != NULL) { vespalib::slime::Cursor& obj = target.insertObject(); for (uint32_t i = 0; i < names.size(); ++i) { - vespalib::slime::Memory name(names[i].c_str(), names[i].size()); + vespalib::Memory name(names[i].c_str(), names[i].size()); obj.setDouble(name, values[i]); } return; @@ -92,7 +92,7 @@ RankFeaturesDFW::insertField(uint32_t docid, featureDump(json, names[i], values[i]); } json.endObject(); - vespalib::slime::Memory value(json.toString().c_str(), + vespalib::Memory value(json.toString().c_str(), json.toString().size()); if (type == RES_STRING || type == RES_LONG_STRING) { target.insertString(value); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index 14a4383009b..66e50fdff1a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -31,7 +31,7 @@ SummaryFeaturesDFW::init(IDocsumEnvironment * env) } static vespalib::string _G_cached("vespa.summaryFeatures.cached"); -static vespalib::slime::Memory _M_cached("vespa.summaryFeatures.cached"); +static vespalib::Memory _M_cached("vespa.summaryFeatures.cached"); void SummaryFeaturesDFW::insertField(uint32_t docid, @@ -51,7 +51,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, if (type == RES_FEATUREDATA && values != NULL) { vespalib::slime::Cursor& obj = target.insertObject(); for (uint32_t i = 0; i < names.size(); ++i) { - vespalib::slime::Memory name(names[i].c_str(), names[i].size()); + vespalib::Memory name(names[i].c_str(), names[i].size()); obj.setDouble(name, values[i]); } if (state->_summaryFeaturesCached) { @@ -75,7 +75,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, json.appendDouble(0.0); } json.endObject(); - vespalib::slime::Memory value(json.toString().c_str(), + vespalib::Memory value(json.toString().c_str(), json.toString().size()); if (type == RES_STRING || type == RES_LONG_STRING) { target.insertString(value); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp index 0be38f632db..716c4a024d8 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/textextractordfw.cpp @@ -49,7 +49,7 @@ TextExtractorDFW::insertField(uint32_t, } else { LOG(warning, "Did not find input entry using field enum %d. Write an empty field", _inputFieldEnum); } - target.insertString(vespalib::slime::Memory(extracted.c_str(), extracted.size())); + target.insertString(vespalib::Memory(extracted.c_str(), extracted.size())); } uint32_t diff --git a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp index 4b02d586222..58f87ffa593 100644 --- a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp @@ -120,7 +120,7 @@ vespalib::string render(const StateExplorer &state, const Url &url) { top.setObject(); } inject_children(state, url, top.get()); - slime::SimpleBuffer buf; + SimpleBuffer buf; slime::JsonFormat::encode(top, buf, true); return buf.get().make_string(); } diff --git a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp index a99ef240833..d6945064eb6 100644 --- a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp @@ -10,7 +10,7 @@ namespace { struct SelfState : slime::ObjectTraverser { Slime result; SelfState() : result() { result.setObject(); } - virtual void field(const slime::Memory &key, const slime::Inspector &value) { + virtual void field(const Memory &key, const slime::Inspector &value) { if (value.type().getId() != slime::OBJECT::ID) { slime::inject(value, slime::ObjectInserter(result.get(), key)); } @@ -19,7 +19,7 @@ struct SelfState : slime::ObjectTraverser { struct ChildrenNames : slime::ObjectTraverser { std::vector<vespalib::string> result; - virtual void field(const slime::Memory &key, const slime::Inspector &value) { + virtual void field(const Memory &key, const slime::Inspector &value) { if (value.type().getId() == slime::OBJECT::ID) { result.push_back(key.make_string()); } diff --git a/storage/src/tests/common/hostreporter/hostinfotest.cpp b/storage/src/tests/common/hostreporter/hostinfotest.cpp index a8157a4c250..96211f9b2cc 100644 --- a/storage/src/tests/common/hostreporter/hostinfotest.cpp +++ b/storage/src/tests/common/hostreporter/hostinfotest.cpp @@ -13,7 +13,7 @@ namespace { using Object = vespalib::JsonStream::Object; using End = vespalib::JsonStream::End; using JsonFormat = vespalib::slime::JsonFormat; -using Memory = vespalib::slime::Memory; +using Memory = vespalib::Memory; class DummyReporter: public HostReporter { public: diff --git a/storage/src/tests/common/hostreporter/util.cpp b/storage/src/tests/common/hostreporter/util.cpp index d4dc4fe01e5..d79455c1184 100644 --- a/storage/src/tests/common/hostreporter/util.cpp +++ b/storage/src/tests/common/hostreporter/util.cpp @@ -12,7 +12,7 @@ namespace { using Object = vespalib::JsonStream::Object; using End = vespalib::JsonStream::End; using JsonFormat = vespalib::slime::JsonFormat; -using Memory = vespalib::slime::Memory; +using Memory = vespalib::Memory; } void diff --git a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp index 9804b1c17ba..c1b5487717e 100644 --- a/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp +++ b/storage/src/tests/distributor/distributor_host_info_reporter_test.cpp @@ -191,11 +191,11 @@ DistributorHostInfoReporterTest::generateExampleJson() std::string path = TEST_PATH("../../../protocols/getnodestate/distributor.json"); std::string goldenString = File::readAll(path); - vespalib::slime::Memory goldenMemory(goldenString); + vespalib::Memory goldenMemory(goldenString); vespalib::Slime goldenSlime; vespalib::slime::JsonFormat::decode(goldenMemory, goldenSlime); - vespalib::slime::Memory jsonMemory(jsonString); + vespalib::Memory jsonMemory(jsonString); vespalib::Slime jsonSlime; vespalib::slime::JsonFormat::decode(jsonMemory, jsonSlime); diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index dcbd20b2a75..83fad3a7484 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -234,7 +234,7 @@ StateManagerTest::testClusterStateVersion() _manager->setClusterState(state); std::string nodeInfoString(_manager->getNodeInfo()); - vespalib::slime::Memory goldenMemory(nodeInfoString); + vespalib::Memory goldenMemory(nodeInfoString); vespalib::Slime nodeInfo; vespalib::slime::JsonFormat::decode(nodeInfoString, nodeInfo); diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index 08558b515c8..87686f06d6c 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -128,8 +128,8 @@ void StateReporterTest::tearDown() { vespalib::Slime slime; \ { \ using namespace vespalib::slime; \ - size_t parsed = JsonFormat::decode(Memory(jsonData), slime); \ - SimpleBuffer buffer; \ + size_t parsed = JsonFormat::decode(vespalib::Memory(jsonData), slime); \ + vespalib::SimpleBuffer buffer; \ JsonFormat::encode(slime, buffer, false); \ if (jsonData.size() != parsed) { \ std::ostringstream error; \ diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt index 05ba9815ddc..fe6873d2b8d 100644 --- a/vbench/CMakeLists.txt +++ b/vbench/CMakeLists.txt @@ -20,8 +20,6 @@ vespa_define_module( src/tests/app_dumpurl src/tests/app_vbench src/tests/benchmark_headers - src/tests/buffered_output - src/tests/byte_input src/tests/dispatcher src/tests/dropped_tagger src/tests/handler_thread @@ -33,7 +31,6 @@ vespa_define_module( src/tests/latency_analyzer src/tests/line_reader src/tests/mapped_file_input - src/tests/memory src/tests/qps_analyzer src/tests/qps_tagger src/tests/request_dumper @@ -42,10 +39,8 @@ vespa_define_module( src/tests/server_socket src/tests/server_spec src/tests/server_tagger - src/tests/simple_buffer src/tests/socket src/tests/taint src/tests/time_queue src/tests/timer - src/tests/writable_memory ) diff --git a/vbench/src/apps/vbench/vbench.cpp b/vbench/src/apps/vbench/vbench.cpp index 614856cd18c..8e13f078b45 100644 --- a/vbench/src/apps/vbench/vbench.cpp +++ b/vbench/src/apps/vbench/vbench.cpp @@ -36,7 +36,7 @@ int run(const std::string &cfg_name) { return 1; } vespalib::Slime cfg; - vespalib::slime::Memory mapped_cfg(cfg_file.get().data, + vespalib::Memory mapped_cfg(cfg_file.get().data, cfg_file.get().size); if (!vespalib::slime::JsonFormat::decode(mapped_cfg, cfg)) { fprintf(stderr, "unable to parse config file: %s\n", diff --git a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp index 2135d53acba..b74f0f4f866 100644 --- a/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp +++ b/vbench/src/tests/app_dumpurl/app_dumpurl_test.cpp @@ -6,16 +6,19 @@ using namespace vbench; using vespalib::SlaveProc; +using InputReader = vespalib::InputReader; +using OutputWriter = vespalib::OutputWriter; + bool endsWith(const Memory &mem, const string &str) { return (mem.size < str.size()) ? false : (strncmp(mem.data + mem.size - str.size(), str.data(), str.size()) == 0); } void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { - ByteInput in(input); + InputReader in(input); while (!endsWith(buffer.get(), end)) { - int c = in.get(); - if (c < 0) { + char c = in.read(); + if (in.failed()) { return; } buffer.reserve(1).data[0] = c; @@ -34,11 +37,11 @@ TEST_MT_F("run dumpurl", 2, ServerSocket()) { Stream::UP stream = f1.accept(); SimpleBuffer ignore; readUntil(*stream, ignore, "\r\n\r\n"); - BufferedOutput out(*stream, 256); - out.append("HTTP/1.1 200\r\n"); - out.append("content-length: 4\r\n"); - out.append("\r\n"); - out.append("data"); + OutputWriter out(*stream, 256); + out.write("HTTP/1.1 200\r\n"); + out.write("content-length: 4\r\n"); + out.write("\r\n"); + out.write("data"); } else { std::string out; EXPECT_TRUE(SlaveProc::run(strfmt("../../apps/dumpurl/vbench_dumpurl_app localhost %d /foo", diff --git a/vbench/src/tests/app_vbench/app_vbench_test.cpp b/vbench/src/tests/app_vbench/app_vbench_test.cpp index d1285255959..e65dd9badc0 100644 --- a/vbench/src/tests/app_vbench/app_vbench_test.cpp +++ b/vbench/src/tests/app_vbench/app_vbench_test.cpp @@ -6,16 +6,19 @@ using namespace vbench; using vespalib::SlaveProc; +using InputReader = vespalib::InputReader; +using OutputWriter = vespalib::OutputWriter; + bool endsWith(const Memory &mem, const string &str) { return (mem.size < str.size()) ? false : (strncmp(mem.data + mem.size - str.size(), str.data(), str.size()) == 0); } void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { - ByteInput in(input); + InputReader in(input); while (!endsWith(buffer.get(), end)) { - int c = in.get(); - if (c < 0) { + char c = in.read(); + if (in.failed()) { return; } buffer.reserve(1).data[0] = c; @@ -38,13 +41,13 @@ TEST_MT_F("run vbench", 2, ServerSocket()) { } SimpleBuffer ignore; readUntil(*stream, ignore, "\r\n\r\n"); - BufferedOutput out(*stream, 256); - out.append("HTTP/1.1 200\r\n"); - out.append("content-length: 4\r\n"); - out.append("X-Yahoo-Vespa-NumHits 10\r\n"); - out.append("X-Yahoo-Vespa-TotalHitCount 100\r\n"); - out.append("\r\n"); - out.append("data"); + OutputWriter out(*stream, 256); + out.write("HTTP/1.1 200\r\n"); + out.write("content-length: 4\r\n"); + out.write("X-Yahoo-Vespa-NumHits 10\r\n"); + out.write("X-Yahoo-Vespa-TotalHitCount 100\r\n"); + out.write("\r\n"); + out.write("data"); } } else { std::string out; diff --git a/vbench/src/tests/buffered_output/.gitignore b/vbench/src/tests/buffered_output/.gitignore deleted file mode 100644 index ba59218c6c5..00000000000 --- a/vbench/src/tests/buffered_output/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vbench_buffered_output_test_app diff --git a/vbench/src/tests/buffered_output/CMakeLists.txt b/vbench/src/tests/buffered_output/CMakeLists.txt deleted file mode 100644 index c9c3386b0e4..00000000000 --- a/vbench/src/tests/buffered_output/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vbench_buffered_output_test_app TEST - SOURCES - buffered_output_test.cpp - DEPENDS - vbench_test - vbench -) -vespa_add_test(NAME vbench_buffered_output_test_app COMMAND vbench_buffered_output_test_app) diff --git a/vbench/src/tests/buffered_output/FILES b/vbench/src/tests/buffered_output/FILES deleted file mode 100644 index 86ca0ac84c1..00000000000 --- a/vbench/src/tests/buffered_output/FILES +++ /dev/null @@ -1 +0,0 @@ -buffered_output_test.cpp diff --git a/vbench/src/tests/buffered_output/buffered_output_test.cpp b/vbench/src/tests/buffered_output/buffered_output_test.cpp deleted file mode 100644 index 9514d619f77..00000000000 --- a/vbench/src/tests/buffered_output/buffered_output_test.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vbench/test/all.h> - -using namespace vbench; - -TEST("buffered output") { - SimpleBuffer buffer; - { // produce data - BufferedOutput dst(buffer, 3); - dst.append('a').append('b').append('c').append('\n'); - dst.append("foo bar").append('\n'); - dst.append(string("str")).append('\n'); - dst.printf("%d + %d = %d\n", 2, 2, 4); - } - { // verify data - LineReader src(buffer); - string str; - EXPECT_TRUE(src.readLine(str)); - EXPECT_EQUAL("abc", str); - EXPECT_TRUE(src.readLine(str)); - EXPECT_EQUAL("foo bar", str); - EXPECT_TRUE(src.readLine(str)); - EXPECT_EQUAL("str", str); - EXPECT_TRUE(src.readLine(str)); - EXPECT_EQUAL("2 + 2 = 4", str); - EXPECT_TRUE(!src.readLine(str)); - EXPECT_EQUAL("", str); - } -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/byte_input/.gitignore b/vbench/src/tests/byte_input/.gitignore deleted file mode 100644 index 350435ffda4..00000000000 --- a/vbench/src/tests/byte_input/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vbench_byte_input_test_app diff --git a/vbench/src/tests/byte_input/CMakeLists.txt b/vbench/src/tests/byte_input/CMakeLists.txt deleted file mode 100644 index 3c946c223b6..00000000000 --- a/vbench/src/tests/byte_input/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vbench_byte_input_test_app TEST - SOURCES - byte_input_test.cpp - DEPENDS - vbench_test - vbench -) -vespa_add_test(NAME vbench_byte_input_test_app COMMAND vbench_byte_input_test_app) diff --git a/vbench/src/tests/byte_input/FILES b/vbench/src/tests/byte_input/FILES deleted file mode 100644 index c625277f587..00000000000 --- a/vbench/src/tests/byte_input/FILES +++ /dev/null @@ -1 +0,0 @@ -byte_input_test.cpp diff --git a/vbench/src/tests/byte_input/byte_input_test.cpp b/vbench/src/tests/byte_input/byte_input_test.cpp deleted file mode 100644 index 28bc3c7b3ad..00000000000 --- a/vbench/src/tests/byte_input/byte_input_test.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vbench/test/all.h> - -using namespace vbench; - -TEST("byte input") { - SimpleBuffer buffer; - { - BufferedOutput out(buffer, 10); - out.append("abcdefgh"); - } - EXPECT_EQUAL(8u, buffer.get().size); - { - ByteInput in(buffer); - EXPECT_EQUAL('a', in.get()); - EXPECT_EQUAL('b', in.get()); - EXPECT_EQUAL('c', in.get()); - EXPECT_EQUAL('d', in.get()); - } - EXPECT_EQUAL(4u, buffer.get().size); - { - ByteInput in(buffer); - EXPECT_EQUAL('e', in.get()); - EXPECT_EQUAL('f', in.get()); - EXPECT_EQUAL('g', in.get()); - EXPECT_EQUAL('h', in.get()); - EXPECT_EQUAL(-1, in.get()); - EXPECT_EQUAL(-1, in.get()); - } - EXPECT_EQUAL(0u, buffer.get().size); - { - ByteInput in(buffer); - EXPECT_EQUAL(-1, in.get()); - } - EXPECT_EQUAL(0u, buffer.get().size); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/http_client/http_client_test.cpp b/vbench/src/tests/http_client/http_client_test.cpp index 55cb899f5cb..50331dc1fbe 100644 --- a/vbench/src/tests/http_client/http_client_test.cpp +++ b/vbench/src/tests/http_client/http_client_test.cpp @@ -4,6 +4,9 @@ using namespace vbench; +using InputReader = vespalib::InputReader; +using OutputWriter = vespalib::OutputWriter; + void checkMemory(const string &expect, const Memory &mem) { EXPECT_EQUAL(expect, string(mem.data, mem.size)); } @@ -14,10 +17,10 @@ bool endsWith(const Memory &mem, const string &str) { } void readUntil(Input &input, SimpleBuffer &buffer, const string &end) { - ByteInput in(input); + InputReader in(input); while (!endsWith(buffer.get(), end)) { - int c = in.get(); - if (c < 0) { + char c = in.read(); + if (in.failed()) { return; } buffer.reserve(1).data[0] = c; @@ -29,13 +32,13 @@ TEST_MT_F("verify request", 2, ServerSocket()) { if (thread_id == 0) { SimpleBuffer expect; { - BufferedOutput dst(expect, 256); - dst.append("GET /this/is/the/url HTTP/1.1\r\n"); - dst.append("Host: localhost\r\n"); - dst.append("User-Agent: vbench\r\n"); - dst.append("X-Yahoo-Vespa-Benchmarkdata: true\r\n"); - dst.append("X-Yahoo-Vespa-Benchmarkdata-Coverage: true\r\n"); - dst.append("\r\n"); + OutputWriter out(expect, 256); + out.write("GET /this/is/the/url HTTP/1.1\r\n"); + out.write("Host: localhost\r\n"); + out.write("User-Agent: vbench\r\n"); + out.write("X-Yahoo-Vespa-Benchmarkdata: true\r\n"); + out.write("X-Yahoo-Vespa-Benchmarkdata-Coverage: true\r\n"); + out.write("\r\n"); } SimpleBuffer actual; Stream::UP stream = f1.accept(); @@ -54,10 +57,10 @@ TEST_MT_F("verify connection close", 2, ServerSocket()) { Stream::UP stream = f1.accept(); SimpleBuffer ignore; readUntil(*stream, ignore, "\r\n\r\n"); - BufferedOutput out(*stream, 256); - out.append("HTTP/1.0 200\r\n"); - out.append("\r\n"); - out.append("data"); + OutputWriter out(*stream, 256); + out.write("HTTP/1.0 200\r\n"); + out.write("\r\n"); + out.write("data"); } else { SimpleHttpResultHandler handler; HttpClient::fetch(ServerSpec("localhost", f1.port()), @@ -73,11 +76,11 @@ TEST_MT_F("verify content length", 2, ServerSocket()) { Stream::UP stream = f1.accept(); SimpleBuffer ignore; readUntil(*stream, ignore, "\r\n\r\n"); - BufferedOutput out(*stream, 256); - out.append("HTTP/1.1 200\r\n"); - out.append("content-length: 4\r\n"); - out.append("\r\n"); - out.append("data"); + OutputWriter out(*stream, 256); + out.write("HTTP/1.1 200\r\n"); + out.write("content-length: 4\r\n"); + out.write("\r\n"); + out.write("data"); } else { SimpleHttpResultHandler handler; HttpClient::fetch(ServerSpec("localhost", f1.port()), @@ -93,16 +96,16 @@ TEST_MT_F("verify chunked encoding", 2, ServerSocket()) { Stream::UP stream = f1.accept(); SimpleBuffer ignore; readUntil(*stream, ignore, "\r\n\r\n"); - BufferedOutput out(*stream, 256); - out.append("HTTP/1.1 200\r\n"); - out.append("transfer-encoding: chunked\r\n"); - out.append("\r\n"); - out.append("2\r\n"); - out.append("da\r\n"); - out.append("2\r\n"); - out.append("ta\r\n"); - out.append("0\r\n"); - out.append("\r\n"); + OutputWriter out(*stream, 256); + out.write("HTTP/1.1 200\r\n"); + out.write("transfer-encoding: chunked\r\n"); + out.write("\r\n"); + out.write("2\r\n"); + out.write("da\r\n"); + out.write("2\r\n"); + out.write("ta\r\n"); + out.write("0\r\n"); + out.write("\r\n"); } else { SimpleHttpResultHandler handler; HttpClient::fetch(ServerSpec("localhost", f1.port()), diff --git a/vbench/src/tests/input_reader/.gitignore b/vbench/src/tests/input_reader/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vbench/src/tests/input_reader/.gitignore +++ /dev/null diff --git a/vbench/src/tests/line_reader/line_reader_test.cpp b/vbench/src/tests/line_reader/line_reader_test.cpp index 1b893a4ca15..e731bc1bc9a 100644 --- a/vbench/src/tests/line_reader/line_reader_test.cpp +++ b/vbench/src/tests/line_reader/line_reader_test.cpp @@ -4,16 +4,19 @@ using namespace vbench; +using OutputWriter = vespalib::OutputWriter; +using SimpleBuffer = vespalib::SimpleBuffer; + TEST("line reader") { SimpleBuffer buffer; { - BufferedOutput dst(buffer, 64); - dst.append("foo\n"); - dst.append("bar\r\n"); - dst.append("\n"); - dst.append("\rbaz\n"); - dst.append("\r\n"); - dst.append("zzz"); + OutputWriter dst(buffer, 64); + dst.write("foo\n"); + dst.write("bar\r\n"); + dst.write("\n"); + dst.write("\rbaz\n"); + dst.write("\r\n"); + dst.write("zzz"); } { LineReader src(buffer); diff --git a/vbench/src/tests/memory/.gitignore b/vbench/src/tests/memory/.gitignore deleted file mode 100644 index e1fd81a3e7c..00000000000 --- a/vbench/src/tests/memory/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vbench_memory_test_app diff --git a/vbench/src/tests/memory/CMakeLists.txt b/vbench/src/tests/memory/CMakeLists.txt deleted file mode 100644 index c2dc5b45306..00000000000 --- a/vbench/src/tests/memory/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vbench_memory_test_app TEST - SOURCES - memory_test.cpp - DEPENDS - vbench_test - vbench -) -vespa_add_test(NAME vbench_memory_test_app COMMAND vbench_memory_test_app) diff --git a/vbench/src/tests/memory/FILES b/vbench/src/tests/memory/FILES deleted file mode 100644 index 3627eec5ec7..00000000000 --- a/vbench/src/tests/memory/FILES +++ /dev/null @@ -1 +0,0 @@ -memory_test.cpp diff --git a/vbench/src/tests/memory/memory_test.cpp b/vbench/src/tests/memory/memory_test.cpp deleted file mode 100644 index 9504914d191..00000000000 --- a/vbench/src/tests/memory/memory_test.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vbench/test/all.h> - -using namespace vbench; - -TEST("empty memory") { - Memory m; - EXPECT_EQUAL((const char*)0, m.data); - EXPECT_EQUAL(0u, m.size); -} - -TEST("from string") { - string str("foo"); - Memory m1(str); - Memory m2 = str; - EXPECT_EQUAL(str.data(), m1.data); - EXPECT_EQUAL(str.size(), m1.size); - EXPECT_EQUAL(str.data(), m2.data); - EXPECT_EQUAL(str.size(), m2.size); -} - -TEST("from cstring") { - const char *str = "foo"; - Memory m1(str); - Memory m2 = str; - EXPECT_EQUAL(str, m1.data); - EXPECT_EQUAL(strlen(str), m1.size); - EXPECT_EQUAL(str, m2.data); - EXPECT_EQUAL(strlen(str), m2.size); -} - -TEST("from ptr and len") { - string str("foo"); - Memory m1(str.data(), str.size()); - EXPECT_EQUAL(str.data(), m1.data); - EXPECT_EQUAL(str.size(), m1.size); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/simple_buffer/.gitignore b/vbench/src/tests/simple_buffer/.gitignore deleted file mode 100644 index f40426a6708..00000000000 --- a/vbench/src/tests/simple_buffer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vbench_simple_buffer_test_app diff --git a/vbench/src/tests/simple_buffer/CMakeLists.txt b/vbench/src/tests/simple_buffer/CMakeLists.txt deleted file mode 100644 index 97ce39faa4e..00000000000 --- a/vbench/src/tests/simple_buffer/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vbench_simple_buffer_test_app TEST - SOURCES - simple_buffer_test.cpp - DEPENDS - vbench_test - vbench -) -vespa_add_test(NAME vbench_simple_buffer_test_app COMMAND vbench_simple_buffer_test_app) diff --git a/vbench/src/tests/simple_buffer/FILES b/vbench/src/tests/simple_buffer/FILES deleted file mode 100644 index 3669f575911..00000000000 --- a/vbench/src/tests/simple_buffer/FILES +++ /dev/null @@ -1 +0,0 @@ -simple_buffer_test.cpp diff --git a/vbench/src/tests/simple_buffer/simple_buffer_test.cpp b/vbench/src/tests/simple_buffer/simple_buffer_test.cpp deleted file mode 100644 index f0be5766e55..00000000000 --- a/vbench/src/tests/simple_buffer/simple_buffer_test.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vbench/test/all.h> - -using namespace vbench; - -void checkMemory(const string &expect, const Memory &mem) { - EXPECT_EQUAL(expect, string(mem.data, mem.size)); -} - -void checkBuffer(const string &expect, const SimpleBuffer &buf) { - TEST_DO(checkMemory(expect, buf.get())); -} - -TEST("simple buffer") { - SimpleBuffer buf; - TEST_DO(checkBuffer("", buf)); - { // read from empty buffer - EXPECT_EQUAL(0u, buf.obtain().size); - } - { // write to buffer - WritableMemory mem = buf.reserve(10); - TEST_DO(checkBuffer("", buf)); - EXPECT_EQUAL(10u, mem.size); - mem.data[0] = 'a'; - mem.data[1] = 'b'; - mem.data[2] = 'c'; - EXPECT_EQUAL(&buf, &buf.commit(3)); - mem = buf.reserve(0); - TEST_DO(checkBuffer("abc", buf)); - EXPECT_EQUAL(0u, mem.size); - } - { // read without evicting last byte - Memory mem = buf.obtain(); - TEST_DO(checkBuffer("abc", buf)); - TEST_DO(checkMemory("abc", mem)); - EXPECT_EQUAL(&buf, &buf.evict(2)); - mem = buf.obtain(); - TEST_DO(checkBuffer("c", buf)); - TEST_DO(checkMemory("c", mem)); - mem = buf.obtain(); - TEST_DO(checkBuffer("c", buf)); - TEST_DO(checkMemory("c", mem)); - } - { // write more to buffer - WritableMemory mem = buf.reserve(10); - EXPECT_EQUAL(10u, mem.size); - TEST_DO(checkBuffer("c", buf)); - mem.data[0] = 'd'; - EXPECT_EQUAL(&buf, &buf.commit(1)); - mem = buf.reserve(5); - TEST_DO(checkBuffer("cd", buf)); - EXPECT_EQUAL(5u, mem.size); - } - { // read until end - Memory mem = buf.obtain(); - TEST_DO(checkBuffer("cd", buf)); - TEST_DO(checkMemory("cd", mem)); - EXPECT_EQUAL(&buf, &buf.evict(1)); - mem = buf.obtain(); - TEST_DO(checkBuffer("d", buf)); - TEST_DO(checkMemory("d", mem)); - EXPECT_EQUAL(&buf, &buf.evict(1)); - mem = buf.obtain(); - TEST_DO(checkBuffer("", buf)); - TEST_DO(checkMemory("", mem)); - } -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/tests/socket/socket_test.cpp b/vbench/src/tests/socket/socket_test.cpp index e49a31773f3..02d4caed507 100644 --- a/vbench/src/tests/socket/socket_test.cpp +++ b/vbench/src/tests/socket/socket_test.cpp @@ -4,17 +4,19 @@ using namespace vbench; +using OutputWriter = vespalib::OutputWriter; + const size_t numLines = 100; struct Agent { Stream::UP socket; Agent(Stream::UP s) : socket(std::move(s)) {} void write(const char *prefix) { - BufferedOutput out(*socket, 32); + OutputWriter out(*socket, 32); for (size_t i = 0; i < numLines; ++i) { out.printf("%s%zu\n", prefix, i); } - out.append("\n"); + out.write("\n"); } void read(const char *prefix) { LineReader reader(*socket); diff --git a/vbench/src/tests/writable_memory/.gitignore b/vbench/src/tests/writable_memory/.gitignore deleted file mode 100644 index 00061ea18f7..00000000000 --- a/vbench/src/tests/writable_memory/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vbench_writable_memory_test_app diff --git a/vbench/src/tests/writable_memory/CMakeLists.txt b/vbench/src/tests/writable_memory/CMakeLists.txt deleted file mode 100644 index 8b480ff05af..00000000000 --- a/vbench/src/tests/writable_memory/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vbench_writable_memory_test_app TEST - SOURCES - writable_memory_test.cpp - DEPENDS - vbench_test - vbench -) -vespa_add_test(NAME vbench_writable_memory_test_app COMMAND vbench_writable_memory_test_app) diff --git a/vbench/src/tests/writable_memory/FILES b/vbench/src/tests/writable_memory/FILES deleted file mode 100644 index 14718444a2e..00000000000 --- a/vbench/src/tests/writable_memory/FILES +++ /dev/null @@ -1 +0,0 @@ -writable_memory_test.cpp diff --git a/vbench/src/tests/writable_memory/writable_memory_test.cpp b/vbench/src/tests/writable_memory/writable_memory_test.cpp deleted file mode 100644 index 2be9d6b5153..00000000000 --- a/vbench/src/tests/writable_memory/writable_memory_test.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vbench/test/all.h> - -using namespace vbench; - -TEST("empty") { - WritableMemory wm; - EXPECT_EQUAL((char*)0, wm.data); - EXPECT_EQUAL(0u, wm.size); -} - -TEST("from buffer") { - char buf[3]; - WritableMemory wm(buf, 3); - EXPECT_EQUAL(buf, wm.data); - EXPECT_EQUAL(3u, wm.size); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vbench/src/vbench/core/CMakeLists.txt b/vbench/src/vbench/core/CMakeLists.txt index 3817a4c6950..c97a2c67cb4 100644 --- a/vbench/src/vbench/core/CMakeLists.txt +++ b/vbench/src/vbench/core/CMakeLists.txt @@ -1,20 +1,14 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vbench_core OBJECT SOURCES - buffered_output.cpp - byte_input.cpp closeable.cpp dispatcher.cpp handler.cpp handler_thread.cpp - input.cpp input_file_reader.cpp line_reader.cpp mapped_file_input.cpp - memory.cpp - output.cpp provider.cpp - simple_buffer.cpp socket.cpp stream.cpp string.cpp @@ -22,6 +16,5 @@ vespa_add_library(vbench_core OBJECT taintable.cpp time_queue.cpp timer.cpp - writable_memory.cpp DEPENDS ) diff --git a/vbench/src/vbench/core/buffered_output.cpp b/vbench/src/vbench/core/buffered_output.cpp deleted file mode 100644 index 36e01e55fd5..00000000000 --- a/vbench/src/vbench/core/buffered_output.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "buffered_output.h" - -namespace vbench { - -BufferedOutput & -BufferedOutput::printf(const char *fmt, ...) -{ - ensureFree(256); - int space = (_data.size - _pos); - int size; - va_list ap; - va_start(ap, fmt); - size = vsnprintf(_data.data + _pos, space, fmt, ap); - va_end(ap); - assert(size >= 0); - if (size >= space) { - space = size + 1; - ensureFree(space); - va_start(ap, fmt); - size = vsnprintf(_data.data + _pos, space, fmt, ap); - va_end(ap); - assert((size + 1) == space); - } - _pos += size; - return *this; -} - -} // namespace vbench diff --git a/vbench/src/vbench/core/buffered_output.h b/vbench/src/vbench/core/buffered_output.h deleted file mode 100644 index 80ad163869e..00000000000 --- a/vbench/src/vbench/core/buffered_output.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "output.h" -#include "string.h" - -namespace vbench { - -/** - * Concrete utility class used to write small amounts of data to an - * output efficiently by buffering the output into larger chunks. - **/ -class BufferedOutput -{ -private: - Output &_output; - WritableMemory _data; - size_t _pos; - size_t _chunkSize; - - void ensureFree(size_t bytes) { - if ((_pos + bytes) > _data.size) { - _data = _output.commit(_pos).reserve(std::max(bytes, _chunkSize)); - _pos = 0; - } - } - -public: - BufferedOutput(Output &output, size_t chunkSize) - : _output(output), _data(), _pos(), _chunkSize(chunkSize) {} - ~BufferedOutput() { _output.commit(_pos); } - - BufferedOutput &append(char c) { - ensureFree(1); - _data.data[_pos++] = c; - return *this; - } - - BufferedOutput &append(const char *data, size_t size) { - ensureFree(size); - memcpy(_data.data + _pos, data, size); - _pos += size; - return *this; - } - - BufferedOutput &append(const string &str) { - return append(str.data(), str.size()); - } - - BufferedOutput &append(const char *str) { - return append(str, strlen(str)); - } - - BufferedOutput &printf(const char *fmt, ...) -#ifdef __GNUC__ - // Add printf format checks with gcc - __attribute__ ((format (printf,2,3))) -#endif - ; -}; - -} // namespace vbench - diff --git a/vbench/src/vbench/core/byte_input.cpp b/vbench/src/vbench/core/byte_input.cpp deleted file mode 100644 index d6753e2cce5..00000000000 --- a/vbench/src/vbench/core/byte_input.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "byte_input.h" - -namespace vbench { - -} // namespace vbench diff --git a/vbench/src/vbench/core/byte_input.h b/vbench/src/vbench/core/byte_input.h deleted file mode 100644 index 7294c9682bb..00000000000 --- a/vbench/src/vbench/core/byte_input.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "input.h" - -namespace vbench { - -/** - * Concrete utility class used to read input data one byte at a time - * by wrapping a generic Input implementation. - **/ -class ByteInput -{ -private: - Input &_input; - Memory _data; - size_t _pos; - -public: - /** - * Wrap an Input to read one byte at a time. - * - * @param input the underlying Input - **/ - ByteInput(Input &input) - : _input(input), _data(), _pos(0) {} - ~ByteInput() { _input.evict(_pos); } - - /** - * Read the next byte of input. - * - * @return next byte of input, or -1 if no more input is available - **/ - int get() { - if (_pos < _data.size) { - return (_data.data[_pos++] & 0xff); - } else { - _data = _input.evict(_pos).obtain(); - if ((_pos = 0) < _data.size) { - return (_data.data[_pos++] & 0xff); - } - } - return -1; - } -}; - -} // namespace vbench - diff --git a/vbench/src/vbench/core/input.cpp b/vbench/src/vbench/core/input.cpp deleted file mode 100644 index b6ffda1f255..00000000000 --- a/vbench/src/vbench/core/input.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "input.h" - -namespace vbench { - -} // namespace vbench diff --git a/vbench/src/vbench/core/input.h b/vbench/src/vbench/core/input.h deleted file mode 100644 index eceb973a94a..00000000000 --- a/vbench/src/vbench/core/input.h +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "memory.h" - -namespace vbench { - -/** - * Interface used to abstract a source of input data. Note that the - * input data itself is owned by the object implementing this - * interface. - **/ -struct Input -{ - /** - * Obtain more input data. - * - * @return the obtained input data - **/ - virtual Memory obtain() = 0; - - /** - * Evict processed input data. Never evict more data than you have - * obtained. - * - * @return this object, for chaining - * @param bytes the number of bytes to evict - **/ - virtual Input &evict(size_t bytes) = 0; - - virtual ~Input() {} -}; - -} // namespace vbench - diff --git a/vbench/src/vbench/core/input_file_reader.h b/vbench/src/vbench/core/input_file_reader.h index 919b188d44a..820962bc971 100644 --- a/vbench/src/vbench/core/input_file_reader.h +++ b/vbench/src/vbench/core/input_file_reader.h @@ -5,7 +5,7 @@ #include "string.h" #include "taint.h" #include "mapped_file_input.h" -#include "byte_input.h" +#include <vespa/vespalib/data/input_reader.h> #include "line_reader.h" namespace vbench { diff --git a/vbench/src/vbench/core/line_reader.cpp b/vbench/src/vbench/core/line_reader.cpp index 4a893a1b9fc..4bee590ec8e 100644 --- a/vbench/src/vbench/core/line_reader.cpp +++ b/vbench/src/vbench/core/line_reader.cpp @@ -22,7 +22,7 @@ bool LineReader::readLine(string &dst) { dst.clear(); - for (int c = _input.get(); c >= 0; c = _input.get()) { + for (char c = _input.read(); !_input.failed(); c = _input.read()) { if (c != '\n') { dst.push_back(c); } else { diff --git a/vbench/src/vbench/core/line_reader.h b/vbench/src/vbench/core/line_reader.h index ba5e9e63fc7..4744ba3f016 100644 --- a/vbench/src/vbench/core/line_reader.h +++ b/vbench/src/vbench/core/line_reader.h @@ -2,11 +2,14 @@ #pragma once -#include "byte_input.h" +#include <vespa/vespalib/data/input_reader.h> #include "string.h" namespace vbench { +using Input = vespalib::Input; +using InputReader = vespalib::InputReader; + /** * Concrete utility class used to read individual lines of text from * an underlying input. This class is implemented in terms of the @@ -15,7 +18,7 @@ namespace vbench { class LineReader { private: - ByteInput _input; + InputReader _input; public: /** diff --git a/vbench/src/vbench/core/mapped_file_input.h b/vbench/src/vbench/core/mapped_file_input.h index 197dbf55959..5b22e15b9bb 100644 --- a/vbench/src/vbench/core/mapped_file_input.h +++ b/vbench/src/vbench/core/mapped_file_input.h @@ -2,11 +2,14 @@ #pragma once -#include "input.h" +#include <vespa/vespalib/data/input.h> #include "taintable.h" namespace vbench { +using Input = vespalib::Input; +using Memory = vespalib::Memory; + /** * A Taintable Input implementation reading sequentially from a memory * mapped file. diff --git a/vbench/src/vbench/core/memory.cpp b/vbench/src/vbench/core/memory.cpp deleted file mode 100644 index 75d2c6cc903..00000000000 --- a/vbench/src/vbench/core/memory.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "memory.h" - -namespace vbench { - -} // namespace vbench diff --git a/vbench/src/vbench/core/memory.h b/vbench/src/vbench/core/memory.h deleted file mode 100644 index 749e7daef15..00000000000 --- a/vbench/src/vbench/core/memory.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "string.h" - -namespace vbench { - -/** - * Simple wrapper referencing a read-only region of memory. - **/ -struct Memory -{ - const char *data; - size_t size; - Memory() : data(0), size(0) {} - Memory(const char *d, size_t s) : data(d), size(s) {} - Memory(const char *str) : data(str), size(strlen(str)) {} - Memory(const string &str) : data(str.data()), size(str.size()) {} -}; - -} // namespace vbench - diff --git a/vbench/src/vbench/core/output.cpp b/vbench/src/vbench/core/output.cpp deleted file mode 100644 index 71f09ccab22..00000000000 --- a/vbench/src/vbench/core/output.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "output.h" - -namespace vbench { - -} // namespace vbench diff --git a/vbench/src/vbench/core/output.h b/vbench/src/vbench/core/output.h deleted file mode 100644 index 6ad218c9718..00000000000 --- a/vbench/src/vbench/core/output.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "writable_memory.h" - -namespace vbench { - -/** - * Interface used to abstract a destination for output data. Note that - * the output data itself is owned by the object implementing this - * interface. - **/ -struct Output -{ - /** - * Reserve space for more output data. - * - * @return the reserved output data - * @param bytes number of bytes to reserve - **/ - virtual WritableMemory reserve(size_t bytes) = 0; - - /** - * Commit produced output data. Never commit more data than you - * have reserved. - * - * @return this object, for chaining - * @param bytes number of bytes to commit - **/ - virtual Output &commit(size_t bytes) = 0; - - virtual ~Output() {} -}; - -} // namespace vbench - diff --git a/vbench/src/vbench/core/simple_buffer.cpp b/vbench/src/vbench/core/simple_buffer.cpp deleted file mode 100644 index b37696b7e8a..00000000000 --- a/vbench/src/vbench/core/simple_buffer.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "simple_buffer.h" - -namespace vbench { - -SimpleBuffer::SimpleBuffer() - : _data(), - _used(0) -{ -} - -Memory -SimpleBuffer::obtain() -{ - return Memory(&_data[0], _used); -} - -Input & -SimpleBuffer::evict(size_t bytes) -{ - assert(bytes <= _used); - _data.erase(_data.begin(), _data.begin() + bytes); - _used -= bytes; - return *this; -} - -WritableMemory -SimpleBuffer::reserve(size_t bytes) -{ - _data.resize(_used + bytes, char(0x55)); - return WritableMemory(&_data[_used], bytes); -} - -Output & -SimpleBuffer::commit(size_t bytes) -{ - assert(bytes <= (_data.size() - _used)); - _used += bytes; - return *this; -} - -bool -SimpleBuffer::operator==(const SimpleBuffer &rhs) const -{ - Memory a = get(); - Memory b = rhs.get(); - if (a.size != b.size) { - return false; - } - for (size_t i = 0; i < a.size; ++i) { - if (a.data[i] != b.data[i]) { - return false; - } - } - return true; -} - -std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf) { - Memory memory = buf.get(); - uint32_t written = 0; - uint32_t hexCount = 25; - os << "size: " << memory.size << "(bytes)" << std::endl; - for (size_t i = 0; i < memory.size; ++i, ++written) { - if (written > hexCount) { - os << std::endl; - written = 0; - } - os << strfmt("0x%02x ", memory.data[i] & 0xff); - } - if (written > 0) { - os << std::endl; - } - return os; -} - -} // namespace vbench diff --git a/vbench/src/vbench/core/simple_buffer.h b/vbench/src/vbench/core/simple_buffer.h deleted file mode 100644 index 3cf55037917..00000000000 --- a/vbench/src/vbench/core/simple_buffer.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "memory.h" -#include "input.h" -#include "output.h" -#include <ostream> - -namespace vbench { - -/** - * Simple buffer class that implements the Input/Output - * interfaces. Requesting the memory region of this buffer or - * comparing buffers will only look at the data conceptually contained - * in the buffer, ignoring evicted data and reserved data not yet - * committed. - **/ -class SimpleBuffer : public Input, - public Output -{ -private: - std::vector<char> _data; - size_t _used; - -public: - SimpleBuffer(); - Memory get() const { return Memory(&_data[0], _used); } - virtual Memory obtain(); - virtual Input &evict(size_t bytes); - virtual WritableMemory reserve(size_t bytes); - virtual Output &commit(size_t bytes); - bool operator==(const SimpleBuffer &rhs) const; -}; - -std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf); - -} // namespace vbench - diff --git a/vbench/src/vbench/core/socket.h b/vbench/src/vbench/core/socket.h index 707ea6359aa..358db8ead67 100644 --- a/vbench/src/vbench/core/socket.h +++ b/vbench/src/vbench/core/socket.h @@ -4,13 +4,19 @@ #include "string.h" #include "stream.h" -#include "simple_buffer.h" +#include <vespa/vespalib/data/simple_buffer.h> #include <memory> class FastOS_SocketInterface; namespace vbench { +using Input = vespalib::Input; +using Memory = vespalib::Memory; +using Output = vespalib::Output; +using SimpleBuffer = vespalib::SimpleBuffer; +using WritableMemory = vespalib::WritableMemory; + class Socket : public Stream { private: diff --git a/vbench/src/vbench/core/stream.h b/vbench/src/vbench/core/stream.h index 8a678647e6f..82dccc14f30 100644 --- a/vbench/src/vbench/core/stream.h +++ b/vbench/src/vbench/core/stream.h @@ -2,13 +2,16 @@ #pragma once -#include "input.h" -#include "output.h" +#include <vespa/vespalib/data/input.h> +#include <vespa/vespalib/data/output.h> #include "taintable.h" #include <memory> namespace vbench { +using Input = vespalib::Input; +using Output = vespalib::Output; + /** * A Stream is an abstract taintable entity that can act as both input * and output. diff --git a/vbench/src/vbench/core/writable_memory.cpp b/vbench/src/vbench/core/writable_memory.cpp deleted file mode 100644 index 2da24dba10d..00000000000 --- a/vbench/src/vbench/core/writable_memory.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastos/fastos.h> -#include "writable_memory.h" - -namespace vbench { - -} // namespace vbench diff --git a/vbench/src/vbench/core/writable_memory.h b/vbench/src/vbench/core/writable_memory.h deleted file mode 100644 index 3dbf85bdfee..00000000000 --- a/vbench/src/vbench/core/writable_memory.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -namespace vbench { - -/** - * Simple wrapper referencing a writable region of memory. This class - * does not have ownership of the referenced memory region. - **/ -struct WritableMemory { - char *data; - size_t size; - WritableMemory() : data(0), size(0) {} - WritableMemory(char *d, size_t s) : data(d), size(s) {} -}; - -} // namespace vbench - diff --git a/vbench/src/vbench/http/http_client.cpp b/vbench/src/vbench/http/http_client.cpp index 3503329991d..cd4a2d6c282 100644 --- a/vbench/src/vbench/http/http_client.cpp +++ b/vbench/src/vbench/http/http_client.cpp @@ -4,21 +4,23 @@ #include <vespa/fastos/fastos.h> #include "http_client.h" #include "hex_number.h" -#include <vbench/core/buffered_output.h> +#include <vespa/vespalib/data/output_writer.h> #include <vbench/core/line_reader.h> #include <algorithm> namespace vbench { +using OutputWriter = vespalib::OutputWriter; + void HttpClient::writeRequest() { - BufferedOutput dst(_conn->stream(), WRITE_SIZE); + OutputWriter dst(_conn->stream(), WRITE_SIZE); dst.printf("GET %s HTTP/1.1\r\n", _url.c_str()); dst.printf("Host: %s\r\n", _conn->server().host.c_str()); - dst.append("User-Agent: vbench\r\n"); - dst.append("X-Yahoo-Vespa-Benchmarkdata: true\r\n"); - dst.append("X-Yahoo-Vespa-Benchmarkdata-Coverage: true\r\n"); - dst.append("\r\n"); + dst.write("User-Agent: vbench\r\n"); + dst.write("X-Yahoo-Vespa-Benchmarkdata: true\r\n"); + dst.write("X-Yahoo-Vespa-Benchmarkdata-Coverage: true\r\n"); + dst.write("\r\n"); } bool diff --git a/vbench/src/vbench/http/http_result_handler.h b/vbench/src/vbench/http/http_result_handler.h index 9c6abb24131..85c230fefb8 100644 --- a/vbench/src/vbench/http/http_result_handler.h +++ b/vbench/src/vbench/http/http_result_handler.h @@ -4,10 +4,12 @@ #pragma once #include <vbench/core/string.h> -#include <vbench/core/memory.h> +#include <vespa/vespalib/data/memory.h> namespace vbench { +using Memory = vespalib::Memory; + /** * Callback interface that must be implemented in order to use the * http client. diff --git a/vbench/src/vbench/test/all.h b/vbench/src/vbench/test/all.h index 37f0a4d2ac7..8e57bc19cd1 100644 --- a/vbench/src/vbench/test/all.h +++ b/vbench/src/vbench/test/all.h @@ -21,23 +21,23 @@ #include <vbench/core/string.h> #include <vbench/core/taint.h> #include <vbench/core/taintable.h> -#include <vbench/core/output.h> -#include <vbench/core/memory.h> +#include <vespa/vespalib/data/output.h> +#include <vespa/vespalib/data/memory.h> #include <vbench/core/mapped_file_input.h> #include <vbench/core/time_queue.h> -#include <vbench/core/buffered_output.h> +#include <vespa/vespalib/data/output_writer.h> #include <vbench/core/socket.h> #include <vbench/core/handler_thread.h> #include <vbench/core/closeable.h> #include <vbench/core/handler.h> #include <vbench/core/timer.h> -#include <vbench/core/writable_memory.h> -#include <vbench/core/simple_buffer.h> +#include <vespa/vespalib/data/writable_memory.h> +#include <vespa/vespalib/data/simple_buffer.h> #include <vbench/core/provider.h> -#include <vbench/core/byte_input.h> +#include <vespa/vespalib/data/input_reader.h> #include <vbench/core/dispatcher.h> #include <vbench/core/stream.h> -#include <vbench/core/input.h> +#include <vespa/vespalib/data/input.h> #include <vbench/test/simple_http_result_handler.h> #include <vbench/test/server_socket.h> #include <vbench/test/request_receptor.h> diff --git a/vbench/src/vbench/test/simple_http_result_handler.cpp b/vbench/src/vbench/test/simple_http_result_handler.cpp index a09a4d30a22..5d223e7ba16 100644 --- a/vbench/src/vbench/test/simple_http_result_handler.cpp +++ b/vbench/src/vbench/test/simple_http_result_handler.cpp @@ -6,6 +6,8 @@ namespace vbench { +using WritableMemory = vespalib::WritableMemory; + SimpleHttpResultHandler::SimpleHttpResultHandler() : _headers(), _content(), diff --git a/vbench/src/vbench/test/simple_http_result_handler.h b/vbench/src/vbench/test/simple_http_result_handler.h index 4a6fd04b852..f5ec1c36fcf 100644 --- a/vbench/src/vbench/test/simple_http_result_handler.h +++ b/vbench/src/vbench/test/simple_http_result_handler.h @@ -4,10 +4,12 @@ #pragma once #include <vbench/http/http_result_handler.h> -#include <vbench/core/simple_buffer.h> +#include <vespa/vespalib/data/simple_buffer.h> namespace vbench { +using SimpleBuffer = vespalib::SimpleBuffer; + class SimpleHttpResultHandler : public HttpResultHandler { private: diff --git a/vespalib/src/tests/data/input_reader/input_reader_test.cpp b/vespalib/src/tests/data/input_reader/input_reader_test.cpp index 345baf12293..54c0613b6da 100644 --- a/vespalib/src/tests/data/input_reader/input_reader_test.cpp +++ b/vespalib/src/tests/data/input_reader/input_reader_test.cpp @@ -48,8 +48,9 @@ TEST("input reader smoke test") { EXPECT_TRUE(!src.failed()); EXPECT_EQUAL(src.get_offset(), strlen(data)); EXPECT_EQUAL(src.obtain(), 0u); - EXPECT_TRUE(src.failed()); + EXPECT_TRUE(!src.failed()); EXPECT_EQUAL(src.read(5), Memory()); + EXPECT_TRUE(src.failed()); EXPECT_EQUAL(src.read(), '\0'); EXPECT_EQUAL(src.obtain(), 0u); EXPECT_EQUAL(src.get_offset(), strlen(data)); @@ -100,4 +101,30 @@ TEST("require that reading a byte sequence crossing the end of input fails") { } } +TEST("expect that obtain does not set failure state on input reader") { + const char *data = "12345"; + for (bool byte_first: {true, false}) { + MemoryInput input(data); + InputReader src(input); + EXPECT_EQUAL(src.obtain(), 5); + EXPECT_EQUAL(src.obtain(), 5); + EXPECT_EQUAL(src.read(5), Memory("12345")); + EXPECT_TRUE(!src.failed()); + EXPECT_EQUAL(src.obtain(), 0); + EXPECT_EQUAL(src.obtain(), 0); + EXPECT_TRUE(!src.failed()); + if (byte_first) { + EXPECT_EQUAL(src.read(), 0); + EXPECT_TRUE(src.failed()); + EXPECT_EQUAL(src.read(5), Memory()); + } else { + EXPECT_EQUAL(src.read(5), Memory()); + EXPECT_TRUE(src.failed()); + EXPECT_EQUAL(src.read(), 0); + } + EXPECT_EQUAL(src.get_error_message(), vespalib::string("input underflow")); + EXPECT_EQUAL(src.obtain(), 0); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/data/output_writer/output_writer_test.cpp b/vespalib/src/tests/data/output_writer/output_writer_test.cpp index 750cc7799d4..3d59b2fcb1e 100644 --- a/vespalib/src/tests/data/output_writer/output_writer_test.cpp +++ b/vespalib/src/tests/data/output_writer/output_writer_test.cpp @@ -13,7 +13,7 @@ TEST("output writer smoke test") { dst.write('b'); dst.write('c'); dst.write('\n'); - dst.write("foo bar", strlen("foo bar")); + dst.write("foo bar"); dst.write('\n'); dst.printf("%d + %d = %d\n", 2, 2, 4); } diff --git a/vespalib/src/tests/data/simple_buffer/simple_buffer_test.cpp b/vespalib/src/tests/data/simple_buffer/simple_buffer_test.cpp index 41408797b49..5cc62b327ce 100644 --- a/vespalib/src/tests/data/simple_buffer/simple_buffer_test.cpp +++ b/vespalib/src/tests/data/simple_buffer/simple_buffer_test.cpp @@ -67,4 +67,10 @@ TEST("simple buffer") { } } +TEST("require that add works as expected") { + SimpleBuffer buf; + buf.add('a').add('b').add('c'); + EXPECT_EQUAL(buf.get(), Memory("abc")); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/slime/json_slime_benchmark.cpp b/vespalib/src/tests/slime/json_slime_benchmark.cpp index ea6ea89375b..e818b2d4c34 100644 --- a/vespalib/src/tests/slime/json_slime_benchmark.cpp +++ b/vespalib/src/tests/slime/json_slime_benchmark.cpp @@ -9,7 +9,7 @@ using namespace vespalib::slime::convenience; std::string make_json(const Slime &slime, bool compact) { - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(slime, buf, compact); return buf.get().make_string(); } diff --git a/vespalib/src/tests/slime/slime_binary_format_test.cpp b/vespalib/src/tests/slime/slime_binary_format_test.cpp index f35e0637a67..cafebf4fc8b 100644 --- a/vespalib/src/tests/slime/slime_binary_format_test.cpp +++ b/vespalib/src/tests/slime/slime_binary_format_test.cpp @@ -7,6 +7,7 @@ using namespace vespalib::slime::convenience; using namespace vespalib::slime::binary_format; using namespace vespalib::slime; +using namespace vespalib; //----------------------------------------------------------------------------- @@ -63,15 +64,15 @@ void verify_cmpr_ulong(uint64_t value, SimpleBuffer expect) { } } { // use write API - BufferedOutput out(buf2); + OutputWriter out(buf2, 32); write_cmpr_ulong(out, value); } EXPECT_EQUAL(MemCmp(expect.get()), MemCmp(buf1.get())); EXPECT_EQUAL(MemCmp(expect.get()), MemCmp(buf2.get())); { - BufferedInput input(expect.get()); + InputReader input(expect); EXPECT_EQUAL(value, read_cmpr_ulong(input)); - EXPECT_EQUAL(MemCmp(expect.get()), MemCmp(input.getConsumed())); + EXPECT_EQUAL(input.get_offset(), buf1.get().size); EXPECT_TRUE(!input.failed()); } } @@ -95,35 +96,35 @@ void verifyMultiEncode(const Slime & slime, const SimpleBuffer &expect) { namespace { template <typename T> -void encodeBasic(BufferedOutput &out, +void encodeBasic(OutputWriter &out, const typename TypeTraits<T>::PassType &value); template <> -void encodeBasic<BOOL>(BufferedOutput &out, const bool &value) +void encodeBasic<BOOL>(OutputWriter &out, const bool &value) { - out.writeByte(encode_type_and_meta(BOOL::ID, value ? 1 : 0)); + out.write(encode_type_and_meta(BOOL::ID, value ? 1 : 0)); } -template <> void encodeBasic<LONG>(BufferedOutput &out, const int64_t &value) +template <> void encodeBasic<LONG>(OutputWriter &out, const int64_t &value) { write_type_and_bytes<false>(out, LONG::ID, encode_zigzag(value)); } -template <> void encodeBasic<DOUBLE>(BufferedOutput &out, const double &value) +template <> void encodeBasic<DOUBLE>(OutputWriter &out, const double &value) { write_type_and_bytes<true>(out, DOUBLE::ID, encode_double(value)); } -template <> void encodeBasic<STRING>(BufferedOutput &out, const Memory &value) +template <> void encodeBasic<STRING>(OutputWriter &out, const Memory &value) { write_type_and_size(out, STRING::ID, value.size); - out.writeBytes(value.data, value.size); + out.write(value.data, value.size); } -template <> void encodeBasic<DATA>(BufferedOutput &out, const Memory &value) +template <> void encodeBasic<DATA>(OutputWriter &out, const Memory &value) { write_type_and_size(out, DATA::ID, value.size); - out.writeBytes(value.data, value.size); + out.write(value.data, value.size); } } // namespace <unnamed> @@ -167,7 +168,7 @@ void verifyBasic(const typename TypeTraits<T>::PassType &value) { SimpleBuffer expect; SimpleBuffer actual; { - BufferedOutput out(expect); + OutputWriter out(expect, 32); write_cmpr_ulong(out, 0); // num symbols encodeBasic<T>(out, value); } @@ -288,121 +289,33 @@ TEST("testCmprUlong") { } } -TEST("testWriteByte") { - SimpleBuffer buf; - { - BufferedOutput out(buf); - out.writeByte(0x55); - } - EXPECT_EQUAL(1u, buf.get().size); - EXPECT_EQUAL(0x55, buf.get().data[0]); - { - BufferedOutput out(buf); - out.writeByte(0x66); - out.writeByte(0x77); - } - EXPECT_EQUAL(3u, buf.get().size); - EXPECT_EQUAL(0x55, buf.get().data[0]); - EXPECT_EQUAL(0x66, buf.get().data[1]); - EXPECT_EQUAL(0x77, buf.get().data[2]); -} - -TEST("testWriteBytes") { - SimpleBuffer buf; - { - BufferedOutput out(buf); - out.writeBytes(0, 0); - } - EXPECT_EQUAL(0u, buf.get().size); - { - BufferedOutput out(buf); - char tmp[] = { 0x55 }; - out.writeBytes(tmp, 1); - } - EXPECT_EQUAL(1u, buf.get().size); - EXPECT_EQUAL(0x55, buf.get().data[0]); - { - BufferedOutput out(buf); - char tmp[] = { 0x66, 0x77 }; - out.writeBytes(tmp, 2); - } - EXPECT_EQUAL(3u, buf.get().size); - EXPECT_EQUAL(0x55, buf.get().data[0]); - EXPECT_EQUAL(0x66, buf.get().data[1]); - EXPECT_EQUAL(0x77, buf.get().data[2]); -} - -TEST("testReadByte") { - SimpleBuffer buf; - buf.add(0x11).add(0x22).add(0x33); - { - BufferedInput in(buf.get()); - EXPECT_EQUAL(0u, in.getConsumed().size); - EXPECT_EQUAL(0x11, in.getByte()); - EXPECT_EQUAL(1u, in.getConsumed().size); - EXPECT_EQUAL(0x22, in.getByte()); - EXPECT_EQUAL(2u, in.getConsumed().size); - EXPECT_EQUAL(0x33, in.getByte()); - EXPECT_EQUAL(3u, in.getConsumed().size); - EXPECT_TRUE(!in.failed()); - EXPECT_EQUAL(0u, in.getOffending().size); - EXPECT_EQUAL(0x00, in.getByte()); - EXPECT_TRUE(in.failed()); - EXPECT_EQUAL(0u, in.getConsumed().size); - EXPECT_EQUAL(3u, in.getOffending().size); - EXPECT_EQUAL(0x00, in.getByte()); - EXPECT_EQUAL(0x00, in.getByte()); - EXPECT_EQUAL(0x00, in.getByte()); - } -} - -TEST("testReadBytes") { - SimpleBuffer buf; - buf.add('a').add('b').add('c'); - { - BufferedInput in(buf.get()); - EXPECT_EQUAL(0u, in.getConsumed().size); - EXPECT_EQUAL("ab", in.getBytes(2).make_string()); - EXPECT_EQUAL(2u, in.getConsumed().size); - EXPECT_TRUE(!in.failed()); - EXPECT_EQUAL(0u, in.getOffending().size); - EXPECT_EQUAL(0u, in.getBytes(2).size); - EXPECT_TRUE(in.failed()); - EXPECT_EQUAL(0u, in.getConsumed().size); - EXPECT_EQUAL(3u, in.getOffending().size); - EXPECT_EQUAL(0u, in.getBytes(2).size); - EXPECT_EQUAL(0u, in.getBytes(2).size); - EXPECT_EQUAL(0u, in.getBytes(2).size); - } -} - TEST("testTypeAndSize") { for (uint32_t type = 0; type < TYPE_LIMIT; ++type) { for (uint32_t size = 0; size < 500; ++size) { SimpleBuffer expect; SimpleBuffer actual; { - BufferedOutput expect_out(expect); + OutputWriter expect_out(expect, 32); if ((size + 1) < META_LIMIT) { - expect_out.writeByte(encode_type_and_meta(type, size + 1)); + expect_out.write(encode_type_and_meta(type, size + 1)); } else { - expect_out.writeByte(type); + expect_out.write(type); write_cmpr_ulong(expect_out, size); } } { - BufferedOutput actual_out(actual); + OutputWriter actual_out(actual, 32); write_type_and_size(actual_out, type, size); } EXPECT_EQUAL(MemCmp(expect.get()), MemCmp(actual.get())); { - BufferedInput input(expect.get()); - char byte = input.getByte(); + InputReader input(expect); + char byte = input.read(); uint32_t decodedType = decode_type(byte); uint64_t decodedSize = read_size(input, decode_meta(byte)); EXPECT_EQUAL(type, decodedType); EXPECT_EQUAL(size, decodedSize); - EXPECT_EQUAL(MemCmp(expect.get()), MemCmp(input.getConsumed())); + EXPECT_EQUAL(input.get_offset(), actual.get().size); EXPECT_TRUE(!input.failed()); } } @@ -445,7 +358,7 @@ TEST("testTypeAndBytes") { uint64_t bits = build_bits(type, n, pre, (hi != 0), expect); { - BufferedOutput out(actual); + OutputWriter out(actual, 32); if (hi != 0) { write_type_and_bytes<true>(out, type, bits); } else { @@ -454,8 +367,8 @@ TEST("testTypeAndBytes") { } EXPECT_EQUAL(MemCmp(expect.get()), MemCmp(actual.get())); { - BufferedInput input(expect.get()); - uint32_t size = decode_meta(input.getByte()); + InputReader input(expect); + uint32_t size = decode_meta(input.read()); uint64_t decodedBits; if (hi != 0) { decodedBits = read_bytes<true>(input, size); @@ -463,8 +376,7 @@ TEST("testTypeAndBytes") { decodedBits = read_bytes<false>(input, size); } EXPECT_EQUAL(bits, decodedBits); - EXPECT_EQUAL(MemCmp(expect.get()), - MemCmp(input.getConsumed())); + EXPECT_EQUAL(input.get_offset(), actual.get().size); EXPECT_TRUE(!input.failed()); } } @@ -478,9 +390,9 @@ TEST("testEmpty") { SimpleBuffer expect; SimpleBuffer actual; { - BufferedOutput out(expect); + OutputWriter out(expect, 32); write_cmpr_ulong(out, 0); // num symbols - out.writeByte(0); // nix + out.write(0); // nix } BinaryFormat::encode(slime, actual); EXPECT_EQUAL(MemCmp(expect.get()), MemCmp(actual.get())); @@ -532,10 +444,10 @@ TEST("testArray") { c.addString(Memory("string")); c.addData(Memory("data")); { - BufferedOutput out(expect); + OutputWriter out(expect, 32); write_cmpr_ulong(out, 0); // num symbols write_type_and_size(out, ARRAY::ID, 6); - out.writeByte(0); + out.write(0); encodeBasic<BOOL>(out, true); encodeBasic<LONG>(out, 5); encodeBasic<DOUBLE>(out, 3.5); @@ -559,23 +471,23 @@ TEST("testObject") { c.setString("e", Memory("string")); c.setData("f", Memory("data")); { - BufferedOutput out(expect); + OutputWriter out(expect, 32); write_cmpr_ulong(out, 6); // num symbols write_cmpr_ulong(out, 1); - out.writeBytes("a", 1); // 0 + out.write("a", 1); // 0 write_cmpr_ulong(out, 1); - out.writeBytes("b", 1); // 1 + out.write("b", 1); // 1 write_cmpr_ulong(out, 1); - out.writeBytes("c", 1); // 2 + out.write("c", 1); // 2 write_cmpr_ulong(out, 1); - out.writeBytes("d", 1); // 3 + out.write("d", 1); // 3 write_cmpr_ulong(out, 1); - out.writeBytes("e", 1); // 4 + out.write("e", 1); // 4 write_cmpr_ulong(out, 1); - out.writeBytes("f", 1); // 5 + out.write("f", 1); // 5 write_type_and_size(out, OBJECT::ID, 6); write_cmpr_ulong(out, 0); - out.writeByte(0); + out.write(0); write_cmpr_ulong(out, 1); encodeBasic<BOOL>(out, true); write_cmpr_ulong(out, 2); @@ -611,14 +523,14 @@ TEST("testNesting") { } } { - BufferedOutput out(expect); + OutputWriter out(expect, 32); write_cmpr_ulong(out, 3); // num symbols write_cmpr_ulong(out, 3); - out.writeBytes("bar", 3); // 0 + out.write("bar", 3); // 0 write_cmpr_ulong(out, 3); - out.writeBytes("foo", 3); // 1 + out.write("foo", 3); // 1 write_cmpr_ulong(out, 6); - out.writeBytes("answer", 6); // 2 + out.write("answer", 6); // 2 write_type_and_size(out, OBJECT::ID, 2); write_cmpr_ulong(out, 0); // bar encodeBasic<LONG>(out, 10); @@ -654,12 +566,12 @@ TEST("testSymbolReuse") { } } { - BufferedOutput out(expect); + OutputWriter out(expect, 32); write_cmpr_ulong(out, 2); // num symbols write_cmpr_ulong(out, 3); - out.writeBytes("foo", 3); // 0 + out.write("foo", 3); // 0 write_cmpr_ulong(out, 3); - out.writeBytes("bar", 3); // 1 + out.write("bar", 3); // 1 write_type_and_size(out, ARRAY::ID, 2); write_type_and_size(out, OBJECT::ID, 2); write_cmpr_ulong(out, 0); // foo @@ -680,18 +592,18 @@ TEST("testSymbolReuse") { TEST("testOptionalDecodeOrder") { SimpleBuffer data; { - BufferedOutput out(data); + OutputWriter out(data, 32); write_cmpr_ulong(out, 5); // num symbols write_cmpr_ulong(out, 1); - out.writeBytes("d", 1); // 0 + out.write("d", 1); // 0 write_cmpr_ulong(out, 1); - out.writeBytes("e", 1); // 1 + out.write("e", 1); // 1 write_cmpr_ulong(out, 1); - out.writeBytes("f", 1); // 2 + out.write("f", 1); // 2 write_cmpr_ulong(out, 1); - out.writeBytes("b", 1); // 3 + out.write("b", 1); // 3 write_cmpr_ulong(out, 1); - out.writeBytes("c", 1); // 4 + out.write("c", 1); // 4 write_type_and_size(out, OBJECT::ID, 5); write_cmpr_ulong(out, 3); // b encodeBasic<BOOL>(out, true); @@ -751,4 +663,11 @@ TEST("require that decode_into without symbol names work") { EXPECT_EQUAL(slime.get()[my_sym].asLong(), 100); } +TEST("require that decode failure results in 0 return value") { + SimpleBuffer buf; + buf.add(char(0)); // empty symbol table, but no value + Slime slime; + EXPECT_EQUAL(BinaryFormat::decode(buf.get(), slime), 0u); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/slime/slime_json_format_test.cpp b/vespalib/src/tests/slime/slime_json_format_test.cpp index c054cc5a428..c87f0d9d4af 100644 --- a/vespalib/src/tests/slime/slime_json_format_test.cpp +++ b/vespalib/src/tests/slime/slime_json_format_test.cpp @@ -8,7 +8,7 @@ using namespace vespalib::slime::convenience; std::string make_json(const Slime &slime, bool compact) { - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(slime, buf, compact); return buf.get().make_string(); } @@ -325,23 +325,23 @@ TEST("decode whitespace") { } TEST("decode invalid input") { - EXPECT_FALSE(check_valid("")); - EXPECT_FALSE(check_valid("[")); - EXPECT_FALSE(check_valid("{")); - EXPECT_FALSE(check_valid("]")); - EXPECT_FALSE(check_valid("}")); - EXPECT_FALSE(check_valid("{]")); - EXPECT_FALSE(check_valid("[}")); - EXPECT_FALSE(check_valid("+5")); - EXPECT_FALSE(check_valid("fals")); - EXPECT_FALSE(check_valid("tru")); - EXPECT_FALSE(check_valid("nul")); - EXPECT_FALSE(check_valid("bar")); - EXPECT_FALSE(check_valid("\"bar")); - EXPECT_FALSE(check_valid("bar\"")); - EXPECT_FALSE(check_valid("'bar\"")); - EXPECT_FALSE(check_valid("\"bar'")); - EXPECT_FALSE(check_valid("{\"foo")); + EXPECT_TRUE(!check_valid("")); + EXPECT_TRUE(!check_valid("[")); + EXPECT_TRUE(!check_valid("{")); + EXPECT_TRUE(!check_valid("]")); + EXPECT_TRUE(!check_valid("}")); + EXPECT_TRUE(!check_valid("{]")); + EXPECT_TRUE(!check_valid("[}")); + EXPECT_TRUE(!check_valid("+5")); + EXPECT_TRUE(!check_valid("fals")); + EXPECT_TRUE(!check_valid("tru")); + EXPECT_TRUE(!check_valid("nul")); + EXPECT_TRUE(!check_valid("bar")); + EXPECT_TRUE(!check_valid("\"bar")); + EXPECT_TRUE(!check_valid("bar\"")); + EXPECT_TRUE(!check_valid("'bar\"")); + EXPECT_TRUE(!check_valid("\"bar'")); + EXPECT_TRUE(!check_valid("{\"foo")); } TEST("decode simplified form") { diff --git a/vespalib/src/tests/slime/slime_test.cpp b/vespalib/src/tests/slime/slime_test.cpp index efb864d9c72..9418a999cbb 100644 --- a/vespalib/src/tests/slime/slime_test.cpp +++ b/vespalib/src/tests/slime/slime_test.cpp @@ -268,7 +268,7 @@ TEST("slime toString produces human-readable JSON") { } std::string expect; { - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(slime, buf, false); expect = buf.get().make_string(); } diff --git a/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp b/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp index ba95c1fef9c..5fd5d3be3e9 100644 --- a/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp +++ b/vespalib/src/tests/slime/summary-feature-benchmark/summary-feature-benchmark.cpp @@ -6,12 +6,17 @@ using namespace vespalib; using namespace vespalib::slime::convenience; -struct MyBuffer : public slime::Output { +struct MyBuffer : public Output { std::vector<char> data; size_t used; MyBuffer() : data(1024 * 1024), used(0) {} - virtual char *exchange(char *, size_t commit, size_t) { - return &data[used += commit]; + WritableMemory reserve(size_t bytes) override { + assert(data.size() >= (used + bytes)); + return WritableMemory(&data[used], data.size() - used); + } + Output &commit(size_t bytes) override { + used += bytes; + return *this; } }; diff --git a/vespalib/src/tests/slime/type_traits.h b/vespalib/src/tests/slime/type_traits.h index 0af21629945..c16c0a68380 100644 --- a/vespalib/src/tests/slime/type_traits.h +++ b/vespalib/src/tests/slime/type_traits.h @@ -3,8 +3,7 @@ #pragma once #include <vespa/vespalib/data/slime/type.h> -#include <vespa/vespalib/data/slime/memory.h> -#include <vespa/vespalib/data/slime/stored_memory.h> +#include <vespa/vespalib/data/memory.h> namespace vespalib { namespace slime { @@ -36,13 +35,11 @@ template<> struct TypeTraits<DOUBLE> { template<> struct TypeTraits<STRING> { typedef Memory PassType; - typedef StoredMemory StoreType; static const Memory unsetValue; }; template<> struct TypeTraits<DATA> { typedef Memory PassType; - typedef StoredMemory StoreType; static const Memory unsetValue; }; diff --git a/vespalib/src/tests/trace/trace_serialization.cpp b/vespalib/src/tests/trace/trace_serialization.cpp index a7896d43338..a146f4d0ae6 100644 --- a/vespalib/src/tests/trace/trace_serialization.cpp +++ b/vespalib/src/tests/trace/trace_serialization.cpp @@ -78,7 +78,7 @@ TEST("that a trace node with children can be deserialized") { bar.setLong("timestamp", 67); bar.setString("payload", "!"); - vespalib::slime::SimpleBuffer buf; + vespalib::SimpleBuffer buf; vespalib::slime::JsonFormat::encode(slime, buf, false); SlimeTraceDeserializer deserializer(root); diff --git a/vespalib/src/vespa/vespalib/data/input_reader.cpp b/vespalib/src/vespa/vespalib/data/input_reader.cpp index 031cb9b1624..b71935c795b 100644 --- a/vespalib/src/vespa/vespalib/data/input_reader.cpp +++ b/vespalib/src/vespa/vespalib/data/input_reader.cpp @@ -9,17 +9,24 @@ namespace vespalib { size_t InputReader::obtain_slow() { - if (!failed()) { - _data = _input.evict(_pos).obtain(); - _bytes_evicted += _pos; - _pos = 0; - if (_data.size == 0) { - fail("input underflow"); - } + _data = _input.evict(_pos).obtain(); + _bytes_evicted += _pos; + _pos = 0; + if (_data.size == 0) { + _eof = true; } return size(); } +char +InputReader::read_slow() +{ + if (!failed()) { + fail("input underflow"); + } + return 0; +} + Memory InputReader::read_slow(size_t bytes) { @@ -32,6 +39,9 @@ InputReader::read_slow(size_t bytes) if (_space.size() == bytes) { return Memory(&_space[0], _space.size()); } + if (!failed()) { + fail("input underflow"); + } return Memory(); } @@ -48,6 +58,7 @@ InputReader::fail(const vespalib::string &msg) { _data = Memory(); _bytes_evicted += _pos; _pos = 0; + _eof = true; } } diff --git a/vespalib/src/vespa/vespalib/data/input_reader.h b/vespalib/src/vespa/vespalib/data/input_reader.h index 3d01eaffca0..a5e2d191147 100644 --- a/vespalib/src/vespa/vespalib/data/input_reader.h +++ b/vespalib/src/vespa/vespalib/data/input_reader.h @@ -23,6 +23,7 @@ private: Memory _data; size_t _pos; size_t _bytes_evicted; + bool _eof; vespalib::string _error; std::vector<char> _space; @@ -30,11 +31,12 @@ private: size_t size() const { return (_data.size - _pos); } size_t obtain_slow(); + char read_slow(); Memory read_slow(size_t bytes); public: explicit InputReader(Input &input) - : _input(input), _data(), _pos(0), _bytes_evicted(0), _error(), _space() {} + : _input(input), _data(), _pos(0), _bytes_evicted(0), _eof(false), _error(), _space() {} ~InputReader(); bool failed() const { return !_error.empty(); } @@ -47,11 +49,11 @@ public: * Make sure we have more input data available. * * @return number of bytes available without requesting more from - * the underlying Input. Returns 0 if and only is there is + * the underlying Input. Returns 0 if and only if there is * no more input data available. **/ size_t obtain() { - if (__builtin_expect(_pos < _data.size, true)) { + if (__builtin_expect((_pos < _data.size) || _eof, true)) { return size(); } return obtain_slow(); @@ -68,7 +70,7 @@ public: if (__builtin_expect(obtain() > 0, true)) { return _data.data[_pos++]; } - return 0; + return read_slow(); } /** diff --git a/vespalib/src/vespa/vespalib/data/memory.h b/vespalib/src/vespa/vespalib/data/memory.h index af4cf6d856b..a8a4760754a 100644 --- a/vespalib/src/vespa/vespalib/data/memory.h +++ b/vespalib/src/vespa/vespalib/data/memory.h @@ -18,6 +18,8 @@ struct Memory Memory() : data(nullptr), size(0) {} Memory(const char *d, size_t s) : data(d), size(s) {} Memory(const char *str) : data(str), size(strlen(str)) {} + Memory(const std::string &str) + : data(str.data()), size(str.size()) {} Memory(const vespalib::string &str) : data(str.data()), size(str.size()) {} Memory(const vespalib::stringref &str_ref) diff --git a/vespalib/src/vespa/vespalib/data/output_writer.h b/vespalib/src/vespa/vespalib/data/output_writer.h index 1e9db9b29fa..cc10c206f6e 100644 --- a/vespalib/src/vespa/vespalib/data/output_writer.h +++ b/vespalib/src/vespa/vespalib/data/output_writer.h @@ -2,6 +2,7 @@ #pragma once +#include "memory.h" #include "writable_memory.h" namespace vespalib { @@ -66,6 +67,11 @@ public: commit(size); } + void write(const Memory &memory) { + memcpy(reserve(memory.size), memory.data, memory.size); + commit(memory.size); + } + void printf(const char *fmt, ...) #ifdef __GNUC__ // Add printf format checks with gcc diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.cpp b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp index b178bb9e569..4fe71df1435 100644 --- a/vespalib/src/vespa/vespalib/data/simple_buffer.cpp +++ b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp @@ -41,4 +41,8 @@ SimpleBuffer::commit(size_t bytes) return *this; } +std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf) { + return os << buf.get(); +} + } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.h b/vespalib/src/vespa/vespalib/data/simple_buffer.h index 5cb2a0f1ad1..ef3b6d284dc 100644 --- a/vespalib/src/vespa/vespalib/data/simple_buffer.h +++ b/vespalib/src/vespa/vespalib/data/simple_buffer.h @@ -28,12 +28,15 @@ public: Input &evict(size_t bytes) override; WritableMemory reserve(size_t bytes) override; Output &commit(size_t bytes) override; + SimpleBuffer &add(char c) { + _data.push_back(c); + ++_used; + return *this; + } Memory get() const { return Memory(&_data[0], _used); } bool operator==(const SimpleBuffer &rhs) const { return (get() == rhs.get()); } }; -std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf) { - return os << buf.get(); -} +std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf); } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt index e1749627ac5..5c662a2dd18 100644 --- a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt @@ -6,8 +6,6 @@ vespa_add_library(vespalib_vespalib_data_slime OBJECT basic_value.cpp basic_value_factory.cpp binary_format.cpp - buffered_input.cpp - buffered_output.cpp convenience.cpp cursor.cpp empty_value_factory.cpp @@ -15,18 +13,14 @@ vespa_add_library(vespalib_vespalib_data_slime OBJECT inserter.cpp inspector.cpp json_format.cpp - memory.cpp named_symbol_inserter.cpp named_symbol_lookup.cpp nix_value.cpp object_traverser.cpp object_value.cpp - output.cpp resolved_symbol.cpp root_value.cpp - simple_buffer.cpp slime.cpp - stored_memory.cpp strfmt.cpp symbol.cpp symbol_inserter.cpp diff --git a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp index be2089b4076..33694006a3b 100644 --- a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp @@ -4,6 +4,7 @@ #include "inserter.h" #include "slime.h" #include <vespa/vespalib/util/array.hpp> +#include <vespa/vespalib/data/memory_input.h> namespace vespalib { namespace slime { @@ -13,13 +14,13 @@ namespace binary_format { struct BinaryEncoder : public ArrayTraverser, public ObjectSymbolTraverser { - BufferedOutput &out; - BinaryEncoder(BufferedOutput &out_in) : out(out_in) {} + OutputWriter &out; + BinaryEncoder(OutputWriter &out_in) : out(out_in) {} void encodeNix() { - out.writeByte(NIX::ID); + out.write(NIX::ID); } void encodeBool(bool value) { - out.writeByte(encode_type_and_meta(BOOL::ID, value ? 1 : 0)); + out.write(encode_type_and_meta(BOOL::ID, value ? 1 : 0)); } void encodeLong(int64_t value) { write_type_and_bytes<false>(out, LONG::ID, encode_zigzag(value)); @@ -29,11 +30,11 @@ struct BinaryEncoder : public ArrayTraverser, } void encodeString(const Memory &memory) { write_type_and_size(out, STRING::ID, memory.size); - out.writeBytes(memory.data, memory.size); + out.write(memory.data, memory.size); } void encodeData(const Memory &memory) { write_type_and_size(out, DATA::ID, memory.size); - out.writeBytes(memory.data, memory.size); + out.write(memory.data, memory.size); } void encodeArray(const Inspector &inspector) { ArrayTraverser &array_traverser = *this; @@ -64,7 +65,7 @@ struct BinaryEncoder : public ArrayTraverser, for (size_t i = 0; i < numSymbols; ++i) { Memory image = slime.inspect(Symbol(i)); write_cmpr_ulong(out, image.size); - out.writeBytes(image.data, image.size); + out.write(image.data, image.size); } } virtual void entry(size_t, const Inspector &inspector); @@ -88,7 +89,7 @@ BinaryEncoder::field(const Symbol &symbol, const Inspector &inspector) struct DirectSymbols { void hint_symbol_count(size_t) {} - bool add_symbol(Symbol symbol, size_t i, BufferedInput &in) { + bool add_symbol(Symbol symbol, size_t i, InputReader &in) { if (symbol.getValue() != i) { in.fail("duplicate symbols in symbol table"); return false; @@ -103,7 +104,7 @@ struct MappedSymbols { void hint_symbol_count(size_t n) { symbol_mapping.reserve(n); } - bool add_symbol(Symbol symbol, size_t, BufferedInput &) { + bool add_symbol(Symbol symbol, size_t, InputReader &) { symbol_mapping.push_back(symbol); return true; } @@ -122,13 +123,13 @@ struct SymbolHandler { template <bool remap_symbols> struct BinaryDecoder : SymbolHandler<remap_symbols>::type { - BufferedInput ∈ + InputReader ∈ using SymbolHandler<remap_symbols>::type::hint_symbol_count; using SymbolHandler<remap_symbols>::type::add_symbol; using SymbolHandler<remap_symbols>::type::map_symbol; - BinaryDecoder(BufferedInput &input) : in(input) {} + BinaryDecoder(InputReader &input) : in(input) {} Cursor &decodeNix(const Inserter &inserter) { return inserter.insertNix(); @@ -148,12 +149,12 @@ struct BinaryDecoder : SymbolHandler<remap_symbols>::type { Cursor &decodeString(const Inserter &inserter, uint32_t meta) { uint64_t size = read_size(in, meta); - return inserter.insertString(in.getBytes(size)); + return inserter.insertString(in.read(size)); } Cursor &decodeData(const Inserter &inserter, uint32_t meta) { uint64_t size = read_size(in, meta); - return inserter.insertData(in.getBytes(size)); + return inserter.insertData(in.read(size)); } Cursor &decodeArray(const Inserter &inserter, uint32_t meta); @@ -175,7 +176,7 @@ struct BinaryDecoder : SymbolHandler<remap_symbols>::type { } void decodeValue(const Inserter &inserter) { - char byte = in.getByte(); + char byte = in.read(); Cursor &cursor = decodeValue(inserter, decode_type(byte), decode_meta(byte)); @@ -189,7 +190,7 @@ struct BinaryDecoder : SymbolHandler<remap_symbols>::type { hint_symbol_count(numSymbols); for (size_t i = 0; i < numSymbols; ++i) { uint64_t size = read_cmpr_ulong(in); - Memory image = in.getBytes(size); + Memory image = in.read(size); Symbol symbol = slime.insert(image); if (!add_symbol(symbol, i, in)) { return; @@ -227,16 +228,17 @@ BinaryDecoder<remap_symbols>::decodeObject(const Inserter &inserter, uint32_t me template <bool remap_symbols> size_t decode(const Memory &memory, Slime &slime, const Inserter &inserter) { - BufferedInput input(memory); + MemoryInput memory_input(memory); + InputReader input(memory_input); binary_format::BinaryDecoder<remap_symbols> decoder(input); decoder.decodeSymbolTable(slime); decoder.decodeValue(inserter); if (input.failed() && !remap_symbols) { slime.wrap("partial_result"); - slime.get().setData("offending_input", input.getOffending()); - slime.get().setString("error_message", input.getErrorMessage()); + slime.get().setLong("offending_offset", input.get_offset()); + slime.get().setString("error_message", input.get_error_message()); } - return input.getConsumed().size; + return input.failed() ? 0 : input.get_offset(); } } // namespace vespalib::slime::binary_format @@ -244,7 +246,8 @@ size_t decode(const Memory &memory, Slime &slime, const Inserter &inserter) { void BinaryFormat::encode(const Slime &slime, Output &output) { - BufferedOutput out(output); + size_t chunk_size = 8000; + OutputWriter out(output, chunk_size); binary_format::BinaryEncoder encoder(out); encoder.encodeSymbolTable(slime); encoder.encodeValue(slime.get()); diff --git a/vespalib/src/vespa/vespalib/data/slime/binary_format.h b/vespalib/src/vespa/vespalib/data/slime/binary_format.h index a49dc55a684..df6e32a0b22 100644 --- a/vespalib/src/vespa/vespalib/data/slime/binary_format.h +++ b/vespalib/src/vespa/vespalib/data/slime/binary_format.h @@ -2,10 +2,11 @@ #pragma once -#include "output.h" +#include <string> +#include <vespa/vespalib/data/output.h> #include "type.h" -#include "buffered_input.h" -#include "buffered_output.h" +#include <vespa/vespalib/data/input_reader.h> +#include <vespa/vespalib/data/output_writer.h> #include "inserter.h" namespace vespalib { @@ -70,26 +71,26 @@ inline uint32_t encode_cmpr_ulong(char *out, return (pos - out); } -inline void write_cmpr_ulong(BufferedOutput &out, +inline void write_cmpr_ulong(OutputWriter &out, uint64_t value) { out.commit(encode_cmpr_ulong(out.reserve(10), value)); } -inline uint64_t read_cmpr_ulong(BufferedInput &in) +inline uint64_t read_cmpr_ulong(InputReader &in) { - uint64_t next = in.getByte(); + uint64_t next = in.read(); uint64_t value = (next & 0x7f); int shift = 7; while ((next & 0x80) != 0) { - next = in.getByte(); + next = in.read(); value |= ((next & 0x7f) << shift); shift += 7; } return value; } -inline void write_type_and_size(BufferedOutput &out, +inline void write_type_and_size(OutputWriter &out, uint32_t type, uint64_t size) { char *start = out.reserve(11); // max size @@ -103,13 +104,13 @@ inline void write_type_and_size(BufferedOutput &out, out.commit(pos - start); } -inline uint64_t read_size(BufferedInput &in, uint32_t meta) +inline uint64_t read_size(InputReader &in, uint32_t meta) { return (meta == 0) ? read_cmpr_ulong(in) : (meta - 1); } template <bool top> -inline void write_type_and_bytes(BufferedOutput &out, +inline void write_type_and_bytes(OutputWriter &out, uint32_t type, uint64_t bits) { char *start = out.reserve(9); // max size @@ -128,13 +129,13 @@ inline void write_type_and_bytes(BufferedOutput &out, } template <bool top> -inline uint64_t read_bytes(BufferedInput &in, +inline uint64_t read_bytes(InputReader &in, uint32_t bytes) { uint64_t value = 0; int shift = top ? 56 : 0; for (uint32_t i = 0; i < bytes; ++i) { - uint64_t byte = in.getByte(); + uint64_t byte = in.read(); value |= ((byte & 0xff) << shift); if (top) { shift -= 8; diff --git a/vespalib/src/vespa/vespalib/data/slime/buffered_input.cpp b/vespalib/src/vespa/vespalib/data/slime/buffered_input.cpp deleted file mode 100644 index 279f3de9f7e..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/buffered_input.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "buffered_input.h" - -namespace vespalib { -namespace slime { - -} // namespace vespalib::slime -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/buffered_input.h b/vespalib/src/vespa/vespalib/data/slime/buffered_input.h deleted file mode 100644 index 5c7f8e1c86f..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/buffered_input.h +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "memory.h" -#include <assert.h> - -namespace vespalib { -namespace slime { - -/** - * A simple class tracking the consumption of input data. Also keeps - * track of data underflow. - **/ -class BufferedInput -{ -private: - const char *_begin; // start of input - const char *_pos; // current read position - const char *_end; // end of input - const char *_fail; // read position at failure - std::string _msg; // error message of failure - -public: - explicit BufferedInput(const Memory &memory) - : _begin(memory.data), _pos(_begin), _end(_pos + memory.size), - _fail(0), _msg() {} - - bool eof() const { return (_pos == _end); } - - bool failed() const { return (_fail != 0); } - - const std::string &getErrorMessage() const { return _msg; } - - Memory getConsumed() const { - if (_fail != 0) { - return Memory(); - } - return Memory(_begin, (_pos - _begin)); - } - - Memory getOffending() const { - if (_fail == 0) { - return Memory(); - } - return Memory(_begin, (_fail - _begin)); - } - - void fail(const std::string &msg) { - if (_fail == 0) { - _fail = _pos; - _msg = msg; - _pos = _end; - } - } - - char getByte() { - if (_pos == _end) { - fail("input buffer underflow"); - return 0; - } - return *_pos++; - } - - Memory getBytes(size_t n) { - if ((_pos + n) > _end) { - assert(_fail == 0 || _pos == _end); - _pos = _end; - fail("input buffer underflow"); - return Memory(); - } - Memory ret(_pos, n); - _pos += n; - return ret; - } -}; - -} // namespace vespalib::slime -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/data/slime/buffered_output.cpp b/vespalib/src/vespa/vespalib/data/slime/buffered_output.cpp deleted file mode 100644 index 1f2d19737d9..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/buffered_output.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "buffered_output.h" -#include <stdarg.h> -#include <stdio.h> -#include <assert.h> - -namespace vespalib { -namespace slime { - -void -BufferedOutput::printf(const char *fmt, ...) -{ - int space = 256; - char *p = reserve(space); - int size; - va_list ap; - va_start(ap, fmt); - size = vsnprintf(p, space, fmt, ap); - va_end(ap); - assert(size >= 0); - if (size >= space) { - space = size + 1; - p = reserve(space); - va_start(ap, fmt); - size = vsnprintf(p, space, fmt, ap); - va_end(ap); - assert((size + 1) == space); - } - commit(size); -} - -} // namespace vespalib::slime -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/buffered_output.h b/vespalib/src/vespa/vespalib/data/slime/buffered_output.h deleted file mode 100644 index c5c9ab5d17d..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/buffered_output.h +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "output.h" -#include <string.h> -#include <algorithm> - -namespace vespalib { -namespace slime { - -/** - * Thin layer on top of the Output interface that supplies us with an - * inlined API for efficient buffer handling. - **/ -class BufferedOutput -{ -private: - enum { CHUNK_SIZE = 8000 }; - - Output &_output; - char *_start; - char *_pos; - char *_end; - -public: - BufferedOutput(Output &out) - : _output(out), _start(0), _pos(0), _end(0) {} - - char *reserve(size_t bytes) { - if (__builtin_expect((size_t)(_end - _pos) < bytes, false)) { - size_t wantBytes = std::max(size_t(CHUNK_SIZE), bytes); - _start = _output.exchange(_start, _pos - _start, wantBytes); - _pos = _start; - _end = _start + wantBytes; - } - return _pos; - } - - void commit(size_t bytes) { - _pos += bytes; - } - - void writeByte(char value) { - reserve(1)[0] = value; - commit(1); - } - - void writeBytes(const char *data, size_t size) { - char *p = reserve(size); - memcpy(p, data, size); - commit(size); - } - - void printf(const char *fmt, ...) -#ifdef __GNUC__ - // Add printf format checks with gcc - __attribute__ ((format (printf,2,3))) -#endif - ; - - ~BufferedOutput() { - if (_pos != _start) { - _output.exchange(_start, _pos - _start, 0); - } - } -}; - -} // namespace vespalib::slime -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/data/slime/convenience.h b/vespalib/src/vespa/vespalib/data/slime/convenience.h index 3d4e1947ab9..445f8c8c7d5 100644 --- a/vespalib/src/vespa/vespalib/data/slime/convenience.h +++ b/vespalib/src/vespa/vespalib/data/slime/convenience.h @@ -11,10 +11,10 @@ namespace vespalib { class Slime; +struct Memory; namespace slime { -struct Memory; class Type; class Symbol; @@ -23,7 +23,7 @@ namespace convenience { using ::vespalib::Slime; using ::vespalib::slime::Symbol; -using ::vespalib::slime::Memory; +using ::vespalib::Memory; using ::vespalib::slime::Cursor; using ::vespalib::slime::Inspector; using ::vespalib::slime::Type; diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h index 76f6120ba55..18b2e900e39 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h @@ -3,7 +3,7 @@ #pragma once #include "type.h" -#include "memory.h" +#include <vespa/vespalib/data/memory.h> #include "symbol.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/slime/inspector.h b/vespalib/src/vespa/vespalib/data/slime/inspector.h index 5f60222b052..228e14a981b 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inspector.h +++ b/vespalib/src/vespa/vespalib/data/slime/inspector.h @@ -3,7 +3,7 @@ #pragma once #include "type.h" -#include "memory.h" +#include <vespa/vespalib/data/memory.h> #include "symbol.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp index 04efc182637..e8ebcfb9e10 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp @@ -7,6 +7,7 @@ #include <errno.h> #include <cmath> #include <sstream> +#include <vespa/vespalib/data/memory_input.h> namespace vespalib { namespace slime { @@ -17,22 +18,22 @@ template <bool COMPACT> struct JsonEncoder : public ArrayTraverser, public ObjectTraverser { - BufferedOutput &out; + OutputWriter &out; int level; bool head; - JsonEncoder(BufferedOutput &out_in) + JsonEncoder(OutputWriter &out_in) : out(out_in), level(0), head(true) {} void openScope(char c) { - out.writeByte(c); + out.write(c); ++level; head = true; } void separate(bool useComma) { if (!head && useComma) { - out.writeByte(','); + out.write(','); } else { head = false; } @@ -44,17 +45,17 @@ struct JsonEncoder : public ArrayTraverser, void closeScope(char c) { --level; separate(false); - out.writeByte(c); + out.write(c); } void encodeNIX() { - out.writeBytes("null", 4); + out.write("null", 4); } void encodeBOOL(bool value) { if (value) { - out.writeBytes("true", 4); + out.write("true", 4); } else { - out.writeBytes("false", 5); + out.write("false", 5); } } void encodeLONG(int64_t value) { @@ -62,7 +63,7 @@ struct JsonEncoder : public ArrayTraverser, } void encodeDOUBLE(double value) { if (std::isnan(value) || std::isinf(value)) { - out.writeBytes("null", 4); + out.write("null", 4); } else { out.printf("%g", value); } @@ -138,11 +139,11 @@ struct JsonEncoder : public ArrayTraverser, virtual void entry(size_t idx, const Inspector &inspector); virtual void field(const Memory &symbol_name, const Inspector &inspector); - static void encode(const Inspector &inspector, BufferedOutput &out) { + static void encode(const Inspector &inspector, OutputWriter &out) { JsonEncoder<COMPACT> encoder(out); encoder.encodeValue(inspector); if (!COMPACT) { - out.writeByte('\n'); + out.write('\n'); } } }; @@ -162,9 +163,9 @@ JsonEncoder<COMPACT>::field(const Memory &symbol_name, const Inspector &inspecto separate(true); encodeSTRING(symbol_name); if (COMPACT) { - out.writeByte(':'); + out.write(':'); } else { - out.writeBytes(": ", 2); + out.write(": ", 2); } encodeValue(inspector); } @@ -172,16 +173,16 @@ JsonEncoder<COMPACT>::field(const Memory &symbol_name, const Inspector &inspecto //----------------------------------------------------------------------------- struct JsonDecoder { - BufferedInput ∈ + InputReader ∈ char c; vespalib::string key; vespalib::string value; - JsonDecoder(BufferedInput &input) : in(input), c(in.getByte()), key(), value() {} + JsonDecoder(InputReader &input) : in(input), c(in.read()), key(), value() {} void next() { - if (!in.eof()) { - c = in.getByte(); + if (in.obtain() > 0) { + c = in.read(); } else { c = 0; } @@ -467,7 +468,8 @@ insertNumber(Inserter &inserter, bool isLong, const vespalib::string & value, ch void JsonFormat::encode(const Inspector &inspector, Output &output, bool compact) { - BufferedOutput out(output); + size_t chunk_size = 8000; + OutputWriter out(output, chunk_size); if (compact) { JsonEncoder<true>::encode(inspector, out); } else { @@ -484,15 +486,16 @@ JsonFormat::encode(const Slime &slime, Output &output, bool compact) size_t JsonFormat::decode(const Memory &memory, Slime &slime) { - BufferedInput input(memory); + MemoryInput memory_input(memory); + InputReader input(memory_input); JsonDecoder decoder(input); decoder.decodeValue(slime); if (input.failed()) { slime.wrap("partial_result"); - slime.get().setString("offending_input", input.getOffending()); - slime.get().setString("error_message", input.getErrorMessage()); + slime.get().setLong("offending_offset", input.get_offset()); + slime.get().setString("error_message", input.get_error_message()); } - return input.getConsumed().size; + return input.failed() ? 0 : input.get_offset(); } } // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.h b/vespalib/src/vespa/vespalib/data/slime/json_format.h index bb23e168bd8..e334cd55546 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.h +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.h @@ -2,10 +2,10 @@ #pragma once -#include "output.h" +#include <vespa/vespalib/data/output.h> #include "type.h" -#include "buffered_input.h" -#include "buffered_output.h" +#include <vespa/vespalib/data/input_reader.h> +#include <vespa/vespalib/data/output_writer.h> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/data/slime/memory.cpp b/vespalib/src/vespa/vespalib/data/slime/memory.cpp deleted file mode 100644 index 991f28a5865..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/memory.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memory.h" -#include "stored_memory.h" - -namespace vespalib { -namespace slime { - -Memory::Memory(const StoredMemory &sm) - : data(sm._data), - size(sm._size) -{ -} - -vespalib::string -Memory::make_string() const -{ - return vespalib::string(data, size); -} - -} // namespace vespalib::slime -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/memory.h b/vespalib/src/vespa/vespalib/data/slime/memory.h deleted file mode 100644 index f89c6aa0730..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/memory.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/stllike/hash_fun.h> - -namespace vespalib { -namespace slime { - -class StoredMemory; - -/** - * Simple wrapper used to reference a region of memory. - **/ -struct Memory -{ - const char *data; - size_t size; - - Memory() : data(0), size(0) {} - Memory(const char *d, size_t s) : data(d), size(s) {} - Memory(const char *str) : data(str), size(strlen(str)) {} - Memory(const std::string &str) - : data(str.data()), size(str.size()) {} - Memory(const vespalib::string &str) - : data(str.data()), size(str.size()) {} - Memory(const stringref &str) - : data(str.data()), size(str.size()) {} - explicit Memory(const StoredMemory &sm); - vespalib::string make_string() const; - bool operator == (const Memory & rhs) const { - return (size == rhs.size) && - ( (data == rhs.data) || - (memcmp(data, rhs.data, size) == 0) ); - } - size_t hash() const { - return vespalib::hashValue(data, size); - } -}; - -} // namespace vespalib::slime -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/data/slime/output.cpp b/vespalib/src/vespa/vespalib/data/slime/output.cpp deleted file mode 100644 index e7353dbe0ed..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/output.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "output.h" - -namespace vespalib { -namespace slime { - -} // namespace vespalib::slime -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/output.h b/vespalib/src/vespa/vespalib/data/slime/output.h deleted file mode 100644 index 26487faa85d..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/output.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <stddef.h> - -namespace vespalib { -namespace slime { - -/** - * Interface used to interact with an external output buffer. - **/ -struct Output { - virtual char *exchange(char *p, size_t commit, size_t reserve) = 0; - virtual ~Output() {} -}; - -} // namespace vespalib::slime -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/data/slime/simple_buffer.cpp b/vespalib/src/vespa/vespalib/data/slime/simple_buffer.cpp deleted file mode 100644 index cbe6a579839..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/simple_buffer.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_buffer.h" - -namespace vespalib { -namespace slime { - -} // namespace vespalib::slime -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/simple_buffer.h b/vespalib/src/vespa/vespalib/data/slime/simple_buffer.h deleted file mode 100644 index ccee73a9125..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/simple_buffer.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "output.h" -#include "memory.h" -#include <vector> -#include <assert.h> - -namespace vespalib { -namespace slime { - -/** - * Simple output buffer implementation. - **/ -class SimpleBuffer : public Output -{ -private: - std::vector<char> _data; - size_t _used; - -public: - SimpleBuffer() : _data(), _used(0) {} - virtual char *exchange(char *p, size_t commit, size_t reserve) { - (void) p; - assert(p == &_data[_used] || commit == 0); - assert((_data.size() - _used) >= commit); - _used += commit; - _data.resize(_used + reserve, char(0x55)); - return &_data[_used]; - } - SimpleBuffer &add(char byte) { - assert(_data.size() == _used); - _data.push_back(byte); - ++_used; - return *this; - } - Memory get() const { - return Memory(&_data[0], _used); - } -}; - -} // namespace vespalib::slime -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index f938f6ca1c5..2ddefb061e6 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -7,8 +7,8 @@ #include "basic_value.h" #include "basic_value_factory.h" #include "binary_format.h" -#include "buffered_input.h" -#include "buffered_output.h" +#include <vespa/vespalib/data/input_reader.h> +#include <vespa/vespalib/data/output_writer.h> #include "convenience.h" #include "cursor.h" #include "empty_value_factory.h" @@ -16,17 +16,16 @@ #include "inserter.h" #include "inspector.h" #include "json_format.h" -#include "memory.h" +#include <vespa/vespalib/data/memory.h> #include "named_symbol_inserter.h" #include "named_symbol_lookup.h" #include "nix_value.h" #include "object_traverser.h" #include "object_value.h" -#include "output.h" +#include <vespa/vespalib/data/output.h> #include "resolved_symbol.h" #include "root_value.h" -#include "simple_buffer.h" -#include "stored_memory.h" +#include <vespa/vespalib/data/simple_buffer.h> #include "symbol.h" #include "symbol_inserter.h" #include "symbol_lookup.h" @@ -57,7 +56,6 @@ private: typedef slime::RootValue RootValue; typedef slime::Cursor Cursor; typedef slime::Inspector Inspector; - typedef slime::Memory Memory; SymbolTable::UP _names; Stash::UP _stash; diff --git a/vespalib/src/vespa/vespalib/data/slime/stored_memory.cpp b/vespalib/src/vespa/vespalib/data/slime/stored_memory.cpp deleted file mode 100644 index 86fcd929539..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/stored_memory.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "stored_memory.h" -#include "memory.h" -#include <stdlib.h> - -namespace vespalib { -namespace slime { - -} // namespace vespalib::slime -} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/data/slime/stored_memory.h b/vespalib/src/vespa/vespalib/data/slime/stored_memory.h deleted file mode 100644 index 7ca4967b220..00000000000 --- a/vespalib/src/vespa/vespalib/data/slime/stored_memory.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <stddef.h> -#include <vespa/vespalib/data/slime/memory.h> - -namespace vespalib { -namespace slime { - -/** - * Simple class used to store a region of memory. - **/ -class StoredMemory -{ -private: - friend struct Memory; - char *_data; - size_t _size; - - StoredMemory(const StoredMemory &); - StoredMemory &operator=(const StoredMemory &); -public: - explicit StoredMemory(const Memory &mem) : - _data(0), - _size(mem.size) - { - if (__builtin_expect(_size > 0, true)) { - _data = static_cast<char*>(malloc(_size)); - memcpy(_data, mem.data, _size); - } - } - - ~StoredMemory() { - free(_data); - } -}; - -} // namespace vespalib::slime -} // namespace vespalib - diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h index e4cafa2b9d6..d9f6d4f3773 100644 --- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h +++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h @@ -3,7 +3,7 @@ #pragma once #include "symbol.h" -#include "memory.h" +#include <vespa/vespalib/data/memory.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/data/memorydatastore.h> @@ -18,7 +18,7 @@ class SymbolTable private: struct hasher { size_t operator () (const Memory & lcm) const { - return lcm.hash(); + return hashValue(lcm.data, lcm.size); } }; using SymbolMap = hash_map<Memory, Symbol, hasher>; diff --git a/vespalib/src/vespa/vespalib/data/slime/value.cpp b/vespalib/src/vespa/vespalib/data/slime/value.cpp index 1b0b1fc01d4..3bc08c2bb3e 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/value.cpp @@ -5,7 +5,7 @@ #include "resolved_symbol.h" #include "empty_value_factory.h" #include "basic_value_factory.h" -#include "simple_buffer.h" +#include <vespa/vespalib/data/simple_buffer.h> #include "json_format.h" namespace vespalib { @@ -75,7 +75,7 @@ Value::traverse(ObjectTraverser &) const {} vespalib::string Value::toString() const { - slime::SimpleBuffer buf; + SimpleBuffer buf; slime::JsonFormat::encode(*this, buf, false); return buf.get().make_string(); } diff --git a/vespalib/src/vespa/vespalib/trace/slime_trace_serializer.h b/vespalib/src/vespa/vespalib/trace/slime_trace_serializer.h index 3b50f0878bf..802f0ab5fe9 100644 --- a/vespalib/src/vespa/vespalib/trace/slime_trace_serializer.h +++ b/vespalib/src/vespa/vespalib/trace/slime_trace_serializer.h @@ -18,9 +18,9 @@ class SlimeTraceSerializer : public TraceVisitor public: SlimeTraceSerializer(slime::Cursor & cursor); void visit(const TraceNode & node); - static const slime::Memory TIMESTAMP; - static const slime::Memory PAYLOAD; - static const slime::Memory CHILDREN; + static const Memory TIMESTAMP; + static const Memory PAYLOAD; + static const Memory CHILDREN; private: void addTimestamp(slime::Cursor & current, const TraceNode & node); void addPayload(slime::Cursor & current, const TraceNode & node); |