input reader/output writer

7 files changed, 227 insertions, 2 deletions

diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt
index 0d6c1b23512..24de559ce0b 100644
--- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt
@@ -2,9 +2,11 @@
 vespa_add_library(vespalib_vespalib_data OBJECT
     SOURCES
     input.cpp
+    input_reader.cpp
     memory.cpp
     memorydatastore.cpp
     output.cpp
+    output_writer.cpp
     simple_buffer.cpp
     writable_memory.cpp
     DEPENDS
diff --git a/vespalib/src/vespa/vespalib/data/input_reader.cpp b/vespalib/src/vespa/vespalib/data/input_reader.cpp
new file mode 100644
index 00000000000..de93ac70807
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/input_reader.cpp
@@ -0,0 +1,55 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "input_reader.h"
+#include "input.h"
+
+namespace vespalib {
+
+size_t
+InputReader::obtain_slow()
+{
+    if (!failed()) {
+        _input.evict(_pos);
+        _chunk_offset += _pos;
+        _data = _input.obtain();
+        _pos = 0;
+        if (_data.size == 0) {
+            fail("input underflow");
+        }
+    }
+    return size();
+}
+
+Memory
+InputReader::read_slow(size_t bytes)
+{
+    _space.clear();
+    while ((_space.size() < bytes) && (obtain() > 0)) {
+        size_t copy_now = std::min(size(), (bytes - _space.size()));
+        _space.insert(_space.end(), data(), data() + copy_now);
+        _pos += copy_now;
+    }
+    if (_space.size() == bytes) {
+        return Memory(&_space[0], _space.size());
+    }
+    return Memory();
+}
+
+InputReader::~InputReader()
+{
+    _input.evict(_pos);
+}
+
+void
+InputReader::fail(const vespalib::string &msg) {
+    if (!failed()) {
+        _error = msg;
+        _input.evict(_pos);
+        _chunk_offset += _pos;
+        _data = Memory();
+        _pos = 0;
+    }
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/input_reader.h b/vespalib/src/vespa/vespalib/data/input_reader.h
new file mode 100644
index 00000000000..fdf8001a7bc
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/input_reader.h
@@ -0,0 +1,69 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "memory.h"
+#include <assert.h>
+
+namespace vespalib {
+
+class Input;
+
+/**
+ * A utility wrapper for the Input interface that supplies us with an
+ * inlined API for efficient buffer handling. Note that reading past
+ * the end of the data is considered an error and will tag the reader
+ * as failed with buffer underflow.
+ **/
+class InputReader
+{
+private:
+    Input            &_input;
+    Memory            _data;
+    size_t            _pos;
+    size_t            _chunk_offset;
+    vespalib::string  _error;
+    std::vector<char> _space;
+
+    const char *data() const { return (_data.data + _pos); }
+    size_t size() const { return (_data.size - _pos); }
+
+    size_t obtain_slow();
+    Memory read_slow(size_t bytes);
+
+public:
+    explicit InputReader(Input &input)
+        : _input(input), _data(), _pos(0), _chunk_offset(0), _error(), _space() {}
+    ~InputReader();
+
+    bool failed() const { return !_error.empty(); }
+    const vespalib::string &get_error_message() const { return _error; }
+    size_t get_offset() const { return (_chunk_offset + _pos); }
+
+    void fail(const vespalib::string &msg);
+
+    size_t obtain() {
+        if (__builtin_expect(_pos < _data.size, true)) {                    
+            return size();
+        }
+        return obtain_slow();
+    }
+
+    char read() {
+        if (__builtin_expect(obtain() > 0, true)) {
+            return _data.data[_pos++];
+        }
+        return 0;
+    }
+
+    Memory read(size_t bytes) {
+        if (__builtin_expect(obtain() >= bytes, true)) {
+            Memory ret(data(), bytes);
+            _pos += bytes;
+            return ret;
+        }
+        return read_slow(bytes);
+    }
+};
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/memory.cpp b/vespalib/src/vespa/vespalib/data/memory.cpp
index 3f76b86ebae..71c8c6c8ced 100644
--- a/vespalib/src/vespa/vespalib/data/memory.cpp
+++ b/vespalib/src/vespa/vespalib/data/memory.cpp
@@ -1,5 +1,6 @@
 // Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
+#include <vespa/fastos/fastos.h>
 #include "memory.h"
 #include <vespa/vespalib/util/stringfmt.h>
 
diff --git a/vespalib/src/vespa/vespalib/data/memory.h b/vespalib/src/vespa/vespalib/data/memory.h
index a8a4760754a..af4cf6d856b 100644
--- a/vespalib/src/vespa/vespalib/data/memory.h
+++ b/vespalib/src/vespa/vespalib/data/memory.h
@@ -18,8 +18,6 @@ struct Memory
     Memory() : data(nullptr), size(0) {}
     Memory(const char *d, size_t s) : data(d), size(s) {}
     Memory(const char *str) : data(str), size(strlen(str)) {}
-    Memory(const std::string &str)
-        : data(str.data()), size(str.size()) {}
     Memory(const vespalib::string &str)
         : data(str.data()), size(str.size()) {}
     Memory(const vespalib::stringref &str_ref)
diff --git a/vespalib/src/vespa/vespalib/data/output_writer.cpp b/vespalib/src/vespa/vespalib/data/output_writer.cpp
new file mode 100644
index 00000000000..5505ee4e6e0
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/output_writer.cpp
@@ -0,0 +1,41 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "output_writer.h"
+#include <stdarg.h>
+#include <stdio.h>
+#include <assert.h>
+
+namespace vespalib {
+
+char *
+OutputWriter::reserve_slow(size_t bytes)
+{
+    _data = _output.commit(_pos).reserve(std::max(_chunk_size, bytes));
+    _pos = 0;
+    return _data.data;
+}
+
+void
+OutputWriter::printf(const char *fmt, ...)
+{
+    char *p = reserve(256);
+    int space = (_data.size - _pos);
+    int size;
+    va_list ap;
+    va_start(ap, fmt);
+    size = vsnprintf(p, space, fmt, ap);
+    va_end(ap);
+    assert(size >= 0);
+    if (size >= space) {
+        space = size + 1;
+        p = reserve(space);
+        va_start(ap, fmt);
+        size = vsnprintf(p, space, fmt, ap);
+        va_end(ap);
+        assert((size + 1) == space);
+    }
+    commit(size);
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/output_writer.h b/vespalib/src/vespa/vespalib/data/output_writer.h
new file mode 100644
index 00000000000..0a263f03280
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/output_writer.h
@@ -0,0 +1,59 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "output.h"
+#include <string.h>
+#include <algorithm>
+
+namespace vespalib {
+
+/**
+ * Thin layer on top of the Output interface that supplies us with an
+ * inlined API for efficient buffer handling.
+ **/
+class OutputWriter
+{
+private:
+    Output        &_output;
+    WritableMemory _data;
+    size_t         _pos;
+    size_t         _chunk_size;
+
+    char *reserve_slow(size_t bytes);
+
+public:
+    OutputWriter(Output &output, size_t chunk_size)
+        : _output(output), _data(), _pos(0), _chunk_size(chunk_size) {}
+    ~OutputWriter() { _output.commit(_pos); }
+
+    char *reserve(size_t bytes) {
+        if (__builtin_expect((_pos + bytes) <= _data.size, true)) {
+            return (_data.data + _pos);
+        }
+        return reserve_slow(bytes);
+    }
+
+    void commit(size_t bytes) {
+        _pos += bytes;
+    }
+
+    void write(char value) {
+        reserve(1)[0] = value;
+        commit(1);
+    }
+
+    void write(const char *data, size_t size) {
+        memcpy(reserve(size), data, size);
+        commit(size);
+    }
+
+    void printf(const char *fmt, ...)
+#ifdef __GNUC__
+        // Add printf format checks with gcc
+        __attribute__ ((format (printf,2,3)))
+#endif
+        ;
+};
+
+} // namespace vespalib