low-level data flow building blocks

14 files changed, 356 insertions, 0 deletions

diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index b6482ec5b1b..c5e568c103b 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -20,6 +20,7 @@ vespa_define_module(
     src/tests/closure
     src/tests/component
     src/tests/compress
+    src/tests/data/simple_buffer
     src/tests/delegatelist
     src/tests/dotproduct
     src/tests/dual_merge_director
diff --git a/vespalib/src/tests/data/simple_buffer/CMakeLists.txt b/vespalib/src/tests/data/simple_buffer/CMakeLists.txt
new file mode 100644
index 00000000000..151bbc29108
--- /dev/null
+++ b/vespalib/src/tests/data/simple_buffer/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_simple_buffer_test_app TEST
+    SOURCES
+    simple_buffer_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_simple_buffer_test_app COMMAND vespalib_simple_buffer_test_app)
diff --git a/vespalib/src/tests/data/simple_buffer/simple_buffer_test.cpp b/vespalib/src/tests/data/simple_buffer/simple_buffer_test.cpp
new file mode 100644
index 00000000000..41408797b49
--- /dev/null
+++ b/vespalib/src/tests/data/simple_buffer/simple_buffer_test.cpp
@@ -0,0 +1,70 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/data/simple_buffer.h>
+
+using namespace vespalib;
+
+void checkMemory(const string &expect, const Memory &mem) {
+    EXPECT_EQUAL(expect, string(mem.data, mem.size));
+}
+
+void checkBuffer(const string &expect, const SimpleBuffer &buf) {
+    TEST_DO(checkMemory(expect, buf.get()));
+}
+
+TEST("simple buffer") {
+    SimpleBuffer buf;
+    TEST_DO(checkBuffer("", buf));
+    { // read from empty buffer
+        EXPECT_EQUAL(0u, buf.obtain().size);
+    }
+    { // write to buffer
+        WritableMemory mem = buf.reserve(10);
+        TEST_DO(checkBuffer("", buf));
+        EXPECT_EQUAL(10u, mem.size);
+        mem.data[0] = 'a';
+        mem.data[1] = 'b';
+        mem.data[2] = 'c';
+        EXPECT_EQUAL(&buf, &buf.commit(3));
+        mem = buf.reserve(0);
+        TEST_DO(checkBuffer("abc", buf));
+        EXPECT_EQUAL(0u, mem.size);
+    }
+    { // read without evicting last byte
+        Memory mem = buf.obtain();
+        TEST_DO(checkBuffer("abc", buf));
+        TEST_DO(checkMemory("abc", mem));
+        EXPECT_EQUAL(&buf, &buf.evict(2));
+        mem = buf.obtain();
+        TEST_DO(checkBuffer("c", buf));
+        TEST_DO(checkMemory("c", mem));
+        mem = buf.obtain();
+        TEST_DO(checkBuffer("c", buf));
+        TEST_DO(checkMemory("c", mem));
+    }
+    { // write more to buffer
+        WritableMemory mem = buf.reserve(10);
+        EXPECT_EQUAL(10u, mem.size);
+        TEST_DO(checkBuffer("c", buf));
+        mem.data[0] = 'd';
+        EXPECT_EQUAL(&buf, &buf.commit(1));
+        mem = buf.reserve(5);
+        TEST_DO(checkBuffer("cd", buf));
+        EXPECT_EQUAL(5u, mem.size);
+    }
+    { // read until end
+        Memory mem = buf.obtain();
+        TEST_DO(checkBuffer("cd", buf));
+        TEST_DO(checkMemory("cd", mem));
+        EXPECT_EQUAL(&buf, &buf.evict(1));
+        mem = buf.obtain();
+        TEST_DO(checkBuffer("d", buf));
+        TEST_DO(checkMemory("d", mem));
+        EXPECT_EQUAL(&buf, &buf.evict(1));
+        mem = buf.obtain();
+        TEST_DO(checkBuffer("", buf));
+        TEST_DO(checkMemory("", mem));
+    }
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt
index 6752d1c99cf..0d6c1b23512 100644
--- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt
@@ -1,6 +1,11 @@
 # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(vespalib_vespalib_data OBJECT
     SOURCES
+    input.cpp
+    memory.cpp
     memorydatastore.cpp
+    output.cpp
+    simple_buffer.cpp
+    writable_memory.cpp
     DEPENDS
 )
diff --git a/vespalib/src/vespa/vespalib/data/input.cpp b/vespalib/src/vespa/vespalib/data/input.cpp
new file mode 100644
index 00000000000..d0dc70eb107
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/input.cpp
@@ -0,0 +1,12 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "input.h"
+
+namespace vespalib {
+
+Input::~Input()
+{
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/input.h b/vespalib/src/vespa/vespalib/data/input.h
new file mode 100644
index 00000000000..7e97d98e9e1
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/input.h
@@ -0,0 +1,35 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "memory.h"
+
+namespace vespalib {
+
+/**
+ * Interface used to abstract a source of input data. Note that the
+ * input data itself is owned by the object implementing this
+ * interface.
+ **/
+struct Input
+{
+    /**
+     * Obtain more input data.
+     *
+     * @return the obtained input data
+     **/
+    virtual Memory obtain() = 0;
+
+    /**
+     * Evict processed input data. Never evict more data than you have
+     * obtained.
+     *
+     * @return this object, for chaining
+     * @param bytes the number of bytes to evict
+     **/
+    virtual Input &evict(size_t bytes) = 0;
+
+    virtual ~Input();
+};
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/memory.cpp b/vespalib/src/vespa/vespalib/data/memory.cpp
new file mode 100644
index 00000000000..3f76b86ebae
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/memory.cpp
@@ -0,0 +1,31 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "memory.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+namespace vespalib {
+
+vespalib::string
+Memory::make_string() const
+{
+    return vespalib::string(data, size);
+}
+
+std::ostream &operator<<(std::ostream &os, const Memory &memory) {
+    uint32_t written = 0;
+    uint32_t hexCount = 25;
+    os << "size: " << memory.size << "(bytes)" << std::endl;
+    for (size_t i = 0; i < memory.size; ++i, ++written) {
+        if (written > hexCount) {
+            os << std::endl;
+            written = 0;
+        }
+        os << vespalib::make_string("0x%02x ", memory.data[i] & 0xff);
+    }
+    if (written > 0) {
+        os << std::endl;
+    }
+    return os;
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/memory.h b/vespalib/src/vespa/vespalib/data/memory.h
new file mode 100644
index 00000000000..a8a4760754a
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/memory.h
@@ -0,0 +1,37 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <ostream>
+
+namespace vespalib {
+
+/**
+ * Simple wrapper referencing a read-only region of memory.
+ **/
+struct Memory
+{
+    const char *data;
+    size_t      size;
+
+    Memory() : data(nullptr), size(0) {}
+    Memory(const char *d, size_t s) : data(d), size(s) {}
+    Memory(const char *str) : data(str), size(strlen(str)) {}
+    Memory(const std::string &str)
+        : data(str.data()), size(str.size()) {}
+    Memory(const vespalib::string &str)
+        : data(str.data()), size(str.size()) {}
+    Memory(const vespalib::stringref &str_ref)
+        : data(str_ref.data()), size(str_ref.size()) {}
+    vespalib::string make_string() const;
+    bool operator == (const Memory &rhs) const {
+        return ((size == rhs.size) &&
+                ((data == rhs.data) ||
+                 (memcmp(data, rhs.data, size) == 0)));
+    }
+};
+
+std::ostream &operator<<(std::ostream &os, const Memory &memory);
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/output.cpp b/vespalib/src/vespa/vespalib/data/output.cpp
new file mode 100644
index 00000000000..27489da4179
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/output.cpp
@@ -0,0 +1,12 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "output.h"
+
+namespace vespalib {
+
+Output::~Output()
+{
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/output.h b/vespalib/src/vespa/vespalib/data/output.h
new file mode 100644
index 00000000000..85b89305d14
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/output.h
@@ -0,0 +1,36 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "writable_memory.h"
+
+namespace vespalib {
+
+/**
+ * Interface used to abstract a destination for output data. Note that
+ * the output data itself is owned by the object implementing this
+ * interface.
+ **/
+struct Output
+{
+    /**
+     * Reserve space for more output data.
+     *
+     * @return the reserved output data
+     * @param bytes number of bytes to reserve
+     **/
+    virtual WritableMemory reserve(size_t bytes) = 0;
+
+    /**
+     * Commit produced output data. Never commit more data than you
+     * have reserved.
+     *
+     * @return this object, for chaining
+     * @param bytes number of bytes to commit
+     **/
+    virtual Output &commit(size_t bytes) = 0;
+
+    virtual ~Output();
+};
+
+} // namespace vbench
diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.cpp b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp
new file mode 100644
index 00000000000..b178bb9e569
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp
@@ -0,0 +1,44 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "simple_buffer.h"
+
+namespace vespalib {
+
+SimpleBuffer::SimpleBuffer()
+    : _data(),
+      _used(0)
+{
+}
+
+Memory
+SimpleBuffer::obtain()
+{
+    return Memory(&_data[0], _used);
+}
+
+Input &
+SimpleBuffer::evict(size_t bytes)
+{
+    assert(bytes <= _used);
+    _data.erase(_data.begin(), _data.begin() + bytes);
+    _used -= bytes;
+    return *this;
+}
+
+WritableMemory
+SimpleBuffer::reserve(size_t bytes)
+{
+    _data.resize(_used + bytes, char(0x55));
+    return WritableMemory(&_data[_used], bytes);
+}
+
+Output &
+SimpleBuffer::commit(size_t bytes)
+{
+    assert(bytes <= (_data.size() - _used));
+    _used += bytes;
+    return *this;
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.h b/vespalib/src/vespa/vespalib/data/simple_buffer.h
new file mode 100644
index 00000000000..5cb2a0f1ad1
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/simple_buffer.h
@@ -0,0 +1,39 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "input.h"
+#include "output.h"
+#include <ostream>
+
+namespace vespalib {
+
+/**
+ * Simple buffer class that implements the Input/Output
+ * interfaces. Requesting the memory region of this buffer or
+ * comparing buffers will only look at the data conceptually contained
+ * in the buffer, ignoring evicted data and reserved data not yet
+ * committed.
+ **/
+class SimpleBuffer : public Input,
+                     public Output
+{
+private:
+    std::vector<char> _data;
+    size_t            _used;
+
+public:
+    SimpleBuffer();
+    Memory obtain() override;
+    Input &evict(size_t bytes) override;
+    WritableMemory reserve(size_t bytes) override;
+    Output &commit(size_t bytes) override;
+    Memory get() const { return Memory(&_data[0], _used); }
+    bool operator==(const SimpleBuffer &rhs) const { return (get() == rhs.get()); }
+};
+
+std::ostream &operator<<(std::ostream &os, const SimpleBuffer &buf) {
+    return os << buf.get();
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/writable_memory.cpp b/vespalib/src/vespa/vespalib/data/writable_memory.cpp
new file mode 100644
index 00000000000..b94c54e13cd
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/writable_memory.cpp
@@ -0,0 +1,8 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "writable_memory.h"
+
+namespace vespalib {
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/writable_memory.h b/vespalib/src/vespa/vespalib/data/writable_memory.h
new file mode 100644
index 00000000000..c04d0140518
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/writable_memory.h
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace vespalib {
+
+/**
+ * Simple wrapper referencing a writable region of memory. This class
+ * does not have ownership of the referenced memory region.
+ **/
+struct WritableMemory {
+    char   *data;
+    size_t  size;
+    WritableMemory() : data(nullptr), size(0) {}
+    WritableMemory(char *d, size_t s) : data(d), size(s) {}
+};
+
+} // namespace vbench