input reader test

bonus: MemoryInput utility with test

10 files changed, 189 insertions, 4 deletions

diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index 59e3cd29c0b..d29b577401e 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -20,6 +20,8 @@ vespa_define_module(
     src/tests/closure
     src/tests/component
     src/tests/compress
+    src/tests/data/input_reader
+    src/tests/data/memory_input
     src/tests/data/output_writer
     src/tests/data/simple_buffer
     src/tests/delegatelist
diff --git a/vespalib/src/tests/data/input_reader/CMakeLists.txt b/vespalib/src/tests/data/input_reader/CMakeLists.txt
new file mode 100644
index 00000000000..21adcf86648
--- /dev/null
+++ b/vespalib/src/tests/data/input_reader/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_input_reader_test_app TEST
+    SOURCES
+    input_reader_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_input_reader_test_app COMMAND vespalib_input_reader_test_app)
diff --git a/vespalib/src/tests/data/input_reader/input_reader_test.cpp b/vespalib/src/tests/data/input_reader/input_reader_test.cpp
new file mode 100644
index 00000000000..e9741ffc033
--- /dev/null
+++ b/vespalib/src/tests/data/input_reader/input_reader_test.cpp
@@ -0,0 +1,102 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/data/memory_input.h>
+#include <vespa/vespalib/data/input_reader.h>
+#include <algorithm>
+
+using namespace vespalib;
+
+// make sure input is split into chunks
+struct ChunkedInput : Input {
+    Input &input;
+    ChunkedInput(Input &input_in) : input(input_in) {}
+    Memory obtain() override {
+        Memory memory = input.obtain();
+        memory.size = std::min(memory.size, size_t(3));
+        return memory;
+    }
+    Input &evict(size_t bytes) override {
+        input.evict(bytes);
+        return *this;
+    }
+};
+
+TEST("input reader smoke test") {
+    const char *data = "abc\n"
+                       "foo bar\n"
+                       "2 + 2 = 4\n";
+    MemoryInput memory_input(data);
+    ChunkedInput input(memory_input);
+    {
+        InputReader src(input);
+        EXPECT_EQUAL(src.get_offset(), 0u);
+        EXPECT_EQUAL(src.read(), 'a');
+        EXPECT_EQUAL(src.read(), 'b');
+        EXPECT_EQUAL(src.read(), 'c');
+        EXPECT_EQUAL(src.read(), '\n');
+        EXPECT_EQUAL(src.get_offset(), 4u);
+        EXPECT_EQUAL(src.obtain(), 2u);
+        EXPECT_EQUAL(src.read(8), Memory("foo bar\n"));
+        EXPECT_EQUAL(src.get_offset(), 12u);
+        EXPECT_EQUAL(src.obtain(), 3u);
+        EXPECT_EQUAL(src.get_offset(), 12u);
+        EXPECT_EQUAL(src.read(2), Memory("2 "));
+        EXPECT_EQUAL(src.get_offset(), 14u);
+        EXPECT_EQUAL(src.obtain(), 1u);
+        EXPECT_EQUAL(src.read(8), Memory("+ 2 = 4\n"));
+        EXPECT_TRUE(!src.failed());
+        EXPECT_EQUAL(src.get_offset(), strlen(data));
+        EXPECT_EQUAL(src.obtain(), 0u);
+        EXPECT_TRUE(src.failed());
+        EXPECT_EQUAL(src.read(5), Memory());
+        EXPECT_EQUAL(src.read(), '\0');
+        EXPECT_EQUAL(src.obtain(), 0u);
+        EXPECT_EQUAL(src.get_offset(), strlen(data));
+        EXPECT_EQUAL(src.get_error_message(), vespalib::string("input underflow"));
+    }
+}
+
+TEST("require that not reading everything leaves the input in appropriate state") {
+    const char *data = "1234567890";
+    MemoryInput input(data);
+    {
+        InputReader src(input);
+        EXPECT_EQUAL(src.obtain(), 10u);
+        EXPECT_EQUAL(src.read(5), Memory("12345"));
+        EXPECT_EQUAL(input.obtain(), Memory("1234567890"));
+    }
+    EXPECT_EQUAL(input.obtain(), Memory("67890"));
+}
+
+TEST("require that input can be explicitly failed with custom message") {
+    const char *data = "1234567890";
+    MemoryInput input(data);
+    {
+        InputReader src(input);
+        EXPECT_EQUAL(src.read(5), Memory("12345"));
+        EXPECT_TRUE(!src.failed());
+        src.fail("custom");
+        EXPECT_TRUE(src.failed());
+        EXPECT_EQUAL(src.read(), '\0');
+        EXPECT_EQUAL(src.read(5), Memory());
+        EXPECT_EQUAL(src.obtain(), 0u);
+        src.fail("ignored");
+        EXPECT_EQUAL(src.get_error_message(), vespalib::string("custom"));
+        EXPECT_EQUAL(src.get_offset(), 5u);
+    }
+}
+
+TEST("require that reading a byte sequence crossing the end of input fails") {
+    const char *data = "1234567890";
+    MemoryInput memory_input(data);
+    ChunkedInput input(memory_input);
+    {
+        InputReader src(input);
+        EXPECT_EQUAL(src.read(15), Memory());
+        EXPECT_TRUE(src.failed());
+        EXPECT_EQUAL(src.get_error_message(), vespalib::string("input underflow"));
+        EXPECT_EQUAL(src.get_offset(), 10u);        
+    }
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/data/memory_input/CMakeLists.txt b/vespalib/src/tests/data/memory_input/CMakeLists.txt
new file mode 100644
index 00000000000..a57f8129c27
--- /dev/null
+++ b/vespalib/src/tests/data/memory_input/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_memory_input_test_app TEST
+    SOURCES
+    memory_input_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_memory_input_test_app COMMAND vespalib_memory_input_test_app)
diff --git a/vespalib/src/tests/data/memory_input/memory_input_test.cpp b/vespalib/src/tests/data/memory_input/memory_input_test.cpp
new file mode 100644
index 00000000000..a62283500f0
--- /dev/null
+++ b/vespalib/src/tests/data/memory_input/memory_input_test.cpp
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/data/memory_input.h>
+
+using namespace vespalib;
+
+TEST("require that MemoryInput wrapper works as expected") {
+    const char *data = "1234567890";
+    Memory memory(data);
+    EXPECT_EQUAL(memory.size, 10);
+    MemoryInput input(memory);
+    EXPECT_EQUAL(input.obtain(), memory);
+    input.evict(5);
+    EXPECT_EQUAL(input.obtain(), Memory(data + 5));    
+    EXPECT_EQUAL(input.obtain(), Memory(data + 5));    
+    input.evict(5);
+    EXPECT_EQUAL(input.obtain(), Memory());
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/data/output_writer/output_writer_test.cpp b/vespalib/src/tests/data/output_writer/output_writer_test.cpp
index a7b00f40846..750cc7799d4 100644
--- a/vespalib/src/tests/data/output_writer/output_writer_test.cpp
+++ b/vespalib/src/tests/data/output_writer/output_writer_test.cpp
@@ -1,4 +1,4 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/data/simple_buffer.h>
 #include <vespa/vespalib/data/output_writer.h>
diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt
index 24de559ce0b..9d574953016 100644
--- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt
@@ -4,6 +4,7 @@ vespa_add_library(vespalib_vespalib_data OBJECT
     input.cpp
     input_reader.cpp
     memory.cpp
+    memory_input.cpp
     memorydatastore.cpp
     output.cpp
     output_writer.cpp
diff --git a/vespalib/src/vespa/vespalib/data/input_reader.cpp b/vespalib/src/vespa/vespalib/data/input_reader.cpp
index de93ac70807..2757cc5c422 100644
--- a/vespalib/src/vespa/vespalib/data/input_reader.cpp
+++ b/vespalib/src/vespa/vespalib/data/input_reader.cpp
@@ -10,9 +10,8 @@ size_t
 InputReader::obtain_slow()
 {
     if (!failed()) {
-        _input.evict(_pos);
+        _data = _input.evict(_pos).obtain();
         _chunk_offset += _pos;
-        _data = _input.obtain();
         _pos = 0;
         if (_data.size == 0) {
             fail("input underflow");
@@ -46,8 +45,8 @@ InputReader::fail(const vespalib::string &msg) {
     if (!failed()) {
         _error = msg;
         _input.evict(_pos);
-        _chunk_offset += _pos;
         _data = Memory();
+        _chunk_offset += _pos;
         _pos = 0;
     }
 }
diff --git a/vespalib/src/vespa/vespalib/data/memory_input.cpp b/vespalib/src/vespa/vespalib/data/memory_input.cpp
new file mode 100644
index 00000000000..485b438f33e
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/memory_input.cpp
@@ -0,0 +1,21 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "memory_input.h"
+
+namespace vespalib {
+
+Memory
+MemoryInput::obtain()
+{
+    return Memory((_data.data + _pos), (_data.size - _pos));
+}
+
+Input &
+MemoryInput::evict(size_t bytes)
+{
+    _pos += bytes;
+    return *this;
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/data/memory_input.h b/vespalib/src/vespa/vespalib/data/memory_input.h
new file mode 100644
index 00000000000..555a94cf3bf
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/memory_input.h
@@ -0,0 +1,24 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "input.h"
+
+namespace vespalib {
+
+/**
+ * Thin wrapper presenting a single chunk of Memory as an Input.
+ **/
+class MemoryInput : public Input
+{
+private:
+    Memory _data;
+    size_t _pos;
+
+public:
+    MemoryInput(const Memory data) : _data(data), _pos(0) {}
+    Memory obtain() override;
+    Input &evict(size_t bytes) override;
+};
+
+} // namespace vespalib