mapped file input in vespalib

7 files changed, 115 insertions, 0 deletions

diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index d29b577401e..b030c4056f8 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -38,6 +38,7 @@ vespa_define_module(
     src/tests/hashmap
     src/tests/host_name
     src/tests/io/fileutil
+    src/tests/io/mapped_file_input
     src/tests/left_right_heap
     src/tests/linkedptr
     src/tests/make_fixture_macros
diff --git a/vespalib/src/tests/io/mapped_file_input/CMakeLists.txt b/vespalib/src/tests/io/mapped_file_input/CMakeLists.txt
new file mode 100644
index 00000000000..2c637cd84a3
--- /dev/null
+++ b/vespalib/src/tests/io/mapped_file_input/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_mapped_file_input_test_app TEST
+    SOURCES
+    mapped_file_input_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_mapped_file_input_test_app COMMAND vespalib_mapped_file_input_test_app)
diff --git a/vespalib/src/tests/io/mapped_file_input/file.txt b/vespalib/src/tests/io/mapped_file_input/file.txt
new file mode 100644
index 00000000000..dd59d098638
--- /dev/null
+++ b/vespalib/src/tests/io/mapped_file_input/file.txt
@@ -0,0 +1 @@
+file content
diff --git a/vespalib/src/tests/io/mapped_file_input/mapped_file_input_test.cpp b/vespalib/src/tests/io/mapped_file_input/mapped_file_input_test.cpp
new file mode 100644
index 00000000000..14e3eb6a822
--- /dev/null
+++ b/vespalib/src/tests/io/mapped_file_input/mapped_file_input_test.cpp
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/io/mapped_file_input.h>
+
+using namespace vespalib;
+
+TEST("require that missing file is invalid") {
+    MappedFileInput file(TEST_PATH("not_found.txt"));
+    EXPECT_TRUE(!file.valid());
+}
+
+TEST("require that file can be accessed as in input") {
+    MappedFileInput file(TEST_PATH("file.txt"));
+    EXPECT_TRUE(file.valid());
+    EXPECT_EQUAL(file.get(), Memory("file content\n"));
+    EXPECT_EQUAL(file.obtain(), Memory("file content\n"));
+    file.evict(5);
+    EXPECT_EQUAL(file.obtain(), Memory("content\n"));
+    file.evict(8);
+    EXPECT_EQUAL(file.obtain(), Memory());
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/io/CMakeLists.txt b/vespalib/src/vespa/vespalib/io/CMakeLists.txt
index 701bdd45414..927bb47c187 100644
--- a/vespalib/src/vespa/vespalib/io/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/io/CMakeLists.txt
@@ -2,5 +2,6 @@
 vespa_add_library(vespalib_vespalib_io OBJECT
     SOURCES
     fileutil.cpp
+    mapped_file_input.cpp
     DEPENDS
 )
diff --git a/vespalib/src/vespa/vespalib/io/mapped_file_input.cpp b/vespalib/src/vespa/vespalib/io/mapped_file_input.cpp
new file mode 100644
index 00000000000..d9108456347
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/io/mapped_file_input.cpp
@@ -0,0 +1,53 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include "mapped_file_input.h"
+
+namespace vespalib {
+
+MappedFileInput::MappedFileInput(const vespalib::string &file_name)
+    : _fd(open(file_name.c_str(), O_RDONLY)),
+      _data((char *)MAP_FAILED),
+      _size(0),
+      _used(0)
+{
+    struct stat info;
+    if ((_fd != -1) && fstat(_fd, &info) == 0) {
+        _data = static_cast<char*>(mmap(0, info.st_size,
+                                        PROT_READ, MAP_SHARED, _fd, 0));
+        if (_data != MAP_FAILED) {
+            _size = info.st_size;
+            madvise(_data, _size, MADV_SEQUENTIAL);
+        }
+    }
+}
+
+MappedFileInput::~MappedFileInput()
+{
+    if (valid()) {
+        munmap(_data, _size);
+    }
+    if (_fd != -1) {
+        close(_fd);
+    }
+}
+
+bool MappedFileInput::valid() const
+{
+    return (_data != MAP_FAILED);
+}
+
+Memory
+MappedFileInput::obtain()
+{
+    return Memory((_data + _used), (_size - _used));
+}
+
+Input &
+MappedFileInput::evict(size_t bytes)
+{
+    _used += bytes;
+    return *this;
+}
+
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/io/mapped_file_input.h b/vespalib/src/vespa/vespalib/io/mapped_file_input.h
new file mode 100644
index 00000000000..a835e2433b8
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/io/mapped_file_input.h
@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/data/input.h>
+
+namespace vespalib {
+
+/**
+ * Abstract input backed by a memory-mapped file.
+ **/
+class MappedFileInput : public Input
+{
+private:
+    int     _fd;
+    char   *_data;
+    size_t  _size;
+    size_t  _used;
+public:
+    MappedFileInput(const vespalib::string &file_name);
+    ~MappedFileInput();
+    bool valid() const;
+    Memory get() const { return Memory(_data, _size); }
+    Memory obtain() override;
+    Input &evict(size_t bytes) override;
+};
+
+} // namespace vespalib