add support for backing data values with external memory

16 files changed, 233 insertions, 4 deletions

diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index 3ac7b388ec4..b6f8775b64b 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -68,6 +68,7 @@ vespa_define_module(
     src/tests/simple_thread_bundle
     src/tests/slaveproc
     src/tests/slime
+    src/tests/slime/external_data_value
     src/tests/slime/summary-feature-benchmark
     src/tests/stash
     src/tests/stllike
diff --git a/vespalib/src/tests/slime/external_data_value/CMakeLists.txt b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt
new file mode 100644
index 00000000000..df4cb8a23c3
--- /dev/null
+++ b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_external_data_value_test_app TEST
+    SOURCES
+    external_data_value_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_external_data_value_test_app COMMAND vespalib_external_data_value_test_app)
diff --git a/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp
new file mode 100644
index 00000000000..acf937c840d
--- /dev/null
+++ b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp
@@ -0,0 +1,86 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/data/slime/slime.h>
+
+using namespace vespalib::slime::convenience;
+using vespalib::slime::ExternalMemory;
+
+struct MyMem : ExternalMemory {
+    const std::vector<char> space;
+    MyMem(Memory memory)
+        : space(memory.data, memory.data + memory.size) {}
+    Memory get() const override {
+        return Memory(&space[0], space.size());
+    }
+    static UP create(Memory memory) {
+        return std::make_unique<MyMem>(memory);
+    }
+};
+
+void verify_data(const Inspector &pos, Memory expect) {
+    EXPECT_TRUE(pos.valid());
+    EXPECT_EQUAL(vespalib::slime::DATA::ID, pos.type().getId());
+    EXPECT_EQUAL(pos.asString(), Memory());
+    EXPECT_EQUAL(pos.asData(), expect);
+}
+
+TEST("require that external memory can be used for data values") {
+    Slime slime;
+    TEST_DO(verify_data(slime.setData(MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get(), Memory("foo")));
+}
+
+TEST("require that nullptr external memory gives empty data value") {
+    Slime slime;
+    TEST_DO(verify_data(slime.setData(ExternalMemory::UP(nullptr)), Memory("")));
+    TEST_DO(verify_data(slime.get(), Memory("")));
+}
+
+TEST("require that external memory can be used with array data values") {
+    Slime slime;
+    TEST_DO(verify_data(slime.setArray().addData(MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get()[0], Memory("foo")));
+}
+
+TEST("require that external memory can be used with object data values (name)") {
+    Slime slime;
+    TEST_DO(verify_data(slime.setObject().setData("field", MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get()["field"], Memory("foo")));
+}
+
+TEST("require that external memory can be used with object data values (symbol)") {
+    Slime slime;
+    TEST_DO(verify_data(slime.setObject().setData(Symbol(5), MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo")));
+}
+
+TEST("require that external memory can be used with slime inserter") {
+    Slime slime;
+    SlimeInserter inserter(slime);
+    TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get(), Memory("foo")));
+}
+
+TEST("require that external memory can be used with array inserter") {
+    Slime slime;
+    ArrayInserter inserter(slime.setArray());
+    TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get()[0], Memory("foo")));
+}
+
+TEST("require that external memory can be used with object inserter") {
+    Slime slime;
+    ObjectInserter inserter(slime.setObject(), "field");
+    TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get()["field"], Memory("foo")));
+}
+
+TEST("require that external memory can be used with object symbol inserter") {
+    Slime slime;
+    ObjectSymbolInserter inserter(slime.setObject(), Symbol(5));
+    TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo")));
+    TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo")));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt
index c5aa9b595d2..46a8d410878 100644
--- a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt
@@ -9,6 +9,9 @@ vespa_add_library(vespalib_vespalib_data_slime OBJECT
     convenience.cpp
     cursor.cpp
     empty_value_factory.cpp
+    external_data_value.cpp
+    external_data_value_factory.cpp
+    external_memory.cpp
     inject.cpp
     inserter.cpp
     inspector.cpp
diff --git a/vespalib/src/vespa/vespalib/data/slime/cursor.h b/vespalib/src/vespa/vespalib/data/slime/cursor.h
index 4242936a483..6815ad3ba83 100644
--- a/vespalib/src/vespa/vespalib/data/slime/cursor.h
+++ b/vespalib/src/vespa/vespalib/data/slime/cursor.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include "inspector.h"
+#include "external_memory.h"
 
 namespace vespalib {
 namespace slime {
@@ -18,6 +19,7 @@ struct Cursor : public Inspector {
     virtual Cursor &addDouble(double d) = 0;
     virtual Cursor &addString(Memory str) = 0;
     virtual Cursor &addData(Memory data) = 0;
+    virtual Cursor &addData(ExternalMemory::UP data) = 0;
     virtual Cursor &addArray() = 0;
     virtual Cursor &addObject() = 0;
 
@@ -27,6 +29,7 @@ struct Cursor : public Inspector {
     virtual Cursor &setDouble(Symbol sym, double d) = 0;
     virtual Cursor &setString(Symbol sym, Memory str) = 0;
     virtual Cursor &setData(Symbol sym, Memory data) = 0;
+    virtual Cursor &setData(Symbol sym, ExternalMemory::UP data) = 0;
     virtual Cursor &setArray(Symbol sym) = 0;
     virtual Cursor &setObject(Symbol sym) = 0;
 
@@ -35,7 +38,8 @@ struct Cursor : public Inspector {
     virtual Cursor &setLong(Memory name, int64_t l) = 0;
     virtual Cursor &setDouble(Memory name, double d) = 0;
     virtual Cursor &setString(Memory name, Memory str) = 0;
-    virtual Cursor &setData(Memory name, Memory str) = 0;
+    virtual Cursor &setData(Memory name, Memory data) = 0;
+    virtual Cursor &setData(Memory name, ExternalMemory::UP data) = 0;
     virtual Cursor &setArray(Memory name) = 0;
     virtual Cursor &setObject(Memory name) = 0;
 
diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp
new file mode 100644
index 00000000000..1004f8aab24
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp
@@ -0,0 +1,7 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "external_data_value.h"
+
+namespace vespalib::slime {
+
+} // namespace vespalib::slime
diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h
new file mode 100644
index 00000000000..3acfb1350bd
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h
@@ -0,0 +1,23 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "value.h"
+#include "external_memory.h"
+
+namespace vespalib::slime {
+
+/**
+ * A data value backed by external memory.
+ **/
+class ExternalDataValue : public Value
+{
+private:
+    ExternalMemory::UP _value;
+public:
+    ExternalDataValue(ExternalMemory::UP data) : _value(std::move(data)) {}
+    Memory asData() const override { return _value->get(); }
+    Type type() const override { return DATA::instance; }
+};
+
+} // namespace vespalib::slime
diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp
new file mode 100644
index 00000000000..592ddcdb519
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp
@@ -0,0 +1,18 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "external_data_value_factory.h"
+#include "external_data_value.h"
+#include "basic_value.h"
+
+namespace vespalib::slime {
+
+Value *
+ExternalDataValueFactory::create(Stash &stash) const
+{
+    if (!input) {
+        return &stash.create<BasicDataValue>(Memory(), stash);
+    }
+    return &stash.create<ExternalDataValue>(std::move(input));
+}
+
+} // namespace vespalib::slime
diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h
new file mode 100644
index 00000000000..be85f28aebe
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h
@@ -0,0 +1,20 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "value_factory.h"
+#include "external_memory.h"
+#include <vespa/vespalib/util/stash.h>
+
+namespace vespalib::slime {
+
+/**
+ * Value factory for data values using external memory.
+ **/
+struct ExternalDataValueFactory : public ValueFactory {
+    mutable ExternalMemory::UP input;
+    ExternalDataValueFactory(ExternalMemory::UP in) : input(std::move(in)) {}
+    Value *create(Stash &stash) const override;
+};
+
+} // namespace vespalib::slime
diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp
new file mode 100644
index 00000000000..ebbe833098a
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp
@@ -0,0 +1,11 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "external_memory.h"
+
+namespace vespalib::slime {
+
+ExternalMemory::~ExternalMemory()
+{
+}
+
+} // namespace vespalib::slime
diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.h b/vespalib/src/vespa/vespalib/data/slime/external_memory.h
new file mode 100644
index 00000000000..a1d4510d0c8
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.h
@@ -0,0 +1,22 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+#include <vespa/vespalib/data/memory.h>
+
+namespace vespalib::slime {
+
+/**
+ * Interface used to access external memory. External memory does not
+ * need to be copied when added to a Slime object. The Memory obtained
+ * by calling the get function must be valid until the object
+ * implementing this interface is destructed.
+ **/
+struct ExternalMemory {
+    using UP = std::unique_ptr<ExternalMemory>;
+    virtual Memory get() const = 0;   
+    virtual ~ExternalMemory();
+};
+
+} // namespace vespalib::slime
diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp
index 81b8ffe334c..f9d80e74e6f 100644
--- a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp
+++ b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp
@@ -6,12 +6,15 @@
 namespace vespalib {
 namespace slime {
 
+using ExtMemUP = ExternalMemory::UP;
+
 Cursor &SlimeInserter::insertNix()                const { return slime.setNix(); }
 Cursor &SlimeInserter::insertBool(bool value)     const { return slime.setBool(value); }
 Cursor &SlimeInserter::insertLong(int64_t value)  const { return slime.setLong(value); }
 Cursor &SlimeInserter::insertDouble(double value) const { return slime.setDouble(value); }
 Cursor &SlimeInserter::insertString(Memory value) const { return slime.setString(value); }
 Cursor &SlimeInserter::insertData(Memory value)   const { return slime.setData(value); }
+Cursor &SlimeInserter::insertData(ExtMemUP value) const { return slime.setData(std::move(value)); }
 Cursor &SlimeInserter::insertArray()              const { return slime.setArray(); }
 Cursor &SlimeInserter::insertObject()             const { return slime.setObject(); }
 
@@ -21,6 +24,7 @@ Cursor &ArrayInserter::insertLong(int64_t value)  const { return cursor.addLong(
 Cursor &ArrayInserter::insertDouble(double value) const { return cursor.addDouble(value); }
 Cursor &ArrayInserter::insertString(Memory value) const { return cursor.addString(value); }
 Cursor &ArrayInserter::insertData(Memory value)   const { return cursor.addData(value); }
+Cursor &ArrayInserter::insertData(ExtMemUP value) const { return cursor.addData(std::move(value)); }
 Cursor &ArrayInserter::insertArray()              const { return cursor.addArray(); }
 Cursor &ArrayInserter::insertObject()             const { return cursor.addObject(); }
 
@@ -30,6 +34,7 @@ Cursor &ObjectSymbolInserter::insertLong(int64_t value)  const { return cursor.s
 Cursor &ObjectSymbolInserter::insertDouble(double value) const { return cursor.setDouble(symbol, value); }
 Cursor &ObjectSymbolInserter::insertString(Memory value) const { return cursor.setString(symbol, value); }
 Cursor &ObjectSymbolInserter::insertData(Memory value)   const { return cursor.setData(symbol, value); }
+Cursor &ObjectSymbolInserter::insertData(ExtMemUP value) const { return cursor.setData(symbol, std::move(value)); }
 Cursor &ObjectSymbolInserter::insertArray()              const { return cursor.setArray(symbol); }
 Cursor &ObjectSymbolInserter::insertObject()             const { return cursor.setObject(symbol); }
 
@@ -39,6 +44,7 @@ Cursor &ObjectInserter::insertLong(int64_t value)  const { return cursor.setLong
 Cursor &ObjectInserter::insertDouble(double value) const { return cursor.setDouble(name, value); }
 Cursor &ObjectInserter::insertString(Memory value) const { return cursor.setString(name, value); }
 Cursor &ObjectInserter::insertData(Memory value)   const { return cursor.setData(name, value); }
+Cursor &ObjectInserter::insertData(ExtMemUP value) const { return cursor.setData(name, std::move(value)); }
 Cursor &ObjectInserter::insertArray()              const { return cursor.setArray(name); }
 Cursor &ObjectInserter::insertObject()             const { return cursor.setObject(name); }
 
diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h
index b8762ed3794..dff37183ac7 100644
--- a/vespalib/src/vespa/vespalib/data/slime/inserter.h
+++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h
@@ -5,6 +5,7 @@
 #include "type.h"
 #include <vespa/vespalib/data/memory.h>
 #include "symbol.h"
+#include "external_memory.h"
 
 namespace vespalib {
 
@@ -27,6 +28,7 @@ struct Inserter {
     virtual Cursor &insertDouble(double value) const = 0;
     virtual Cursor &insertString(Memory value) const = 0;
     virtual Cursor &insertData(Memory value) const = 0;
+    virtual Cursor &insertData(ExternalMemory::UP value) const = 0;
     virtual Cursor &insertArray() const = 0;
     virtual Cursor &insertObject() const = 0;
     virtual ~Inserter() {}
@@ -44,6 +46,7 @@ struct SlimeInserter : Inserter {
     Cursor &insertDouble(double value) const override;
     Cursor &insertString(Memory value) const override;
     Cursor &insertData(Memory value) const override;
+    Cursor &insertData(ExternalMemory::UP value) const override;
     Cursor &insertArray() const override;
     Cursor &insertObject() const override;
 };
@@ -58,6 +61,7 @@ struct ArrayInserter : Inserter {
     Cursor &insertDouble(double value) const override;
     Cursor &insertString(Memory value) const override;
     Cursor &insertData(Memory value) const override;
+    Cursor &insertData(ExternalMemory::UP value) const override;
     Cursor &insertArray() const override;
     Cursor &insertObject() const override;
 };
@@ -73,6 +77,7 @@ struct ObjectSymbolInserter : Inserter {
     Cursor &insertDouble(double value) const override;
     Cursor &insertString(Memory value) const override;
     Cursor &insertData(Memory value) const override;
+    Cursor &insertData(ExternalMemory::UP value) const override;
     Cursor &insertArray() const override;
     Cursor &insertObject() const override;
 };
@@ -88,6 +93,7 @@ struct ObjectInserter : Inserter {
     Cursor &insertDouble(double value) const override;
     Cursor &insertString(Memory value) const override;
     Cursor &insertData(Memory value) const override;
+    Cursor &insertData(ExternalMemory::UP value) const override;
     Cursor &insertArray() const override;
     Cursor &insertObject() const override;
 };
diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h
index e5c1d6db1a8..af081887c9a 100644
--- a/vespalib/src/vespa/vespalib/data/slime/slime.h
+++ b/vespalib/src/vespa/vespalib/data/slime/slime.h
@@ -28,6 +28,7 @@
 #include "type.h"
 #include "value.h"
 #include "value_factory.h"
+#include "external_data_value_factory.h"
 #include <vespa/vespalib/data/input_reader.h>
 #include <vespa/vespalib/data/output_writer.h>
 #include <vespa/vespalib/data/simple_buffer.h>
@@ -147,6 +148,9 @@ public:
     Cursor &setData(const Memory& data) {
         return _root.set(slime::DataValueFactory(data));
     }
+    Cursor &setData(slime::ExternalMemory::UP data) {
+        return _root.set(slime::ExternalDataValueFactory(std::move(data)));
+    }
     Cursor &setArray() {
         return _root.set(slime::ArrayValueFactory(*_names));
     }
diff --git a/vespalib/src/vespa/vespalib/data/slime/value.cpp b/vespalib/src/vespa/vespalib/data/slime/value.cpp
index 9b415080d7c..2eae660f431 100644
--- a/vespalib/src/vespa/vespalib/data/slime/value.cpp
+++ b/vespalib/src/vespa/vespalib/data/slime/value.cpp
@@ -5,6 +5,7 @@
 #include "resolved_symbol.h"
 #include "empty_value_factory.h"
 #include "basic_value_factory.h"
+#include "external_data_value_factory.h"
 #include <vespa/vespalib/data/simple_buffer.h>
 #include "json_format.h"
 
@@ -80,7 +81,7 @@ Value::toString() const
     return buf.get().make_string();
 }
 
-// 6 x add
+// 7 x add
 Cursor &
 Value::addNix() { return addLeaf(NixValueFactory()); }
 Cursor &
@@ -93,8 +94,10 @@ Cursor &
 Value::addString(Memory str) { return addLeaf(StringValueFactory(str)); }
 Cursor &
 Value::addData(Memory data) { return addLeaf(DataValueFactory(data)); }
+Cursor &
+Value::addData(ExternalMemory::UP data) { return addLeaf(ExternalDataValueFactory(std::move(data))); }
 
-// 6 x set (with numeric symbol id)
+// 7 x set (with numeric symbol id)
 Cursor &
 Value::setNix(Symbol sym) { return setLeaf(sym, NixValueFactory()); }
 Cursor &
@@ -107,8 +110,10 @@ Cursor &
 Value::setString(Symbol sym, Memory str) { return setLeaf(sym, StringValueFactory(str)); }
 Cursor &
 Value::setData(Symbol sym, Memory data) { return setLeaf(sym, DataValueFactory(data)); }
+Cursor &
+Value::setData(Symbol sym, ExternalMemory::UP data) { return setLeaf(sym, ExternalDataValueFactory(std::move(data))); }
 
-// 6 x set (with symbol name)
+// 7 x set (with symbol name)
 Cursor &
 Value::setNix(Memory name) { return setLeaf(name, NixValueFactory()); }
 Cursor &
@@ -121,6 +126,8 @@ Cursor &
 Value::setString(Memory name, Memory str) { return setLeaf(name, StringValueFactory(str)); }
 Cursor &
 Value::setData(Memory name, Memory data) { return setLeaf(name, DataValueFactory(data)); }
+Cursor &
+Value::setData(Memory name, ExternalMemory::UP data) { return setLeaf(name, ExternalDataValueFactory(std::move(data))); }
 
 // nop defaults for array/objects
 Cursor &
diff --git a/vespalib/src/vespa/vespalib/data/slime/value.h b/vespalib/src/vespa/vespalib/data/slime/value.h
index dacc9b1800e..eaf14829ed9 100644
--- a/vespalib/src/vespa/vespalib/data/slime/value.h
+++ b/vespalib/src/vespa/vespalib/data/slime/value.h
@@ -57,6 +57,7 @@ public:
     Cursor &addDouble(double d) override;
     Cursor &addString(Memory str) override;
     Cursor &addData(Memory data) override;
+    Cursor &addData(ExternalMemory::UP data) override;
     Cursor &addArray() override;
     Cursor &addObject() override;
 
@@ -66,6 +67,7 @@ public:
     Cursor &setDouble(Symbol sym, double d) override;
     Cursor &setString(Symbol sym, Memory str) override;
     Cursor &setData(Symbol sym, Memory data) override;
+    Cursor &setData(Symbol sym, ExternalMemory::UP data) override;
     Cursor &setArray(Symbol sym) override;
     Cursor &setObject(Symbol sym) override;
 
@@ -75,6 +77,7 @@ public:
     Cursor &setDouble(Memory name, double d) override;
     Cursor &setString(Memory name, Memory str) override;
     Cursor &setData(Memory name, Memory str) override;
+    Cursor &setData(Memory name, ExternalMemory::UP data) override;
     Cursor &setArray(Memory name) override;
     Cursor &setObject(Memory name) override;