diff options
author | Håvard Pettersen <havardpe@oath.com> | 2017-08-29 13:39:58 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2017-08-29 13:51:24 +0000 |
commit | 7f9aa079f12dc89271c00735fe501be7d70b5d74 (patch) | |
tree | a36c65fa5236ecbb6ee54340024fec1827f32a72 /vespalib/src | |
parent | f6e0a778040ffac378ddcf2f39846d65ef056092 (diff) |
add support for backing data values with external memory
Diffstat (limited to 'vespalib/src')
15 files changed, 232 insertions, 4 deletions
diff --git a/vespalib/src/tests/slime/external_data_value/CMakeLists.txt b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt new file mode 100644 index 00000000000..df4cb8a23c3 --- /dev/null +++ b/vespalib/src/tests/slime/external_data_value/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_external_data_value_test_app TEST + SOURCES + external_data_value_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_external_data_value_test_app COMMAND vespalib_external_data_value_test_app) diff --git a/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp new file mode 100644 index 00000000000..acf937c840d --- /dev/null +++ b/vespalib/src/tests/slime/external_data_value/external_data_value_test.cpp @@ -0,0 +1,86 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/data/slime/slime.h> + +using namespace vespalib::slime::convenience; +using vespalib::slime::ExternalMemory; + +struct MyMem : ExternalMemory { + const std::vector<char> space; + MyMem(Memory memory) + : space(memory.data, memory.data + memory.size) {} + Memory get() const override { + return Memory(&space[0], space.size()); + } + static UP create(Memory memory) { + return std::make_unique<MyMem>(memory); + } +}; + +void verify_data(const Inspector &pos, Memory expect) { + EXPECT_TRUE(pos.valid()); + EXPECT_EQUAL(vespalib::slime::DATA::ID, pos.type().getId()); + EXPECT_EQUAL(pos.asString(), Memory()); + EXPECT_EQUAL(pos.asData(), expect); +} + +TEST("require that external memory can be used for data values") { + Slime slime; + TEST_DO(verify_data(slime.setData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get(), Memory("foo"))); +} + +TEST("require that nullptr external memory gives empty data value") { + Slime slime; + TEST_DO(verify_data(slime.setData(ExternalMemory::UP(nullptr)), Memory(""))); + TEST_DO(verify_data(slime.get(), Memory(""))); +} + +TEST("require that external memory can be used with array data values") { + Slime slime; + TEST_DO(verify_data(slime.setArray().addData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[0], Memory("foo"))); +} + +TEST("require that external memory can be used with object data values (name)") { + Slime slime; + TEST_DO(verify_data(slime.setObject().setData("field", MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()["field"], Memory("foo"))); +} + +TEST("require that external memory can be used with object data values (symbol)") { + Slime slime; + TEST_DO(verify_data(slime.setObject().setData(Symbol(5), MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo"))); +} + +TEST("require that external memory can be used with slime inserter") { + Slime slime; + SlimeInserter inserter(slime); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get(), Memory("foo"))); +} + +TEST("require that external memory can be used with array inserter") { + Slime slime; + ArrayInserter inserter(slime.setArray()); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[0], Memory("foo"))); +} + +TEST("require that external memory can be used with object inserter") { + Slime slime; + ObjectInserter inserter(slime.setObject(), "field"); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()["field"], Memory("foo"))); +} + +TEST("require that external memory can be used with object symbol inserter") { + Slime slime; + ObjectSymbolInserter inserter(slime.setObject(), Symbol(5)); + TEST_DO(verify_data(inserter.insertData(MyMem::create("foo")), Memory("foo"))); + TEST_DO(verify_data(slime.get()[Symbol(5)], Memory("foo"))); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt index c5aa9b595d2..46a8d410878 100644 --- a/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/slime/CMakeLists.txt @@ -9,6 +9,9 @@ vespa_add_library(vespalib_vespalib_data_slime OBJECT convenience.cpp cursor.cpp empty_value_factory.cpp + external_data_value.cpp + external_data_value_factory.cpp + external_memory.cpp inject.cpp inserter.cpp inspector.cpp diff --git a/vespalib/src/vespa/vespalib/data/slime/cursor.h b/vespalib/src/vespa/vespalib/data/slime/cursor.h index 4242936a483..6815ad3ba83 100644 --- a/vespalib/src/vespa/vespalib/data/slime/cursor.h +++ b/vespalib/src/vespa/vespalib/data/slime/cursor.h @@ -3,6 +3,7 @@ #pragma once #include "inspector.h" +#include "external_memory.h" namespace vespalib { namespace slime { @@ -18,6 +19,7 @@ struct Cursor : public Inspector { virtual Cursor &addDouble(double d) = 0; virtual Cursor &addString(Memory str) = 0; virtual Cursor &addData(Memory data) = 0; + virtual Cursor &addData(ExternalMemory::UP data) = 0; virtual Cursor &addArray() = 0; virtual Cursor &addObject() = 0; @@ -27,6 +29,7 @@ struct Cursor : public Inspector { virtual Cursor &setDouble(Symbol sym, double d) = 0; virtual Cursor &setString(Symbol sym, Memory str) = 0; virtual Cursor &setData(Symbol sym, Memory data) = 0; + virtual Cursor &setData(Symbol sym, ExternalMemory::UP data) = 0; virtual Cursor &setArray(Symbol sym) = 0; virtual Cursor &setObject(Symbol sym) = 0; @@ -35,7 +38,8 @@ struct Cursor : public Inspector { virtual Cursor &setLong(Memory name, int64_t l) = 0; virtual Cursor &setDouble(Memory name, double d) = 0; virtual Cursor &setString(Memory name, Memory str) = 0; - virtual Cursor &setData(Memory name, Memory str) = 0; + virtual Cursor &setData(Memory name, Memory data) = 0; + virtual Cursor &setData(Memory name, ExternalMemory::UP data) = 0; virtual Cursor &setArray(Memory name) = 0; virtual Cursor &setObject(Memory name) = 0; diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp new file mode 100644 index 00000000000..1004f8aab24 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.cpp @@ -0,0 +1,7 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_data_value.h" + +namespace vespalib::slime { + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h new file mode 100644 index 00000000000..3acfb1350bd --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value.h @@ -0,0 +1,23 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" +#include "external_memory.h" + +namespace vespalib::slime { + +/** + * A data value backed by external memory. + **/ +class ExternalDataValue : public Value +{ +private: + ExternalMemory::UP _value; +public: + ExternalDataValue(ExternalMemory::UP data) : _value(std::move(data)) {} + Memory asData() const override { return _value->get(); } + Type type() const override { return DATA::instance; } +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp new file mode 100644 index 00000000000..592ddcdb519 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.cpp @@ -0,0 +1,18 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_data_value_factory.h" +#include "external_data_value.h" +#include "basic_value.h" + +namespace vespalib::slime { + +Value * +ExternalDataValueFactory::create(Stash &stash) const +{ + if (!input) { + return &stash.create<BasicDataValue>(Memory(), stash); + } + return &stash.create<ExternalDataValue>(std::move(input)); +} + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h new file mode 100644 index 00000000000..be85f28aebe --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_data_value_factory.h @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value_factory.h" +#include "external_memory.h" +#include <vespa/vespalib/util/stash.h> + +namespace vespalib::slime { + +/** + * Value factory for data values using external memory. + **/ +struct ExternalDataValueFactory : public ValueFactory { + mutable ExternalMemory::UP input; + ExternalDataValueFactory(ExternalMemory::UP in) : input(std::move(in)) {} + Value *create(Stash &stash) const override; +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp new file mode 100644 index 00000000000..ebbe833098a --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.cpp @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "external_memory.h" + +namespace vespalib::slime { + +ExternalMemory::~ExternalMemory() +{ +} + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/external_memory.h b/vespalib/src/vespa/vespalib/data/slime/external_memory.h new file mode 100644 index 00000000000..a1d4510d0c8 --- /dev/null +++ b/vespalib/src/vespa/vespalib/data/slime/external_memory.h @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <memory> +#include <vespa/vespalib/data/memory.h> + +namespace vespalib::slime { + +/** + * Interface used to access external memory. External memory does not + * need to be copied when added to a Slime object. The Memory obtained + * by calling the get function must be valid until the object + * implementing this interface is destructed. + **/ +struct ExternalMemory { + using UP = std::unique_ptr<ExternalMemory>; + virtual Memory get() const = 0; + virtual ~ExternalMemory(); +}; + +} // namespace vespalib::slime diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp index 81b8ffe334c..f9d80e74e6f 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.cpp @@ -6,12 +6,15 @@ namespace vespalib { namespace slime { +using ExtMemUP = ExternalMemory::UP; + Cursor &SlimeInserter::insertNix() const { return slime.setNix(); } Cursor &SlimeInserter::insertBool(bool value) const { return slime.setBool(value); } Cursor &SlimeInserter::insertLong(int64_t value) const { return slime.setLong(value); } Cursor &SlimeInserter::insertDouble(double value) const { return slime.setDouble(value); } Cursor &SlimeInserter::insertString(Memory value) const { return slime.setString(value); } Cursor &SlimeInserter::insertData(Memory value) const { return slime.setData(value); } +Cursor &SlimeInserter::insertData(ExtMemUP value) const { return slime.setData(std::move(value)); } Cursor &SlimeInserter::insertArray() const { return slime.setArray(); } Cursor &SlimeInserter::insertObject() const { return slime.setObject(); } @@ -21,6 +24,7 @@ Cursor &ArrayInserter::insertLong(int64_t value) const { return cursor.addLong( Cursor &ArrayInserter::insertDouble(double value) const { return cursor.addDouble(value); } Cursor &ArrayInserter::insertString(Memory value) const { return cursor.addString(value); } Cursor &ArrayInserter::insertData(Memory value) const { return cursor.addData(value); } +Cursor &ArrayInserter::insertData(ExtMemUP value) const { return cursor.addData(std::move(value)); } Cursor &ArrayInserter::insertArray() const { return cursor.addArray(); } Cursor &ArrayInserter::insertObject() const { return cursor.addObject(); } @@ -30,6 +34,7 @@ Cursor &ObjectSymbolInserter::insertLong(int64_t value) const { return cursor.s Cursor &ObjectSymbolInserter::insertDouble(double value) const { return cursor.setDouble(symbol, value); } Cursor &ObjectSymbolInserter::insertString(Memory value) const { return cursor.setString(symbol, value); } Cursor &ObjectSymbolInserter::insertData(Memory value) const { return cursor.setData(symbol, value); } +Cursor &ObjectSymbolInserter::insertData(ExtMemUP value) const { return cursor.setData(symbol, std::move(value)); } Cursor &ObjectSymbolInserter::insertArray() const { return cursor.setArray(symbol); } Cursor &ObjectSymbolInserter::insertObject() const { return cursor.setObject(symbol); } @@ -39,6 +44,7 @@ Cursor &ObjectInserter::insertLong(int64_t value) const { return cursor.setLong Cursor &ObjectInserter::insertDouble(double value) const { return cursor.setDouble(name, value); } Cursor &ObjectInserter::insertString(Memory value) const { return cursor.setString(name, value); } Cursor &ObjectInserter::insertData(Memory value) const { return cursor.setData(name, value); } +Cursor &ObjectInserter::insertData(ExtMemUP value) const { return cursor.setData(name, std::move(value)); } Cursor &ObjectInserter::insertArray() const { return cursor.setArray(name); } Cursor &ObjectInserter::insertObject() const { return cursor.setObject(name); } diff --git a/vespalib/src/vespa/vespalib/data/slime/inserter.h b/vespalib/src/vespa/vespalib/data/slime/inserter.h index b8762ed3794..dff37183ac7 100644 --- a/vespalib/src/vespa/vespalib/data/slime/inserter.h +++ b/vespalib/src/vespa/vespalib/data/slime/inserter.h @@ -5,6 +5,7 @@ #include "type.h" #include <vespa/vespalib/data/memory.h> #include "symbol.h" +#include "external_memory.h" namespace vespalib { @@ -27,6 +28,7 @@ struct Inserter { virtual Cursor &insertDouble(double value) const = 0; virtual Cursor &insertString(Memory value) const = 0; virtual Cursor &insertData(Memory value) const = 0; + virtual Cursor &insertData(ExternalMemory::UP value) const = 0; virtual Cursor &insertArray() const = 0; virtual Cursor &insertObject() const = 0; virtual ~Inserter() {} @@ -44,6 +46,7 @@ struct SlimeInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -58,6 +61,7 @@ struct ArrayInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -73,6 +77,7 @@ struct ObjectSymbolInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; @@ -88,6 +93,7 @@ struct ObjectInserter : Inserter { Cursor &insertDouble(double value) const override; Cursor &insertString(Memory value) const override; Cursor &insertData(Memory value) const override; + Cursor &insertData(ExternalMemory::UP value) const override; Cursor &insertArray() const override; Cursor &insertObject() const override; }; diff --git a/vespalib/src/vespa/vespalib/data/slime/slime.h b/vespalib/src/vespa/vespalib/data/slime/slime.h index e5c1d6db1a8..af081887c9a 100644 --- a/vespalib/src/vespa/vespalib/data/slime/slime.h +++ b/vespalib/src/vespa/vespalib/data/slime/slime.h @@ -28,6 +28,7 @@ #include "type.h" #include "value.h" #include "value_factory.h" +#include "external_data_value_factory.h" #include <vespa/vespalib/data/input_reader.h> #include <vespa/vespalib/data/output_writer.h> #include <vespa/vespalib/data/simple_buffer.h> @@ -147,6 +148,9 @@ public: Cursor &setData(const Memory& data) { return _root.set(slime::DataValueFactory(data)); } + Cursor &setData(slime::ExternalMemory::UP data) { + return _root.set(slime::ExternalDataValueFactory(std::move(data))); + } Cursor &setArray() { return _root.set(slime::ArrayValueFactory(*_names)); } diff --git a/vespalib/src/vespa/vespalib/data/slime/value.cpp b/vespalib/src/vespa/vespalib/data/slime/value.cpp index 9b415080d7c..2eae660f431 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/value.cpp @@ -5,6 +5,7 @@ #include "resolved_symbol.h" #include "empty_value_factory.h" #include "basic_value_factory.h" +#include "external_data_value_factory.h" #include <vespa/vespalib/data/simple_buffer.h> #include "json_format.h" @@ -80,7 +81,7 @@ Value::toString() const return buf.get().make_string(); } -// 6 x add +// 7 x add Cursor & Value::addNix() { return addLeaf(NixValueFactory()); } Cursor & @@ -93,8 +94,10 @@ Cursor & Value::addString(Memory str) { return addLeaf(StringValueFactory(str)); } Cursor & Value::addData(Memory data) { return addLeaf(DataValueFactory(data)); } +Cursor & +Value::addData(ExternalMemory::UP data) { return addLeaf(ExternalDataValueFactory(std::move(data))); } -// 6 x set (with numeric symbol id) +// 7 x set (with numeric symbol id) Cursor & Value::setNix(Symbol sym) { return setLeaf(sym, NixValueFactory()); } Cursor & @@ -107,8 +110,10 @@ Cursor & Value::setString(Symbol sym, Memory str) { return setLeaf(sym, StringValueFactory(str)); } Cursor & Value::setData(Symbol sym, Memory data) { return setLeaf(sym, DataValueFactory(data)); } +Cursor & +Value::setData(Symbol sym, ExternalMemory::UP data) { return setLeaf(sym, ExternalDataValueFactory(std::move(data))); } -// 6 x set (with symbol name) +// 7 x set (with symbol name) Cursor & Value::setNix(Memory name) { return setLeaf(name, NixValueFactory()); } Cursor & @@ -121,6 +126,8 @@ Cursor & Value::setString(Memory name, Memory str) { return setLeaf(name, StringValueFactory(str)); } Cursor & Value::setData(Memory name, Memory data) { return setLeaf(name, DataValueFactory(data)); } +Cursor & +Value::setData(Memory name, ExternalMemory::UP data) { return setLeaf(name, ExternalDataValueFactory(std::move(data))); } // nop defaults for array/objects Cursor & diff --git a/vespalib/src/vespa/vespalib/data/slime/value.h b/vespalib/src/vespa/vespalib/data/slime/value.h index dacc9b1800e..eaf14829ed9 100644 --- a/vespalib/src/vespa/vespalib/data/slime/value.h +++ b/vespalib/src/vespa/vespalib/data/slime/value.h @@ -57,6 +57,7 @@ public: Cursor &addDouble(double d) override; Cursor &addString(Memory str) override; Cursor &addData(Memory data) override; + Cursor &addData(ExternalMemory::UP data) override; Cursor &addArray() override; Cursor &addObject() override; @@ -66,6 +67,7 @@ public: Cursor &setDouble(Symbol sym, double d) override; Cursor &setString(Symbol sym, Memory str) override; Cursor &setData(Symbol sym, Memory data) override; + Cursor &setData(Symbol sym, ExternalMemory::UP data) override; Cursor &setArray(Symbol sym) override; Cursor &setObject(Symbol sym) override; @@ -75,6 +77,7 @@ public: Cursor &setDouble(Memory name, double d) override; Cursor &setString(Memory name, Memory str) override; Cursor &setData(Memory name, Memory str) override; + Cursor &setData(Memory name, ExternalMemory::UP data) override; Cursor &setArray(Memory name) override; Cursor &setObject(Memory name) override; |