Add skeleton for multivaluemapping 2, which will replace multivaluemapping

later on.

10 files changed, 192 insertions, 0 deletions

diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index 7fac6802b89..0ca4fe41d2e 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -81,6 +81,7 @@ vespa_define_module(
     src/tests/attribute/enumstore
     src/tests/attribute/extendattributes
     src/tests/attribute/guard
+    src/tests/attribute/multi_value_mapping2
     src/tests/attribute/multivaluemapping
     src/tests/attribute/postinglist
     src/tests/attribute/postinglistattribute
diff --git a/searchlib/src/tests/attribute/multi_value_mapping2/CMakeLists.txt b/searchlib/src/tests/attribute/multi_value_mapping2/CMakeLists.txt
new file mode 100644
index 00000000000..3d450a93e06
--- /dev/null
+++ b/searchlib/src/tests/attribute/multi_value_mapping2/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_multi_value_mapping2_test_app TEST
+    SOURCES
+    multi_value_mapping2_test.cpp
+    DEPENDS
+    searchlib
+)
+vespa_add_test(NAME searchlib_multi_value_mapping2_test_app COMMAND searchlib_multi_value_mapping2_test_app)
diff --git a/searchlib/src/tests/attribute/multi_value_mapping2/DESC b/searchlib/src/tests/attribute/multi_value_mapping2/DESC
new file mode 100644
index 00000000000..ef382c63998
--- /dev/null
+++ b/searchlib/src/tests/attribute/multi_value_mapping2/DESC
@@ -0,0 +1 @@
+This is a test for the MultivalueMapping2 class.
diff --git a/searchlib/src/tests/attribute/multi_value_mapping2/FILES b/searchlib/src/tests/attribute/multi_value_mapping2/FILES
new file mode 100644
index 00000000000..a2355329b3d
--- /dev/null
+++ b/searchlib/src/tests/attribute/multi_value_mapping2/FILES
@@ -0,0 +1 @@
+multi_value_mapping2_test.cpp
diff --git a/searchlib/src/tests/attribute/multi_value_mapping2/multi_value_mapping2_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping2/multi_value_mapping2_test.cpp
new file mode 100644
index 00000000000..0c1bd1db474
--- /dev/null
+++ b/searchlib/src/tests/attribute/multi_value_mapping2/multi_value_mapping2_test.cpp
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+LOG_SETUP("multivaluemapping2_test");
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/attribute/multi_value_mapping2.h>
+#include <vespa/searchlib/attribute/multi_value_mapping2.hpp>
+#include <vespa/vespalib/util/generationhandler.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+
+template <typename EntryT>
+void
+assertArray(std::vector<EntryT> exp, vespalib::ConstArrayRef<EntryT> values)
+{
+    EXPECT_EQUAL(exp, std::vector<EntryT>(values.cbegin(), values.cend()));
+}
+
+template <typename EntryT>
+class Fixture
+{
+    search::attribute::MultiValueMapping2<EntryT> _mvMapping;
+    using generation_t = vespalib::GenerationHandler::generation_t;
+
+public:
+    using ConstArrayRef = vespalib::ConstArrayRef<EntryT>;
+    Fixture(uint32_t maxSmallArraySize)
+        : _mvMapping(maxSmallArraySize)
+    {
+    }
+    ~Fixture() { }
+
+    void set(uint32_t docId, std::vector<EntryT> values) { _mvMapping.set(docId, values); }
+    ConstArrayRef get(uint32_t docId) { return _mvMapping.get(docId); }
+    void assertGet(uint32_t docId, std::vector<EntryT> exp)
+    {
+        ConstArrayRef act = get(docId);
+        EXPECT_EQUAL(exp, std::vector<EntryT>(act.cbegin(), act.cend()));
+    }
+    void transferHoldLists(generation_t generation) { _mvMapping.transferHoldLists(generation); }
+    void trimHoldLists(generation_t firstUsed) { _mvMapping.trimHoldLists(firstUsed); }
+};
+
+TEST_F("Test that set and get works", Fixture<int>(3))
+{
+    f.set(1, {});
+    f.set(2, {4, 7});
+    f.set(3, {5});
+    f.set(4, {10, 14, 17, 16});
+    f.set(5, {3});
+    TEST_DO(f.assertGet(1, {}));
+    TEST_DO(f.assertGet(2, {4, 7}));
+    TEST_DO(f.assertGet(3, {5}));
+    TEST_DO(f.assertGet(4, {10, 14, 17, 16}));
+    TEST_DO(f.assertGet(5, {3}));
+}
+
+TEST_F("Test that old value is not overwritten while held", Fixture<int>(3))
+{
+    f.set(3, {5});
+    typename F1::ConstArrayRef old3 = f.get(3);
+    TEST_DO(assertArray({5}, old3));
+    f.set(3, {7});
+    f.transferHoldLists(10);
+    TEST_DO(assertArray({5}, old3));
+    TEST_DO(f.assertGet(3, {7}));
+    f.trimHoldLists(10);
+    TEST_DO(assertArray({5}, old3));
+    f.trimHoldLists(11);
+    TEST_DO(assertArray({0}, old3));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
index 4983103045b..2c96b590055 100644
--- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
@@ -51,6 +51,7 @@ vespa_add_library(searchlib_attribute OBJECT
     loadednumericvalue.cpp
     loadedstringvalue.cpp
     loadedvalue.cpp
+    multi_value_mapping2.cpp
     multienumattribute.cpp
     multienumattributesaver.cpp
     multinumericattribute.cpp
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.cpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.cpp
new file mode 100644
index 00000000000..26951f24be7
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.cpp
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/fastos/fastos.h>
+#include <vespa/log/log.h>
+#include "multi_value_mapping2.h"
+#include "multi_value_mapping2.hpp"
+#include <vespa/vespalib/stllike/string.h>
+
+LOG_SETUP(".searchlib.attribute.multivaluemapping2");
+
+namespace search {
+namespace attribute {
+
+template class MultiValueMapping2<int32_t>;
+
+} // namespace search::attribute
+} // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.h
new file mode 100644
index 00000000000..f70bdbc05ea
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.h
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/datastore/entryref.h>
+#include <vespa/searchlib/common/rcuvector.h>
+#include <vespa/searchlib/datastore/array_store.h>
+
+namespace search {
+namespace attribute {
+
+template <typename EntryT, typename RefT = datastore::EntryRefT<17> >
+class MultiValueMapping2
+{
+    using EntryRef = datastore::EntryRef;
+    using IndexVector = RcuVectorBase<EntryRef>;
+    using ArrayStore = datastore::ArrayStore<EntryT, RefT>;
+    using generation_t = vespalib::GenerationHandler::generation_t;
+    using ConstArrayRef = vespalib::ConstArrayRef<EntryT>;
+
+    ArrayStore _store;
+    IndexVector _indices;
+public:
+    MultiValueMapping2(uint32_t maxSmallArraySize);
+    ~MultiValueMapping2();
+    ConstArrayRef get(uint32_t docId) const { return _store.get(_indices[docId]); }
+    void set(uint32_t docId, ConstArrayRef values);
+
+    // replace is generally unsafe and should only be used when
+    // compacting enum store (replacing old enum index with updated enum index)
+    void replace(uint32_t docId, ConstArrayRef values);
+
+    // Pass on hold list management to underlying store
+    void transferHoldLists(generation_t generation) { _store.transferHoldLists(generation); }
+    void trimHoldLists(generation_t firstUsed) { _store.trimHoldLists(firstUsed); }
+};
+
+} // namespace search::attribute
+} // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.hpp
new file mode 100644
index 00000000000..76cf7e5213f
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping2.hpp
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchlib/datastore/array_store.hpp>
+
+namespace search {
+namespace attribute {
+
+template <typename EntryT, typename RefT>
+MultiValueMapping2<EntryT,RefT>::MultiValueMapping2(uint32_t maxSmallArraySize)
+    : _store(maxSmallArraySize),
+      _indices(_store.getGenerationHolder())
+{
+}
+
+template <typename EntryT, typename RefT>
+MultiValueMapping2<EntryT,RefT>::~MultiValueMapping2()
+{
+}
+
+
+template <typename EntryT, typename RefT>
+void
+MultiValueMapping2<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values)
+{
+    _indices.ensure_size(docId + 1);
+    _store.remove(_indices[docId]);
+    _indices[docId] = _store.add(values);
+}
+
+template <typename EntryT, typename RefT>
+void
+MultiValueMapping2<EntryT,RefT>::replace(uint32_t docId, ConstArrayRef values)
+{
+    ConstArrayRef oldValues = _store.get(docId);
+    assert(oldValues.size() == values.size());
+    EntryT *dst = const_cast<EntryT *>(&oldValues[0]);
+    for (auto &src : values) {
+        *dst = src;
+        ++dst;
+    }
+}
+
+} // namespace search::attribute
+} // namespace search
diff --git a/searchlib/src/vespa/searchlib/datastore/array_store.h b/searchlib/src/vespa/searchlib/datastore/array_store.h
index 59012855e08..4ae73004e0d 100644
--- a/searchlib/src/vespa/searchlib/datastore/array_store.h
+++ b/searchlib/src/vespa/searchlib/datastore/array_store.h
@@ -36,6 +36,7 @@ private:
     std::vector<std::unique_ptr<SmallArrayType>> _smallArrayTypes;
     LargeArrayType _largeArrayType;
     uint32_t _largeArrayTypeId;
+    using generation_t = vespalib::GenerationHandler::generation_t;
 
     void initArrayTypes();
     // 1-to-1 mapping between type ids and sizes for small arrays is enforced during initialization.
@@ -55,6 +56,11 @@ public:
 
     // Should only be used for unit testing
     const BufferState &bufferState(EntryRef ref) const;
+
+    // Pass on hold list management to underlying store
+    void transferHoldLists(generation_t generation) { _store.transferHoldLists(generation); }
+    void trimHoldLists(generation_t firstUsed) { _store.trimHoldLists(firstUsed); }
+    vespalib::GenerationHolder &getGenerationHolder(void) { return _store.getGenerationHolder(); }
 };
 
 }