Add multi value read views for extendable attributes.

12 files changed, 476 insertions, 15 deletions

diff --git a/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp
index 150490ddd92..43d819ca966 100644
--- a/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp
+++ b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp
@@ -3,6 +3,7 @@
 #include <vespa/searchcommon/attribute/i_multi_value_attribute.h>
 #include <vespa/searchcommon/attribute/multi_value_traits.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/extendableattributes.h>
 #include <vespa/searchlib/attribute/floatbase.h>
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/attribute/stringbase.h>
@@ -41,7 +42,7 @@ protected:
     ~MultiValueReadViewTest() override = default;
 
     template <typename AttributeBaseType, typename BaseType>
-    void populate_helper(AttributeBaseType& attr, const std::vector<BaseType>& values);
+    void populate_helper(AttributeVector& attr, const std::vector<BaseType>& values);
     void populate(AttributeVector& attr);
     template <typename MultiValueType>
     void check_values_helper(const AttributeVector &attr, const std::vector<multivalue::ValueType_t<MultiValueType>>& exp_values);
@@ -51,23 +52,41 @@ protected:
     void check_floating_point_values(const AttributeVector &attr);
     void check_string_values(const AttributeVector &attr);
     void check_values(const AttributeVector& attr);
+    std::shared_ptr<AttributeVector> make_extendable_attribute(CollectionType collection_type);
     void test_normal_attribute_vector(CollectionType collection_type, bool fast_search);
+    void test_extendable_attribute_vector(CollectionType collection_type);
 };
 
 template <typename AttributeBaseType, typename BaseType>
 void
-MultiValueReadViewTest::populate_helper(AttributeBaseType& attr, const std::vector<BaseType>& values)
+MultiValueReadViewTest::populate_helper(AttributeVector& attr, const std::vector<BaseType>& values)
 {
-    attr.addReservedDoc();
+    auto extend_interface = attr.getExtendInterface();
+    if (extend_interface == nullptr) {
+        attr.addReservedDoc();
+    } else {
+        uint32_t doc_id = 0;
+        EXPECT_TRUE(attr.addDoc(doc_id));
+        EXPECT_EQ(0u, doc_id);
+    }
     uint32_t doc_id = 0;
     attr.addDoc(doc_id);
-    EXPECT_NE(0u, doc_id);
-    attr.clearDoc(doc_id);
+    EXPECT_EQ(1u, doc_id);
+    if (extend_interface == nullptr) {
+        attr.clearDoc(doc_id);
+    }
     attr.addDoc(doc_id);
-    EXPECT_NE(0u, doc_id);
-    attr.clearDoc(doc_id);
-    attr.append(doc_id, values[0], 2);
-    attr.append(doc_id, values[1], 7);
+    EXPECT_EQ(2u, doc_id);
+    if (extend_interface == nullptr) {
+        attr.clearDoc(doc_id);
+        auto& spec_attr = dynamic_cast<AttributeBaseType&>(attr);
+        EXPECT_TRUE(spec_attr.append(doc_id, values[0], 2));
+        EXPECT_TRUE(spec_attr.append(doc_id, values[1], 7));
+        attr.commit();
+    } else {
+        EXPECT_TRUE(extend_interface->add(values[0], 2));
+        EXPECT_TRUE(extend_interface->add(values[1], 7));
+    }
 }
 
 void
@@ -78,19 +97,18 @@ MultiValueReadViewTest::populate(AttributeVector& attr)
     case BasicType::Type::INT16:
     case BasicType::Type::INT32:
     case BasicType::Type::INT64:
-        populate_helper<IntegerAttribute, int64_t>(dynamic_cast<IntegerAttribute&>(attr), {42, 44});
+        populate_helper<IntegerAttribute, int64_t>(attr, {42, 44});
         break;
     case BasicType::Type::FLOAT:
     case BasicType::Type::DOUBLE:
-        populate_helper<FloatingPointAttribute, double>(dynamic_cast<FloatingPointAttribute&>(attr), {42.0, 44.0});
+        populate_helper<FloatingPointAttribute, double>(attr, {42.0, 44.0});
         break;
     case BasicType::Type::STRING:
-        populate_helper<StringAttribute, const char*>(dynamic_cast<StringAttribute&>(attr), {"42", "44"});
+        populate_helper<StringAttribute, const char*>(attr, {"42", "44"});
         break;
     default:
         FAIL() << "Cannot populate attribute vector";
     }
-    attr.commit();
 }
 
 namespace {
@@ -202,6 +220,50 @@ MultiValueReadViewTest::check_values(const AttributeVector& attr)
     }
 }
 
+std::shared_ptr<AttributeVector>
+MultiValueReadViewTest::make_extendable_attribute(CollectionType collection_type)
+{
+    vespalib::string name("attr");
+    // Match strategy in streaming visitor
+    switch (collection_type.type()) {
+    case CollectionType::Type::ARRAY:
+        switch (GetParam().basic_type().type()) {
+        case BasicType::Type::INT8:
+        case BasicType::Type::INT16:
+        case BasicType::Type::INT32:
+        case BasicType::Type::INT64:
+            return std::make_shared<MultiIntegerExtAttribute>(name);
+        case BasicType::Type::FLOAT:
+        case BasicType::Type::DOUBLE:
+            return std::make_shared<MultiFloatExtAttribute>(name);
+        case BasicType::Type::STRING:
+            return std::make_shared<MultiStringExtAttribute>(name);
+        default:
+            ;
+        }
+        break;
+    case CollectionType::Type::WSET:
+        switch (GetParam().basic_type().type()) {
+        case BasicType::Type::INT8:
+        case BasicType::Type::INT16:
+        case BasicType::Type::INT32:
+        case BasicType::Type::INT64:
+            return std::make_shared<WeightedSetIntegerExtAttribute>(name);
+        case BasicType::Type::FLOAT:
+        case BasicType::Type::DOUBLE:
+            return std::make_shared<WeightedSetFloatExtAttribute>(name);
+        case BasicType::Type::STRING:
+            return std::make_shared<WeightedSetStringExtAttribute>(name);
+        default:
+            ;
+        }
+        break;
+    default:
+        ;
+    }
+    return {};
+}
+
 void
 MultiValueReadViewTest::test_normal_attribute_vector(CollectionType collection_type, bool fast_search)
 {
@@ -213,6 +275,17 @@ MultiValueReadViewTest::test_normal_attribute_vector(CollectionType collection_t
     check_values(*attr);
 }
 
+void
+MultiValueReadViewTest::test_extendable_attribute_vector(CollectionType collection_type)
+{
+    auto attr = make_extendable_attribute(collection_type);
+    if (attr == nullptr) {
+        FAIL() << "Cannot create extend attribute";
+    }
+    populate(*attr);
+    check_values(*attr);
+}
+
 TEST_P(MultiValueReadViewTest, test_array)
 {
     test_normal_attribute_vector(CollectionType::Type::ARRAY, false);
@@ -233,6 +306,16 @@ TEST_P(MultiValueReadViewTest, test_enumerated_weighted_set)
     test_normal_attribute_vector(CollectionType::Type::WSET, true);
 };
 
+TEST_P(MultiValueReadViewTest, test_extendable_array)
+{
+    test_extendable_attribute_vector(CollectionType::Type::ARRAY);
+}
+
+TEST_P(MultiValueReadViewTest, test_extendable_weighted_set)
+{
+    test_extendable_attribute_vector(CollectionType::Type::WSET);
+}
+
 auto test_values = ::testing::Values(TestParam(BasicType::Type::INT8),
                                      TestParam(BasicType::Type::INT16),
                                      TestParam(BasicType::Type::INT32),
diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
index 8caa6299025..108beb96833 100644
--- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt
@@ -48,6 +48,10 @@ vespa_add_library(searchlib_attribute OBJECT
     enumstore.cpp
     enumerated_multi_value_read_view.cpp
     extendableattributes.cpp
+    extendable_numeric_array_multi_value_read_view.cpp
+    extendable_numeric_weighted_set_multi_value_read_view.cpp
+    extendable_string_array_multi_value_read_view.cpp
+    extendable_string_weighted_set_multi_value_read_view.cpp
     fixedsourceselector.cpp
     flagattribute.cpp
     floatbase.cpp
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.cpp
new file mode 100644
index 00000000000..c0b5239ca7f
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.cpp
@@ -0,0 +1,48 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "extendable_numeric_array_multi_value_read_view.h"
+
+namespace search::attribute {
+
+template <class MultiValueType, typename BaseType>
+ExtendableNumericArrayMultiValueReadView<MultiValueType, BaseType>::ExtendableNumericArrayMultiValueReadView(vespalib::ConstArrayRef<BaseType> data, vespalib::ConstArrayRef<uint32_t> idx)
+    : attribute::IMultiValueReadView<MultiValueType>(),
+      _data(data),
+      _idx(idx),
+      _copy()
+{
+}
+
+template <class MultiValueType, typename BaseType>
+ExtendableNumericArrayMultiValueReadView<MultiValueType, BaseType>::~ExtendableNumericArrayMultiValueReadView() = default;
+
+template <class MultiValueType, typename BaseType>
+vespalib::ConstArrayRef<MultiValueType>
+ExtendableNumericArrayMultiValueReadView<MultiValueType, BaseType>::get_values(uint32_t doc_id) const
+{
+    auto offset = _idx[doc_id];
+    auto next_offset = _idx[doc_id + 1];
+    vespalib::ConstArrayRef<BaseType> raw(_data.data() + offset, next_offset - offset);
+    if (_copy.size() < raw.size()) {
+        _copy.resize(raw.size());
+    }
+    auto dst = _copy.data();
+    for (auto &src : raw) {
+        *dst = multivalue::ValueBuilder<MultiValueType>::build(src, 1);
+        ++dst;
+    }
+    return vespalib::ConstArrayRef(_copy.data(), raw.size());
+}
+
+template class ExtendableNumericArrayMultiValueReadView<int8_t, int8_t>;
+template class ExtendableNumericArrayMultiValueReadView<int16_t, int16_t>;
+template class ExtendableNumericArrayMultiValueReadView<int32_t, int32_t>;
+template class ExtendableNumericArrayMultiValueReadView<int64_t, int64_t>;
+template class ExtendableNumericArrayMultiValueReadView<double, double>;
+template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int8_t>, int8_t>;
+template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int16_t>, int16_t>;
+template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int32_t>, int32_t>;
+template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<int64_t>, int64_t>;
+template class ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<double>, double>;
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.h
new file mode 100644
index 00000000000..f763b7ff7b6
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_array_multi_value_read_view.h
@@ -0,0 +1,28 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/attribute/i_multi_value_read_view.h>
+
+namespace search::attribute {
+
+/**
+ * Read view for the data stored in an extendable multi-value numeric
+ * array attribute vector (used by streaming visitor) that handles
+ * optional addition of weight.
+ * @tparam MultiValueType The multi-value type of the data to access.
+ * @tparam BaseType The base type of the raw data to access.
+ */
+template <typename MultiValueType, typename BaseType>
+class ExtendableNumericArrayMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType>
+{
+    vespalib::ConstArrayRef<BaseType>   _data;
+    vespalib::ConstArrayRef<uint32_t>   _idx;
+    mutable std::vector<MultiValueType> _copy;
+public:
+    ExtendableNumericArrayMultiValueReadView(vespalib::ConstArrayRef<BaseType> data, vespalib::ConstArrayRef<uint32_t> idx);
+    ~ExtendableNumericArrayMultiValueReadView() override;
+    vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.cpp
new file mode 100644
index 00000000000..087ef8d4549
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.cpp
@@ -0,0 +1,43 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "extendable_numeric_weighted_set_multi_value_read_view.h"
+
+namespace search::attribute {
+
+template <class MultiValueType, typename BaseType>
+ExtendableNumericWeightedSetMultiValueReadView<MultiValueType, BaseType>::ExtendableNumericWeightedSetMultiValueReadView(vespalib::ConstArrayRef<BaseType> data, vespalib::ConstArrayRef<uint32_t> idx, vespalib::ConstArrayRef<int32_t> weights)
+    : attribute::IMultiValueReadView<MultiValueType>(),
+      _data(data),
+      _idx(idx),
+      _weights(weights),
+      _copy()
+{
+}
+
+template <class MultiValueType, typename BaseType>
+ExtendableNumericWeightedSetMultiValueReadView<MultiValueType, BaseType>::~ExtendableNumericWeightedSetMultiValueReadView() = default;
+
+template <class MultiValueType, typename BaseType>
+vespalib::ConstArrayRef<MultiValueType>
+ExtendableNumericWeightedSetMultiValueReadView<MultiValueType, BaseType>::get_values(uint32_t doc_id) const
+{
+    auto offset = _idx[doc_id];
+    auto next_offset = _idx[doc_id + 1];
+    vespalib::ConstArrayRef<BaseType> raw(_data.data() + offset, next_offset - offset);
+    if (_copy.size() < raw.size()) {
+        _copy.resize(raw.size());
+    }
+    auto dst = _copy.data();
+    auto* src_weight = _weights.data() + offset;
+    for (auto &src : raw) {
+        *dst = multivalue::ValueBuilder<MultiValueType>::build(src, *src_weight);
+        ++src_weight;
+        ++dst;
+    }
+    return vespalib::ConstArrayRef(_copy.data(), raw.size());
+}
+
+template class ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<int64_t>, int64_t>;
+template class ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<double>, double>;
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.h
new file mode 100644
index 00000000000..6831926d84a
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_numeric_weighted_set_multi_value_read_view.h
@@ -0,0 +1,29 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/attribute/i_multi_value_read_view.h>
+
+namespace search::attribute {
+
+/**
+ * Read view for the data stored in an extendable multi-value numeric
+ * weighted set attribute vector (used by streaming visitor) that handles
+ * optional removal of weight.
+ * @tparam MultiValueType The multi-value type of the data to access.
+ * @tparam BaseType The base type of the raw data to access.
+ */
+template <typename MultiValueType, typename BaseType>
+class ExtendableNumericWeightedSetMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType>
+{
+    vespalib::ConstArrayRef<BaseType>   _data;
+    vespalib::ConstArrayRef<uint32_t>   _idx;
+    vespalib::ConstArrayRef<int32_t>    _weights;
+    mutable std::vector<MultiValueType> _copy;
+public:
+    ExtendableNumericWeightedSetMultiValueReadView(vespalib::ConstArrayRef<BaseType> data, vespalib::ConstArrayRef<uint32_t> idx, vespalib::ConstArrayRef<int32_t> weights);
+    ~ExtendableNumericWeightedSetMultiValueReadView() override;
+    vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp
new file mode 100644
index 00000000000..7994d40c480
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp
@@ -0,0 +1,41 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "extendable_string_array_multi_value_read_view.h"
+
+namespace search::attribute {
+
+template <class MultiValueType>
+ExtendableStringArrayMultiValueReadView<MultiValueType>::ExtendableStringArrayMultiValueReadView(vespalib::ConstArrayRef<char> buffer, vespalib::ConstArrayRef<uint32_t> offsets, vespalib::ConstArrayRef<uint32_t> idx)
+    : attribute::IMultiValueReadView<MultiValueType>(),
+      _buffer(buffer),
+      _offsets(offsets),
+      _idx(idx),
+      _copy()
+{
+}
+
+template <class MultiValueType>
+ExtendableStringArrayMultiValueReadView<MultiValueType>::~ExtendableStringArrayMultiValueReadView() = default;
+
+template <class MultiValueType>
+vespalib::ConstArrayRef<MultiValueType>
+ExtendableStringArrayMultiValueReadView<MultiValueType>::get_values(uint32_t doc_id) const
+{
+    auto offset = _idx[doc_id];
+    auto next_offset = _idx[doc_id + 1];
+    vespalib::ConstArrayRef<uint32_t> raw(_offsets.data() + offset, next_offset - offset);
+    if (_copy.size() < raw.size()) {
+        _copy.resize(raw.size());
+    }
+    auto dst = _copy.data();
+    for (auto &src : raw) {
+        *dst = multivalue::ValueBuilder<MultiValueType>::build(_buffer.data() + src, 1);
+        ++dst;
+    }
+    return vespalib::ConstArrayRef(_copy.data(), raw.size());
+}
+
+template class ExtendableStringArrayMultiValueReadView<const char*>;
+template class ExtendableStringArrayMultiValueReadView<multivalue::WeightedValue<const char*>>;
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h
new file mode 100644
index 00000000000..1ce94370a09
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h
@@ -0,0 +1,28 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/attribute/i_multi_value_read_view.h>
+
+namespace search::attribute {
+
+/**
+ * Read view for the data stored in an extendable multi-value string
+ * array attribute vector (used by streaming visitor) that handles
+ * optional addition of weight.
+ * @tparam MultiValueType The multi-value type of the data to access.
+ */
+template <typename MultiValueType>
+class ExtendableStringArrayMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType>
+{
+    vespalib::ConstArrayRef<char>       _buffer;
+    vespalib::ConstArrayRef<uint32_t>   _offsets;
+    vespalib::ConstArrayRef<uint32_t>   _idx;
+    mutable std::vector<MultiValueType> _copy;
+public:
+    ExtendableStringArrayMultiValueReadView(vespalib::ConstArrayRef<char> buffer, vespalib::ConstArrayRef<uint32_t> offsets, vespalib::ConstArrayRef<uint32_t> idx);
+    ~ExtendableStringArrayMultiValueReadView() override;
+    vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp
new file mode 100644
index 00000000000..9c515b7dccc
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp
@@ -0,0 +1,43 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "extendable_string_weighted_set_multi_value_read_view.h"
+
+namespace search::attribute {
+
+template <class MultiValueType>
+ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::ExtendableStringWeightedSetMultiValueReadView(vespalib::ConstArrayRef<char> buffer, vespalib::ConstArrayRef<uint32_t> offsets, vespalib::ConstArrayRef<uint32_t> idx, vespalib::ConstArrayRef<int32_t> weights)
+    : attribute::IMultiValueReadView<MultiValueType>(),
+      _buffer(buffer),
+      _offsets(offsets),
+      _idx(idx),
+      _weights(weights),
+      _copy()
+{
+}
+
+template <class MultiValueType>
+ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::~ExtendableStringWeightedSetMultiValueReadView() = default;
+
+template <class MultiValueType>
+vespalib::ConstArrayRef<MultiValueType>
+ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::get_values(uint32_t doc_id) const
+{
+    auto offset = _idx[doc_id];
+    auto next_offset = _idx[doc_id + 1];
+    vespalib::ConstArrayRef<uint32_t> raw(_offsets.data() + offset, next_offset - offset);
+    if (_copy.size() < raw.size()) {
+        _copy.resize(raw.size());
+    }
+    auto dst = _copy.data();
+    auto* src_weight = _weights.data() + offset;
+   for (auto &src : raw) {
+        *dst = multivalue::ValueBuilder<MultiValueType>::build(_buffer.data() + src, *src_weight);
+        ++src_weight;
+        ++dst;
+    }
+    return vespalib::ConstArrayRef(_copy.data(), raw.size());
+}
+
+template class ExtendableStringWeightedSetMultiValueReadView<multivalue::WeightedValue<const char*>>;
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h
new file mode 100644
index 00000000000..9f0ea8e9fa0
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h
@@ -0,0 +1,29 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchcommon/attribute/i_multi_value_read_view.h>
+
+namespace search::attribute {
+
+/**
+ * Read view for the data stored in an extendable multi-value string
+ * weighted set attribute vector (used by streaming visitor) that handles
+ * optional removal of weight.
+ * @tparam MultiValueType The multi-value type of the data to access.
+ */
+template <typename MultiValueType>
+class ExtendableStringWeightedSetMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType>
+{
+    vespalib::ConstArrayRef<char>       _buffer;
+    vespalib::ConstArrayRef<uint32_t>   _offsets;
+    vespalib::ConstArrayRef<uint32_t>   _idx;
+    vespalib::ConstArrayRef<int32_t>    _weights;
+    mutable std::vector<MultiValueType> _copy;
+public:
+    ExtendableStringWeightedSetMultiValueReadView(vespalib::ConstArrayRef<char> buffer, vespalib::ConstArrayRef<uint32_t> offsets, vespalib::ConstArrayRef<uint32_t> idx, vespalib::ConstArrayRef<int32_t> weights);
+    ~ExtendableStringWeightedSetMultiValueReadView() override;
+    vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp
index 12507a73564..9c944a70b94 100644
--- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp
@@ -2,6 +2,11 @@
 
 #include "extendableattributes.h"
 #include "attrvector.hpp"
+#include "extendable_numeric_array_multi_value_read_view.h"
+#include "extendable_numeric_weighted_set_multi_value_read_view.h"
+#include "extendable_string_array_multi_value_read_view.h"
+#include "extendable_string_weighted_set_multi_value_read_view.h"
+#include <vespa/vespalib/util/stash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.attribute.extendable_attributes");
@@ -41,6 +46,26 @@ bool SingleStringExtAttribute::add(const char * v, int32_t)
 
 //******************** CollectionType::ARRAY ********************//
 
+template <typename T>
+const attribute::IMultiValueAttribute*
+MultiExtAttribute<T>::as_multi_value_attribute() const
+{
+    return this;
+}
+
+template <typename T>
+const attribute::IMultiValueReadView<T>*
+MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const
+{
+    return &stash.create<attribute::ExtendableNumericArrayMultiValueReadView<T, T>>(this->_data, this->_idx);
+}
+
+template <typename T>
+const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>*
+MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const
+{
+    return &stash.create<attribute::ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<T>, T>>(this->_data, this->_idx);
+}
 
 MultiStringExtAttribute::MultiStringExtAttribute(const vespalib::string & name, const CollectionType & ctype) :
     StringDirectAttrVector< AttrVector::Features<true> >
@@ -79,6 +104,23 @@ bool MultiStringExtAttribute::add(const char * v, int32_t)
     return true;
 }
 
+const attribute::IMultiValueAttribute*
+MultiStringExtAttribute::as_multi_value_attribute() const
+{
+    return this;
+}
+
+const attribute::IMultiValueReadView<const char*>*
+MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const
+{
+    return &stash.create<attribute::ExtendableStringArrayMultiValueReadView<const char*>>(this->_buffer, this->_offsets, this->_idx);
+}
+
+const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>*
+MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const
+{
+    return &stash.create<attribute::ExtendableStringArrayMultiValueReadView<multivalue::WeightedValue<const char*>>>(this->_buffer, this->_offsets, this->_idx);
+}
 
 //******************** CollectionType::WSET ********************//
 
@@ -108,6 +150,12 @@ WeightedSetIntegerExtAttribute::get(DocId doc, AttributeVector::WeightedInt * v,
     return valueCount;
 }
 
+const attribute::IMultiValueReadView<multivalue::WeightedValue<int64_t>>*
+WeightedSetIntegerExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<int64_t>>, vespalib::Stash& stash) const
+{
+    return &stash.create<attribute::ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<int64_t>, int64_t>>(this->_data, this->_idx, this->get_weights());
+}
+
 WeightedSetFloatExtAttribute::WeightedSetFloatExtAttribute(const vespalib::string & name) :
     WeightedSetExtAttributeBase<MultiFloatExtAttribute>(name)
 {
@@ -134,6 +182,12 @@ WeightedSetFloatExtAttribute::get(DocId doc, AttributeVector::WeightedFloat * v,
     return valueCount;
 }
 
+const attribute::IMultiValueReadView<multivalue::WeightedValue<double>>*
+WeightedSetFloatExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<double>>, vespalib::Stash& stash) const
+{
+    return &stash.create<attribute::ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<double>, double>>(this->_data, this->_idx, this->get_weights());
+}
+
 WeightedSetStringExtAttribute::WeightedSetStringExtAttribute(const vespalib::string & name) :
     WeightedSetExtAttributeBase<MultiStringExtAttribute>(name)
 {
@@ -162,4 +216,16 @@ WeightedSetStringExtAttribute::get(DocId doc, AttributeVector::WeightedConstChar
     return getAllHelper(doc, v, sz);
 }
 
+const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>*
+WeightedSetStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const
+{
+    return &stash.create<attribute::ExtendableStringWeightedSetMultiValueReadView<multivalue::WeightedValue<const char*>>>(this->_buffer, this->_offsets, this->_idx, this->get_weights());
+}
+
+template class MultiExtAttribute<int8_t>;
+template class MultiExtAttribute<int16_t>;
+template class MultiExtAttribute<int32_t>;
+template class MultiExtAttribute<int64_t>;
+template class MultiExtAttribute<double>;
+
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h
index 5a1d005686e..ebd1d1d764c 100644
--- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h
+++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h
@@ -8,6 +8,7 @@
 
 #include "attrvector.h"
 #include "attrvector.hpp"
+#include <vespa/searchcommon/attribute/i_multi_value_attribute.h>
 
 namespace search {
 
@@ -99,7 +100,8 @@ public:
 template <typename T>
 class MultiExtAttribute
     : public NumericDirectAttrVector<AttrVector::Features<true>, typename AttributeTemplate<T>::Type>,
-      public IExtendAttribute
+      public IExtendAttribute,
+      public attribute::IMultiValueAttribute
 {
 protected:
     typedef typename MultiExtAttribute<T>::NumDirectAttrVec Super;
@@ -146,6 +148,11 @@ public:
     void onAddDocs(uint32_t lidLimit) override {
         this->_data.reserve(lidLimit);
     }
+    const attribute::IMultiValueAttribute* as_multi_value_attribute() const override;
+
+    // Implements attribute::IMultiValueAttribute
+    const attribute::IMultiValueReadView<T>* make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const override;
+    const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const override;
 };
 
 template <typename T>
@@ -161,7 +168,8 @@ typedef MultiInt64ExtAttribute MultiIntegerExtAttribute;
 
 class MultiStringExtAttribute :
     public StringDirectAttrVector< AttrVector::Features<true> >,
-    public IExtendAttribute
+    public IExtendAttribute,
+    public attribute::IMultiValueAttribute
 {
     IExtendAttribute * getExtendInterface() override { return this; }
 protected:
@@ -175,6 +183,10 @@ public:
         return false; // Emulate that this attribute is never loaded
     }
     void onAddDocs(DocId ) override { }
+    const attribute::IMultiValueAttribute* as_multi_value_attribute() const override;
+    // Implements attribute::IMultiValueAttribute
+    const attribute::IMultiValueReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const override;
+    const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override;
 };
 
 
@@ -198,6 +210,7 @@ protected:
         _weights()
     {}
     ~WeightedSetExtAttributeBase() {}
+    const std::vector<int32_t>& get_weights() const noexcept { return _weights; }
 };
 
 class WeightedSetIntegerExtAttribute
@@ -215,6 +228,8 @@ public:
     ~WeightedSetIntegerExtAttribute();
     bool add(int64_t v, int32_t w = 1) override;
     uint32_t get(DocId doc, AttributeVector::WeightedInt * v, uint32_t sz) const override;
+    // Implements attribute::IMultiValueAttribute
+    const attribute::IMultiValueReadView<multivalue::WeightedValue<int64_t>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<int64_t>>, vespalib::Stash& stash) const override;
 };
 
 class WeightedSetFloatExtAttribute
@@ -232,6 +247,8 @@ public:
     ~WeightedSetFloatExtAttribute();
     bool add(double v, int32_t w = 1) override;
     uint32_t get(DocId doc, AttributeVector::WeightedFloat * v, uint32_t sz) const override;
+    // Implements attribute::IMultiValueAttribute
+    const attribute::IMultiValueReadView<multivalue::WeightedValue<double>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<double>>, vespalib::Stash& stash) const override;
 };
 
 class WeightedSetStringExtAttribute
@@ -258,6 +275,8 @@ public:
     bool add(const char * v, int32_t w = 1) override;
     uint32_t get(DocId doc, AttributeVector::WeightedString * v, uint32_t sz) const override;
     uint32_t get(DocId doc, AttributeVector::WeightedConstChar * v, uint32_t sz) const override;
+    // Implements attribute::IMultiValueAttribute
+    const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override;
 };
 
 }  // namespace search