Support filtering of matched elements for multi-value attributes.

4 files changed, 192 insertions, 139 deletions

diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt
index cffdef25e5b..3ac95211aec 100644
--- a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt
+++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt
@@ -5,6 +5,7 @@ vespa_add_executable(searchsummary_attribute_combiner_test_app TEST
     attribute_combiner_test.cpp
     DEPENDS
     searchsummary
+    searchsummary_test
     GTest::GTest
 )
 vespa_add_test(NAME searchsummary_attribute_combiner_test_app COMMAND searchsummary_attribute_combiner_test_app)
diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp
index c64c5b2ed7d..eaeaa27f053 100644
--- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp
+++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp
@@ -1,19 +1,16 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/searchcommon/common/undefinedvalues.h>
-#include <vespa/searchlib/attribute/attributefactory.h>
-#include <vespa/searchlib/attribute/attributemanager.h>
 #include <vespa/searchlib/attribute/attributevector.h>
-#include <vespa/searchlib/attribute/attributevector.hpp>
-#include <vespa/searchlib/attribute/floatbase.h>
-#include <vespa/searchlib/attribute/integerbase.h>
-#include <vespa/searchlib/attribute/stringbase.h>
 #include <vespa/searchlib/common/matching_elements.h>
 #include <vespa/searchlib/common/matching_elements_fields.h>
 #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h>
+#include <vespa/searchsummary/docsummary/docsumfieldwriter.h>
 #include <vespa/searchsummary/docsummary/docsumstate.h>
 #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h>
 #include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h>
+#include <vespa/searchsummary/test/mock_attribute_manager.h>
+#include <vespa/searchsummary/test/mock_state_callback.h>
 #include <vespa/searchsummary/test/slime_value.h>
 #include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/gtest/gtest.h>
@@ -21,150 +18,24 @@
 #include <vespa/log/log.h>
 LOG_SETUP("attribute_combiner_test");
 
-using search::AttributeFactory;
-using search::AttributeManager;
-using search::AttributeVector;
-using search::FloatingPointAttribute;
-using search::IntegerAttribute;
-using search::MatchingElements;
-using search::StringAttribute;
 using search::attribute::BasicType;
-using search::attribute::CollectionType;
-using search::attribute::Config;
-using search::attribute::IAttributeVector;
 using search::attribute::getUndefined;
 using search::docsummary::AttributeCombinerDFW;
 using search::docsummary::GetDocsumsState;
 using search::docsummary::GetDocsumsStateCallback;
 using search::docsummary::IDocsumEnvironment;
 using search::docsummary::IDocsumFieldWriter;
+using search::docsummary::test::MockAttributeManager;
+using search::docsummary::test::MockStateCallback;
 using search::docsummary::test::SlimeValue;
 
 namespace {
 
-struct AttributeManagerFixture
-{
-    AttributeManager mgr;
-
-    AttributeManagerFixture();
-
-    ~AttributeManagerFixture();
-
-    template <typename AttributeType, typename ValueType>
-    void
-    buildAttribute(const vespalib::string &name,
-                   BasicType type,
-                   std::vector<std::vector<ValueType>> values);
-
-    void
-    buildStringAttribute(const vespalib::string &name,
-                         std::vector<std::vector<vespalib::string>> values);
-    void
-    buildFloatAttribute(const vespalib::string &name,
-                        std::vector<std::vector<double>> values);
-
-    void
-    buildIntegerAttribute(const vespalib::string &name,
-                          BasicType type,
-                          std::vector<std::vector<IAttributeVector::largeint_t>> values);
-};
-
-AttributeManagerFixture::AttributeManagerFixture()
-    : mgr()
-{
-    buildStringAttribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}});
-    buildIntegerAttribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}});
-    buildFloatAttribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}});
-    buildStringAttribute("smap.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1", "k3.2"}, {"", "k4.2"}, {}});
-    buildStringAttribute("smap.value.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}});
-    buildIntegerAttribute("smap.value.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}});
-    buildFloatAttribute("smap.value.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}});
-    buildStringAttribute("map.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1"}, {"", "k4.2"}, {}});
-    buildStringAttribute("map.value", {{"n1.1", "n1.2"}, {}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}});
-}
-
-AttributeManagerFixture::~AttributeManagerFixture() = default;
-
-template <typename AttributeType, typename ValueType>
-void
-AttributeManagerFixture::buildAttribute(const vespalib::string &name,
-                                        BasicType type,
-                                        std::vector<std::vector<ValueType>> values)
-{
-    Config cfg(type, CollectionType::Type::ARRAY);
-    auto attrBase = AttributeFactory::createAttribute(name, cfg);
-    EXPECT_TRUE(attrBase);
-    auto attr = std::dynamic_pointer_cast<AttributeType>(attrBase);
-    EXPECT_TRUE(attr);
-    attr->addReservedDoc();
-    for (const auto &docValues : values) {
-        uint32_t docId = 0;
-        EXPECT_TRUE(attr->addDoc(docId));
-        EXPECT_NE(0u, docId);
-        for (const auto &value : docValues) {
-            attr->append(docId, value, 1);
-        }
-        attr->commit();
-    }
-    EXPECT_TRUE(mgr.add(attr));
-}
-
-void
-AttributeManagerFixture::buildStringAttribute(const vespalib::string &name,
-                                              std::vector<std::vector<vespalib::string>> values)
-{
-    buildAttribute<StringAttribute, vespalib::string>(name, BasicType::Type::STRING, std::move(values));
-}
-
-void
-AttributeManagerFixture::buildFloatAttribute(const vespalib::string &name,
-                                             std::vector<std::vector<double>> values)
-{
-    buildAttribute<FloatingPointAttribute, double>(name, BasicType::Type::DOUBLE, std::move(values));
-}
-
-void
-AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name,
-                                               BasicType type,
-                                               std::vector<std::vector<IAttributeVector::largeint_t>> values)
-{
-    buildAttribute<IntegerAttribute, IAttributeVector::largeint_t>(name, type, std::move(values));
-}
-
-
-class DummyStateCallback : public GetDocsumsStateCallback
-{
-public:
-    MatchingElements _matching_elements;
-
-    DummyStateCallback();
-    void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { }
-    void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { }
-    void ParseLocation(GetDocsumsState *) override { }
-    std::unique_ptr<MatchingElements> fill_matching_elements(const search::MatchingElementsFields &) override { return std::make_unique<MatchingElements>(_matching_elements); }
-    ~DummyStateCallback() override { }
-};
-
-DummyStateCallback::DummyStateCallback()
-    : GetDocsumsStateCallback(),
-      _matching_elements()
-{
-    _matching_elements.add_matching_elements(1, "array", {1});
-    _matching_elements.add_matching_elements(3, "array", {0});
-    _matching_elements.add_matching_elements(4, "array", {1});
-    _matching_elements.add_matching_elements(1, "smap", {1});
-    _matching_elements.add_matching_elements(3, "smap", {0});
-    _matching_elements.add_matching_elements(4, "smap", {1});
-    _matching_elements.add_matching_elements(1, "map", {1});
-    _matching_elements.add_matching_elements(3, "map", {0});
-    _matching_elements.add_matching_elements(4, "map", {1});
-}
-
 struct AttributeCombinerTest : public ::testing::Test
 {
-    AttributeManagerFixture             attrs;
+    MockAttributeManager                attrs;
     std::unique_ptr<IDocsumFieldWriter> writer;
-    DummyStateCallback                  stateCallback;
+    MockStateCallback                   callback;
     GetDocsumsState                     state;
     std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields;
 
@@ -177,11 +48,31 @@ struct AttributeCombinerTest : public ::testing::Test
 AttributeCombinerTest::AttributeCombinerTest()
     : attrs(),
       writer(),
-      stateCallback(),
-      state(stateCallback),
+      callback(),
+      state(callback),
       _matching_elems_fields()
 {
-    state._attrCtx = attrs.mgr.createContext();
+    attrs.build_string_attribute("array.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}});
+    attrs.build_int_attribute("array.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}});
+    attrs.build_float_attribute("array.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}});
+    attrs.build_string_attribute("smap.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1", "k3.2"}, {"", "k4.2"}, {}});
+    attrs.build_string_attribute("smap.value.name", {{"n1.1", "n1.2"}, {"n2"}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}});
+    attrs.build_int_attribute("smap.value.val", BasicType::Type::INT8, {{ 10, 11}, {20, 21 }, {30}, { getUndefined<int8_t>(), 41}, {}});
+    attrs.build_float_attribute("smap.value.fval", {{ 110.0}, { 120.0, 121.0 }, { 130.0, 131.0}, { getUndefined<double>(), 141.0 }, {}});
+    attrs.build_string_attribute("map.key", {{"k1.1", "k1.2"}, {"k2"}, {"k3.1"}, {"", "k4.2"}, {}});
+    attrs.build_string_attribute("map.value", {{"n1.1", "n1.2"}, {}, {"n3.1", "n3.2"}, {"", "n4.2"}, {}});
+
+    callback.add_matching_elements(1, "array", {1});
+    callback.add_matching_elements(3, "array", {0});
+    callback.add_matching_elements(4, "array", {1});
+    callback.add_matching_elements(1, "smap", {1});
+    callback.add_matching_elements(3, "smap", {0});
+    callback.add_matching_elements(4, "smap", {1});
+    callback.add_matching_elements(1, "map", {1});
+    callback.add_matching_elements(3, "map", {0});
+    callback.add_matching_elements(4, "map", {1});
+
+    state._attrCtx = attrs.mgr().createContext();
 }
 
 AttributeCombinerTest::~AttributeCombinerTest() = default;
diff --git a/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt
new file mode 100644
index 00000000000..8bdf30273f2
--- /dev/null
+++ b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+find_package(GTest REQUIRED)
+vespa_add_executable(searchsummary_attributedfw_test_app TEST
+    SOURCES
+    attributedfw_test.cpp
+    DEPENDS
+    searchsummary
+    searchsummary_test
+    GTest::GTest
+)
+vespa_add_test(NAME searchsummary_attributedfw_test_app COMMAND searchsummary_attributedfw_test_app)
diff --git a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp
new file mode 100644
index 00000000000..7bea92ec8f3
--- /dev/null
+++ b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp
@@ -0,0 +1,150 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchlib/common/matching_elements_fields.h>
+#include <vespa/searchsummary/docsummary/attributedfw.h>
+#include <vespa/searchsummary/test/mock_attribute_manager.h>
+#include <vespa/searchsummary/test/mock_state_callback.h>
+#include <vespa/searchsummary/test/slime_value.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("attributedfw_test");
+
+using search::MatchingElements;
+using search::MatchingElementsFields;
+using search::attribute::BasicType;
+using search::attribute::CollectionType;
+using search::docsummary::AttributeDFWFactory;
+using search::docsummary::GetDocsumsState;
+using search::docsummary::IDocsumFieldWriter;
+using search::docsummary::test::MockAttributeManager;
+using search::docsummary::test::MockStateCallback;
+using search::docsummary::test::SlimeValue;
+
+using ElementVector = std::vector<uint32_t>;
+
+class AttributeDFWTest : public ::testing::Test {
+protected:
+    MockAttributeManager _attrs;
+    std::unique_ptr<IDocsumFieldWriter> _writer;
+    MockStateCallback _callback;
+    GetDocsumsState _state;
+    std::shared_ptr<search::MatchingElementsFields> _matching_elems_fields;
+    vespalib::string _field_name;
+
+public:
+    AttributeDFWTest()
+        : _attrs(),
+          _writer(),
+          _callback(),
+          _state(_callback),
+          _matching_elems_fields(),
+          _field_name()
+    {
+        _attrs.build_string_attribute("array_str", { {"a", "b", "c"}, {} });
+        _attrs.build_int_attribute("array_int", BasicType::INT32, { {10, 20, 30}, {} });
+        _attrs.build_float_attribute("array_float", { {10.5, 20.5, 30.5}, {} });
+
+        _attrs.build_string_attribute("wset_str", { {"a", "b", "c"}, {} }, CollectionType::WSET);
+        _attrs.build_int_attribute("wset_int", BasicType::INT32, { {10, 20, 30}, {} }, CollectionType::WSET);
+        _attrs.build_float_attribute("wset_float", { {10.5, 20.5, 30.5}, {} }, CollectionType::WSET);
+
+        _state._attrCtx = _attrs.mgr().createContext();
+    }
+    ~AttributeDFWTest() {}
+
+    void setup(const vespalib::string& field_name, bool filter_elements) {
+        if (filter_elements) {
+            _matching_elems_fields = std::make_shared<MatchingElementsFields>();
+        }
+        _writer = AttributeDFWFactory::create(_attrs.mgr(), field_name, filter_elements, _matching_elems_fields);
+        _writer->setIndex(0);
+        _field_name = field_name;
+        _state._attributes.resize(1);
+        _state._attributes[0] = _state._attrCtx->getAttribute(field_name);
+    }
+
+    void expect_field(const vespalib::string& exp_slime_as_json, uint32_t docid) {
+        vespalib::Slime act;
+        vespalib::slime::SlimeInserter inserter(act);
+        _writer->insertField(docid, nullptr, &_state, search::docsummary::RES_JSONSTRING, inserter);
+
+        SlimeValue exp(exp_slime_as_json);
+        EXPECT_EQ(exp.slime, act);
+    }
+
+    void expect_filtered(const ElementVector& matching_elems, const std::string& exp_slime_as_json, uint32_t docid = 1) {
+        _callback.clear();
+        _callback.add_matching_elements(docid, _field_name, matching_elems);
+        _state._matching_elements = std::unique_ptr<MatchingElements>();
+        expect_field(exp_slime_as_json, docid);
+    }
+};
+
+TEST_F(AttributeDFWTest, outputs_slime_for_array_of_string)
+{
+    setup("array_str", false);
+    expect_field("[ 'a', 'b', 'c' ]", 1);
+    expect_field("null", 2);
+}
+
+TEST_F(AttributeDFWTest, outputs_slime_for_array_of_int)
+{
+    setup("array_int", false);
+    expect_field("[ 10, 20, 30 ]", 1);
+    expect_field("null", 2);
+}
+
+TEST_F(AttributeDFWTest, outputs_slime_for_array_of_float)
+{
+    setup("array_float", false);
+    expect_field("[ 10.5, 20.5, 30.5 ]", 1);
+    expect_field("null", 2);
+}
+
+TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_string)
+{
+    setup("wset_str", false);
+    expect_field("[ {'item':'a', 'weight':1}, {'item':'b', 'weight':1}, {'item':'c', 'weight':1} ]", 1);
+    expect_field("null", 2);
+}
+
+TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_int)
+{
+    setup("wset_int", false);
+    expect_field("[ {'item':10, 'weight':1}, {'item':20, 'weight':1}, {'item':30, 'weight':1} ]", 1);
+    expect_field("null", 2);
+}
+
+TEST_F(AttributeDFWTest, outputs_slime_for_wset_of_float)
+{
+    setup("wset_float", false);
+    expect_field("[ {'item':10.5, 'weight':1}, {'item':20.5, 'weight':1}, {'item':30.5, 'weight':1} ]", 1);
+    expect_field("null", 2);
+}
+
+TEST_F(AttributeDFWTest, matched_elements_fields_is_populated)
+{
+    setup("array_str", true);
+    EXPECT_TRUE(_matching_elems_fields->has_field("array_str"));
+}
+
+TEST_F(AttributeDFWTest, filteres_matched_elements_in_array_attribute)
+{
+    setup("array_str", true);
+    expect_filtered({}, "[]");
+    expect_filtered({0}, "[ 'a' ]");
+    expect_filtered({1, 2}, "[ 'b', 'c' ]");
+    expect_filtered({3}, "[]");
+}
+
+TEST_F(AttributeDFWTest, filteres_matched_elements_in_wset_attribute)
+{
+    setup("wset_str", true);
+    expect_filtered({}, "[]");
+    expect_filtered({0}, "[ {'item':'a', 'weight':1} ]");
+    expect_filtered({1, 2}, "[ {'item':'b', 'weight':1}, {'item':'c', 'weight':1} ]");
+    expect_filtered({3}, "[]");
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()