Change matched elements filter docsum field writer to insert an empty

array if input field is empty or not set. This was the old behavior
when jsonstring entries were stored in docsum blobs.

2 files changed, 58 insertions, 16 deletions

diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
index 22b3ae69165..623b66bad18 100644
--- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
+++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
@@ -70,6 +70,8 @@ private:
     ArrayDataType _array_type;
     MapDataType _map_type;
     WeightedSetDataType _wset_type;
+    bool _empty_values;
+    bool _skip_set_values;
 
     StructFieldValue::UP make_elem_value(const std::string& name, int weight) const {
         auto result = std::make_unique<StructFieldValue>(*_elem_type);
@@ -85,7 +87,9 @@ public:
           _elem_type(make_struct_elem_type()),
           _array_type(*_elem_type),
           _map_type(*DataType::STRING, *_elem_type),
-          _wset_type(*DataType::STRING, false, false)
+          _wset_type(*DataType::STRING, false, false),
+          _empty_values(false),
+          _skip_set_values(false)
     {
         _doc_type.addField(Field("array", _array_type));
         _doc_type.addField(Field("map", _map_type));
@@ -105,32 +109,50 @@ public:
         auto doc = std::make_unique<Document>(_doc_type, DocumentId("id:test:test::0"));
         {
             ArrayFieldValue array_value(_array_type);
-            array_value.append(make_elem_value("a", 3));
-            array_value.append(make_elem_value("b", 5));
-            array_value.append(make_elem_value("c", 7));
-            doc->setValue("array", array_value);
+            if (!_empty_values) {
+                array_value.append(make_elem_value("a", 3));
+                array_value.append(make_elem_value("b", 5));
+                array_value.append(make_elem_value("c", 7));
+            }
+            if (!_skip_set_values) {
+                doc->setValue("array", array_value);
+            }
         }
         {
             MapFieldValue map_value(_map_type);
-            map_value.put(StringFieldValue("a"), *make_elem_value("a", 3));
-            map_value.put(StringFieldValue("b"), *make_elem_value("b", 5));
-            map_value.put(StringFieldValue("c"), *make_elem_value("c", 7));
-            doc->setValue("map", map_value);
+            if (!_empty_values) {
+                map_value.put(StringFieldValue("a"), *make_elem_value("a", 3));
+                map_value.put(StringFieldValue("b"), *make_elem_value("b", 5));
+                map_value.put(StringFieldValue("c"), *make_elem_value("c", 7));
+            }
+            if (!_skip_set_values) {
+                doc->setValue("map", map_value);
+            }
         }
         {
             MapFieldValue map2_value(_map_type);
-            map2_value.put(StringFieldValue("dummy"), *make_elem_value("dummy", 2));
-            doc->setValue("map2", map2_value);
+            if (!_empty_values) {
+                map2_value.put(StringFieldValue("dummy"), *make_elem_value("dummy", 2));
+            }
+            if (!_skip_set_values) {
+                doc->setValue("map2", map2_value);
+            }
         }
         {
             WeightedSetFieldValue wset_value(_wset_type);
-            wset_value.add(StringFieldValue("a"), 13);
-            wset_value.add(StringFieldValue("b"), 15);
-            wset_value.add(StringFieldValue("c"), 17);
-            doc->setValue("wset", wset_value);
+            if (!_empty_values) {
+                wset_value.add(StringFieldValue("a"), 13);
+                wset_value.add(StringFieldValue("b"), 15);
+                wset_value.add(StringFieldValue("c"), 17);
+            }
+            if (!_skip_set_values) {
+                doc->setValue("wset", wset_value);
+            }
         }
         return std::make_unique<DocsumStoreDocument>(std::move(doc));
     }
+    void set_empty_values() { _empty_values = true; }
+    void set_skip_set_values() { _skip_set_values = true; }
 };
 
 DocsumStore::~DocsumStore() = default;
@@ -218,7 +240,9 @@ public:
         EXPECT_EQ(exp.slime, act);
     }
     const MatchingElementsFields& fields() const { return *_fields; }
-};
+    void set_empty_values() { _doc_store.set_empty_values(); }
+    void set_skip_set_values() { _doc_store.set_skip_set_values(); }
+ };
 
 MatchedElementsFilterTest::~MatchedElementsFilterTest() = default;
 
@@ -232,6 +256,10 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value)
                                         "{'name':'b','weight':5},"
                                         "{'name':'c','weight':7}]");
     expect_filtered("array", {0, 1, 100}, "[]");
+    set_empty_values();
+    expect_filtered("array", {}, "[]");
+    set_skip_set_values();
+    expect_filtered("array", {}, "[]");
 }
 
 TEST_F(MatchedElementsFilterTest, matching_elements_fields_is_setup_for_array_field_value)
@@ -252,6 +280,10 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value)
                                       "{'key':'b','value':{'name':'b','weight':5}},"
                                       "{'key':'c','value':{'name':'c','weight':7}}]");
     expect_filtered("map", {0, 1, 100}, "[]");
+    set_empty_values();
+    expect_filtered("map", {}, "[]");
+    set_skip_set_values();
+    expect_filtered("map", {}, "[]");
 }
 
 TEST_F(MatchedElementsFilterTest, filter_elements_in_weighed_set_field_value)
@@ -262,6 +294,10 @@ TEST_F(MatchedElementsFilterTest, filter_elements_in_weighed_set_field_value)
     expect_filtered("wset", {2}, "[{'item':'c','weight':17}]");
     expect_filtered("wset", {0, 1, 2}, "[{'item':'a','weight':13},{'item':'b','weight':15},{'item':'c','weight':17}]");
     expect_filtered("wset", {0, 1, 100}, "[]");
+    set_empty_values();
+    expect_filtered("wset", {}, "[]");
+    set_skip_set_values();
+    expect_filtered("wset", {}, "[]");
 }
 
 TEST_F(MatchedElementsFilterTest, matching_elements_fields_is_setup_for_map_field_value)
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp
index e83860ed8bf..7bec9cb6365 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp
@@ -76,6 +76,10 @@ filter_matching_elements_in_input_field_while_converting_to_slime(const FieldVal
     assert(converted->isLiteral());
     auto& literal = static_cast<const LiteralFieldValueB&>(*converted);
     vespalib::stringref buf = literal.getValueRef();
+    if (buf.empty()) {
+        target.insertArray(0);
+        return;
+    }
     Slime input_field_as_slime;
     BinaryFormat::decode(vespalib::Memory(buf.data(), buf.size()), input_field_as_slime);
     inject(input_field_as_slime.get(), target);
@@ -91,6 +95,8 @@ MatchedElementsFilterDFW::insertField(uint32_t docid, const IDocsumStoreDocument
     auto field_value = doc->get_field_value(_input_field_name);
     if (field_value) {
         filter_matching_elements_in_input_field_while_converting_to_slime(*field_value, get_matching_elements(docid, *state), target);
+    } else {
+        target.insertArray(0);
     }
 }