Add support for "matched-elements-only" flag on complex summary fields.

Supported field types are array of simple struct, map of primitive type to simple struct
and map of primitive type to primitive type.

5 files changed, 223 insertions, 1 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java
new file mode 100644
index 00000000000..8bb834d4697
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java
@@ -0,0 +1,64 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes;
+import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField;
+
+/**
+ * Iterates all summary fields with 'matched-elements-only' and adjusts transform (if all struct-fields are attributes)
+ * and validates that the field type is supported.
+ *
+ * @author geirst
+ */
+public class MatchedElementsOnlyResolver extends Processor {
+
+    public MatchedElementsOnlyResolver(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process(boolean validate, boolean documentsOnly) {
+        for (var entry : search.getSummaries().entrySet()) {
+            var summary = entry.getValue();
+            for (var field : summary.getSummaryFields()) {
+                if (field.getTransform().equals(SummaryTransform.MATCHED_ELEMENTS_FILTER)) {
+                    processSummaryField(summary, field, validate);
+                }
+            }
+        }
+    }
+
+    private void processSummaryField(DocumentSummary summary, SummaryField field, boolean validate) {
+        var sourceField = search.getField(field.getSingleSource());
+        if (sourceField != null) {
+            if (isSupportedComplexField(sourceField)) {
+                if (isComplexFieldWithOnlyStructFieldAttributes(sourceField)) {
+                    field.setTransform(SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER);
+                }
+            } else if (validate) {
+                fail(summary, field, "'matched-elements-only' is not supported for this field type. " +
+                        "Supported field types are array of simple struct, map of primitive type to simple struct, " +
+                        "and map of primitive type to primitive type");
+            }
+        }
+        // else case is handled in SummaryFieldsMustHaveValidSource
+    }
+
+    private void fail(DocumentSummary summary, SummaryField field, String msg) {
+        throw new IllegalArgumentException(formatError(search, summary, field, msg));
+    }
+
+    private String formatError(Search search, DocumentSummary summary, SummaryField field, String msg) {
+        return "For search '" + search.getName() + "', document summary '" + summary.getName()
+                + "', summary field '" + field.getName() + "': " + msg;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 47433479588..b0ba8e30f06 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -46,6 +46,7 @@ public class Processing {
                 SummaryConsistency::new,
                 SummaryNamesFieldCollisions::new,
                 SummaryFieldsMustHaveValidSource::new,
+                MatchedElementsOnlyResolver::new,
                 AddAttributeTransformToSummaryOfImportedFields::new,
                 MakeDefaultSummaryTheSuperSet::new,
                 Bolding::new,
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
index 4f85dd0c84e..4279003bb9d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
@@ -20,7 +20,9 @@ public enum SummaryTransform {
     SUMMARYFEATURES("summaryfeatures"),
     TEXTEXTRACTOR("textextractor"),
     GEOPOS("geopos"),
-    ATTRIBUTECOMBINER("attributecombiner");
+    ATTRIBUTECOMBINER("attributecombiner"),
+    MATCHED_ELEMENTS_FILTER("matchedelementsfilter"),
+    MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter");
 
     private String name;
 
@@ -88,6 +90,7 @@ public enum SummaryTransform {
         case RANKFEATURES:
         case SUMMARYFEATURES:
         case ATTRIBUTECOMBINER:
+        case MATCHED_ATTRIBUTE_ELEMENTS_FILTER:
             return true;
 
         default:
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index db22e73268c..e560d78a116 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -241,6 +241,7 @@ TOKEN :
 | < FULL: "full" >
 | < STATIC: "static" >
 | < DYNAMIC: "dynamic" >
+| < MATCHEDELEMENTSONLY: "matched-elements-only" >
 | < SSCONTEXTUAL: "contextual" >
 | < SSOVERRIDE: "override" >
 | < SSTITLE: "title" >
@@ -1312,6 +1313,7 @@ SummaryInFieldOperation summaryInFieldShort(FieldOperationContainer field) :
     <COLON> ( <DYNAMIC> { op.setTransform(SummaryTransform.DYNAMICTEASER);
                           op.addSource(name);
                         }
+              | <MATCHEDELEMENTSONLY> { op.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); }
               | (<FULL> | <STATIC>) { op.setTransform(SummaryTransform.NONE); } )
     { return op; }
 }
@@ -1362,6 +1364,7 @@ Object summaryItem(SummaryInFieldLongOperation field) : { }
 Object summaryTransform(SummaryInFieldOperation field) : { }
 {
     ( <DYNAMIC>             { field.setTransform(SummaryTransform.DYNAMICTEASER); }
+     | <MATCHEDELEMENTSONLY> { field.setTransform(SummaryTransform.MATCHED_ELEMENTS_FILTER); }
      | (<FULL> | <STATIC>) { field.setTransform(SummaryTransform.NONE); } )
     { return null; }
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java
new file mode 100644
index 00000000000..3b6918c04ae
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java
@@ -0,0 +1,151 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author geirst
+ */
+public class MatchedElementsOnlyResolverTestCase {
+
+    @Rule
+    public final ExpectedException exceptionRule = ExpectedException.none();
+
+    @Test
+    public void complex_field_with_some_struct_field_attributes_gets_default_transform() throws ParseException {
+        assertSummaryField(joinLines("field my_field type map<string, string> {",
+                "  indexing: summary",
+                "  summary: matched-elements-only",
+                "  struct-field key { indexing: attribute }",
+                "}"),
+                "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER);
+
+        assertSummaryField(joinLines("field my_field type map<string, elem> {",
+                "  indexing: summary",
+                "  summary: matched-elements-only",
+                "  struct-field key { indexing: attribute }",
+                "}"),
+                "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER);
+
+        assertSummaryField(joinLines("field my_field type array<elem> {",
+                "  indexing: summary",
+                "  summary: matched-elements-only",
+                "  struct-field name { indexing: attribute }",
+                "}"),
+                "my_field", SummaryTransform.MATCHED_ELEMENTS_FILTER);
+    }
+
+    @Test
+    public void complex_field_with_only_struct_field_attributes_gets_attribute_transform() throws ParseException {
+        assertSummaryField(joinLines("field my_field type map<string, string> {",
+                "  indexing: summary",
+                "  summary: matched-elements-only",
+                "  struct-field key { indexing: attribute }",
+                "  struct-field value { indexing: attribute }",
+                "}"),
+                "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER);
+
+        assertSummaryField(joinLines("field my_field type map<string, elem> {",
+                "  indexing: summary",
+                "  summary: matched-elements-only",
+                "  struct-field key { indexing: attribute }",
+                "  struct-field value.name { indexing: attribute }",
+                "  struct-field value.weight { indexing: attribute }",
+                "}"),
+                "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER);
+
+        assertSummaryField(joinLines("field my_field type array<elem> {",
+                "  indexing: summary",
+                "  summary: matched-elements-only",
+                "  struct-field name { indexing: attribute }",
+                "  struct-field weight { indexing: attribute }",
+                "}"),
+                "my_field", SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER);
+    }
+
+    @Test
+    public void explicit_summary_field_can_use_filter_transform_with_reference_to_source_field() throws ParseException {
+        String documentSummary = joinLines("document-summary my_summary {",
+                "  summary my_filter_field type map<string, string> {",
+                "    source: my_field",
+                "    matched-elements-only",
+                "  }",
+                "}");
+        {
+            var search = buildSearch(joinLines("field my_field type map<string, string> {",
+                    "  indexing: summary",
+                    "  struct-field key { indexing: attribute }",
+                    "}"),
+                    documentSummary);
+            assertSummaryField(search.getSummaryField("my_filter_field"),
+                    SummaryTransform.MATCHED_ELEMENTS_FILTER, "my_field");
+            assertSummaryField(search.getSummaryField("my_field"),
+                    SummaryTransform.NONE, "my_field");
+        }
+        {
+            var search = buildSearch(joinLines("field my_field type map<string, string> {",
+                    "  indexing: summary",
+                    "  struct-field key { indexing: attribute }",
+                    "  struct-field value { indexing: attribute }",
+                    "}"),
+                    documentSummary);
+            assertSummaryField(search.getSummaryField("my_filter_field"),
+                    SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER, "my_field");
+            assertSummaryField(search.getSummaryField("my_field"),
+                    SummaryTransform.ATTRIBUTECOMBINER, "my_field");
+        }
+    }
+
+    @Test
+    public void unsupported_field_type_throws() throws ParseException {
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage("For search 'test', document summary 'default', summary field 'my_field': " +
+                "'matched-elements-only' is not supported for this field type. " +
+                "Supported field types are array of simple struct, map of primitive type to simple struct, and map of primitive type to primitive type");
+        buildSearch(joinLines("field my_field type string {",
+                "  indexing: summary",
+                "  summary: matched-elements-only",
+                "}"));
+    }
+
+    private void assertSummaryField(String fieldContent, String fieldName, SummaryTransform expTransform) throws ParseException {
+        var search = buildSearch(fieldContent);
+        assertSummaryField(search.getSummaryField(fieldName), expTransform, fieldName);
+    }
+
+    private void assertSummaryField(SummaryField field, SummaryTransform expTransform, String expSourceField) {
+        assertEquals(expTransform, field.getTransform());
+        assertEquals(expSourceField, field.getSingleSource());
+    }
+
+    private Search buildSearch(String field) throws ParseException {
+        return buildSearch(field, "");
+    }
+
+    private Search buildSearch(String field, String summary) throws ParseException {
+        var builder = new SearchBuilder(new RankProfileRegistry());
+        builder.importString(joinLines("search test {",
+                "  document test {",
+                "    struct elem {",
+                "      field name type string {}",
+                "      field weight type int {}",
+                "    }",
+                field,
+                "  }",
+                summary,
+                "}"));
+        builder.build();
+        return builder.getSearch();
+    }
+}