Add attribute-tokens summary transform to config model.

8 files changed, 52 insertions, 9 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
index 300a55e521a..799324d26e9 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
@@ -156,7 +156,8 @@ public class SummaryClass extends Derived {
                 summaryField.getTransform() == SummaryTransform.POSITIONS ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER ||
-                summaryField.getTransform() == SummaryTransform.TOKENS)
+                summaryField.getTransform() == SummaryTransform.TOKENS ||
+                summaryField.getTransform() == SummaryTransform.ATTRIBUTE_TOKENS)
         {
             return summaryField.getSingleSource();
         } else if (summaryField.getTransform().isDynamic()) {
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
index 2f60cd8eb06..349fd06ebdd 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
@@ -94,6 +94,8 @@ public class SummaryClassField {
                 return Type.FEATUREDATA;
             } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) {
                 return Type.JSONSTRING;
+            } else if (transform != null && transform.equals(SummaryTransform.ATTRIBUTE_TOKENS)) {
+                return Type.JSONSTRING;
             } else {
                 return Type.LONGSTRING;
             }
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java
index dd6f118d113..fe26ada5c8b 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java
@@ -36,6 +36,7 @@ public class AdjustSummaryTransforms extends Processor {
                 makeDocumentIdTransformIfAppropriate(summaryField);
                 makeAttributeTransformIfAppropriate(summaryField, schema);
                 makeAttributeCombinerTransformIfAppropriate(summaryField, schema);
+                makeAttributeTokensTransformIfAppropriate(summaryField, summary.getName(), schema);
                 makeCopyTransformIfAppropriate(summaryField, schema);
             }
         }
@@ -67,6 +68,25 @@ public class AdjustSummaryTransforms extends Processor {
         }
     }
 
+    private void makeAttributeTokensTransformIfAppropriate(SummaryField summaryField, String docsumName, Schema schema) {
+        if (summaryField.getTransform() == SummaryTransform.TOKENS) {
+            String sourceFieldName = summaryField.getSingleSource();
+            Attribute attribute = schema.getAttribute(sourceFieldName);
+            ImmutableSDField source = schema.getField(sourceFieldName);
+            if (!source.doesIndexing()) {
+                if (attribute != null) {
+                    summaryField.setTransform(SummaryTransform.ATTRIBUTE_TOKENS);
+                } else {
+                    throw new IllegalArgumentException("For schema '" + schema.getName() +
+                            "', document-summary '" + docsumName +
+                            "', summary field '" + summaryField.getName() +
+                            "', source field '" + sourceFieldName +
+                            "': tokens summary field setting requires index or attribute for source field");
+                }
+            }
+        }
+    }
+
     /*
      * This function must be called after makeAttributeCombinerTransformIfAppropriate().
      */
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
index e4116c3f9d5..3dd2ffdb02e 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
@@ -79,7 +79,8 @@ public class IndexingOutputs extends Processor {
             }
             dynamicSummary.add(summaryName);
         } else if (summaryTransform != SummaryTransform.ATTRIBUTE &&
-                summaryTransform != SummaryTransform.TOKENS) {
+                summaryTransform != SummaryTransform.TOKENS &&
+                summaryTransform != SummaryTransform.ATTRIBUTE_TOKENS) {
             staticSummary.add(summaryName);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java
index 420df3ee575..6ee82b31d57 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java
@@ -42,6 +42,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor {
             if (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER) continue;
             if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) continue;
             if (summaryField.getTransform() == SummaryTransform.TOKENS) continue;
+            if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE_TOKENS) continue;
 
             defaultSummary.add(summaryField.clone());
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
index 58f47680f9f..36d300a9b0b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
@@ -24,7 +24,8 @@ public enum SummaryTransform {
     MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"),
     COPY("copy"),
     DOCUMENT_ID("documentid"),
-    TOKENS("tokens");
+    TOKENS("tokens"),
+    ATTRIBUTE_TOKENS("attribute-tokens");
 
     private final String name;
 
@@ -69,7 +70,7 @@ public enum SummaryTransform {
         return this==DYNAMICBOLDED || this==DYNAMICTEASER;
     }
 
-    public boolean isTokens() { return this == TOKENS; }
+    public boolean isTokens() { return this == TOKENS || this == ATTRIBUTE_TOKENS; }
 
     /** Returns whether this transform always gets its value by accessing memory only */
     public boolean isInMemory() {
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
index 5019ed0dd60..5e02f63ec39 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java
@@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 /**
  * Tests summary extraction
@@ -226,9 +227,10 @@ public class SummaryTestCase extends AbstractSchemaTestCase {
         assertOverride(schema, "documentid", SummaryTransform.DOCUMENT_ID.getName(), "", "bar");
     }
 
-    @Test
-    void tokens_override() throws ParseException {
-        var schema = buildSchema("field foo type string { indexing: summary }",
+    private void check_tokens_override(boolean index, boolean attribute, SummaryTransform exp) throws ParseException {
+        var schema = buildSchema("field foo type string { indexing: " +
+                (index ? "index | " : "") +
+                (attribute ? "attribute | " : "") + "summary }",
                 joinLines("document-summary bar {",
                         "    summary baz {",
                         "        source: foo ",
@@ -236,11 +238,26 @@ public class SummaryTestCase extends AbstractSchemaTestCase {
                         "     }",
                         "    from-disk",
                         "}"));
-        assertOverride(schema, "baz", SummaryTransform.TOKENS.getName(), "foo", "bar");
+        assertOverride(schema, "baz", exp.getName(), "foo", "bar");
         assert(!schema.getSummary("default").getSummaryFields().containsKey("baz"));
     }
 
     @Test
+    void tokens_override() throws ParseException {
+        try {
+            check_tokens_override(false, false, SummaryTransform.TOKENS);
+            fail("Expected IllegalArgumentException");
+        } catch (IllegalArgumentException e) {
+            assertEquals("For schema 'test', document-summary 'bar'" +
+                    ", summary field 'baz', source field 'foo'" +
+                    ": tokens summary field setting requires index or attribute for source field", e.getMessage());
+        }
+        check_tokens_override(false, true, SummaryTransform.ATTRIBUTE_TOKENS);
+        check_tokens_override(true, false, SummaryTransform.TOKENS);
+        check_tokens_override(true, true, SummaryTransform.TOKENS);
+    }
+
+    @Test
     void documentid_summary_transform_requires_disk_access() {
         assertFalse(SummaryTransform.DOCUMENT_ID.isInMemory());
     }
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java
index 6ca62321617..6da536efe86 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/TokensTransformValidatorTest.java
@@ -17,7 +17,7 @@ public class TokensTransformValidatorTest {
                 "search test {",
                 "  document test {",
                 "      field f type " + fieldType + " {",
-                "          indexing: summary",
+                "          indexing: index | summary",
                 "          summary: tokens",
                 "      }",
                 "  }",