Add attribute-tokens summary transform to config model.

5 files changed, 27 insertions, 2 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
index 300a55e521a..799324d26e9 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
@@ -156,7 +156,8 @@ public class SummaryClass extends Derived {
                 summaryField.getTransform() == SummaryTransform.POSITIONS ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER ||
                 summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER ||
-                summaryField.getTransform() == SummaryTransform.TOKENS)
+                summaryField.getTransform() == SummaryTransform.TOKENS ||
+                summaryField.getTransform() == SummaryTransform.ATTRIBUTE_TOKENS)
         {
             return summaryField.getSingleSource();
         } else if (summaryField.getTransform().isDynamic()) {
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
index 2f60cd8eb06..349fd06ebdd 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java
@@ -94,6 +94,8 @@ public class SummaryClassField {
                 return Type.FEATUREDATA;
             } else if (transform != null && transform.equals(SummaryTransform.TOKENS)) {
                 return Type.JSONSTRING;
+            } else if (transform != null && transform.equals(SummaryTransform.ATTRIBUTE_TOKENS)) {
+                return Type.JSONSTRING;
             } else {
                 return Type.LONGSTRING;
             }
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java
index dd6f118d113..fe26ada5c8b 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustSummaryTransforms.java
@@ -36,6 +36,7 @@ public class AdjustSummaryTransforms extends Processor {
                 makeDocumentIdTransformIfAppropriate(summaryField);
                 makeAttributeTransformIfAppropriate(summaryField, schema);
                 makeAttributeCombinerTransformIfAppropriate(summaryField, schema);
+                makeAttributeTokensTransformIfAppropriate(summaryField, summary.getName(), schema);
                 makeCopyTransformIfAppropriate(summaryField, schema);
             }
         }
@@ -67,6 +68,25 @@ public class AdjustSummaryTransforms extends Processor {
         }
     }
 
+    private void makeAttributeTokensTransformIfAppropriate(SummaryField summaryField, String docsumName, Schema schema) {
+        if (summaryField.getTransform() == SummaryTransform.TOKENS) {
+            String sourceFieldName = summaryField.getSingleSource();
+            Attribute attribute = schema.getAttribute(sourceFieldName);
+            ImmutableSDField source = schema.getField(sourceFieldName);
+            if (!source.doesIndexing()) {
+                if (attribute != null) {
+                    summaryField.setTransform(SummaryTransform.ATTRIBUTE_TOKENS);
+                } else {
+                    throw new IllegalArgumentException("For schema '" + schema.getName() +
+                            "', document-summary '" + docsumName +
+                            "', summary field '" + summaryField.getName() +
+                            "', source field '" + sourceFieldName +
+                            "': tokens summary field setting requires index or attribute for source field");
+                }
+            }
+        }
+    }
+
     /*
      * This function must be called after makeAttributeCombinerTransformIfAppropriate().
      */
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
index e4116c3f9d5..3dd2ffdb02e 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
@@ -79,7 +79,8 @@ public class IndexingOutputs extends Processor {
             }
             dynamicSummary.add(summaryName);
         } else if (summaryTransform != SummaryTransform.ATTRIBUTE &&
-                summaryTransform != SummaryTransform.TOKENS) {
+                summaryTransform != SummaryTransform.TOKENS &&
+                summaryTransform != SummaryTransform.ATTRIBUTE_TOKENS) {
             staticSummary.add(summaryName);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java
index 420df3ee575..6ee82b31d57 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java
@@ -42,6 +42,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor {
             if (summaryField.getTransform() == SummaryTransform.ATTRIBUTECOMBINER) continue;
             if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) continue;
             if (summaryField.getTransform() == SummaryTransform.TOKENS) continue;
+            if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE_TOKENS) continue;
 
             defaultSummary.add(summaryField.clone());
         }