Merge pull request #29222 from vespa-engine/toregge/write-to-summary-source

Write to summary field source.

8 files changed, 76 insertions, 9 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java
index 52652de81d4..67297245ff1 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java
@@ -70,7 +70,7 @@ public class AddExtraFieldsToDocument extends Processor {
     }
 
     private void addSdField(Schema schema, SDDocumentType document, SDField field, boolean validate) {
-        if (! field.hasIndex() && field.getAttributes().isEmpty()) {
+        if (! field.hasIndex() && field.getAttributes().isEmpty() && !field.doesSummarying()) {
             return;
         }
         for (Attribute atr : field.getAttributes().values()) {
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
index 3dd2ffdb02e..071c2878ae8 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java
@@ -137,6 +137,15 @@ public class IndexingOutputs extends Processor {
                 for (String fieldName : summaryFields) {
                     ret.add(new SummaryExpression(fieldName));
                 }
+                /*
+                 * Write to summary field source. AddExtraFieldsToDocument processor adds the "copy"
+                 * summary transform to summary fields without a corresponding explicitly declared
+                 * document field (2023-11-01). Future vespa versions will stop adding document
+                 * fields for those summary fields.
+                 */
+                if (!summaryFields.contains(field.getName())) {
+                    ret.add(new SummaryExpression(field.getName()));
+                }
             } else {
                 throw new UnsupportedOperationException(exp.getClass().getName());
             }
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java
index 9a98958fca9..3f336544a99 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java
@@ -131,10 +131,18 @@ public class IndexingValidation extends Processor {
             } else if (exp instanceof SummaryExpression) {
                 SummaryField field = schema.getSummaryField(fieldName);
                 if (field == null) {
-                    throw new VerificationException(exp, "Summary field '" + fieldName + "' not found.");
+                    // Use document field if summary field is not found
+                    SDField sdField = schema.getConcreteField(fieldName);
+                    if (sdField != null && sdField.doesSummarying()) {
+                        fieldDesc = "document field";
+                        fieldType = sdField.getDataType();
+                    } else {
+                        throw new VerificationException(exp, "Summary field '" + fieldName + "' not found.");
+                    }
+                } else {
+                    fieldDesc = "summary field";
+                    fieldType = field.getDataType();
                 }
-                fieldDesc = "summary field";
-                fieldType = field.getDataType();
             } else {
                 throw new UnsupportedOperationException();
             }
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java
index 63eee474095..34dcc9139b3 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java
@@ -39,9 +39,17 @@ public abstract class TypedTransformProvider extends ValueTransformProvider {
             }
             else if (exp instanceof SummaryExpression) {
                 Field field = schema.getSummaryField(fieldName);
-                if (field == null)
-                    throw new IllegalArgumentException("Summary field '" + fieldName + "' not found.");
-                fieldType = field.getDataType();
+                if (field == null) {
+                    // Use document field if summary field is not found
+                    var sdField = schema.getConcreteField(fieldName);
+                    if (sdField != null && sdField.doesSummarying()) {
+                        fieldType = sdField.getDataType();
+                    } else {
+                        throw new IllegalArgumentException("Summary field '" + fieldName + "' not found.");
+                    }
+                } else {
+                    fieldType = field.getDataType();
+                }
             }
             else {
                 throw new UnsupportedOperationException();
diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
index 1993bbd14ac..514dfffac9d 100644
--- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
+++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg
@@ -14,7 +14,7 @@ ilscript[].docfield[] "h"
 ilscript[].docfield[] "loc"
 ilscript[].docfield[] "mytags"
 ilscript[].content[] "clear_state | guard { input loc | to_pos | zcurve | attribute loc_pos_zcurve; }"
-ilscript[].content[] "clear_state | guard { input a | summary abolded2 | summary aboldeddynamic | summary adynamic2 | attribute a; }"
+ilscript[].content[] "clear_state | guard { input a | summary abolded2 | summary aboldeddynamic | summary adynamic2 | summary a | attribute a; }"
 ilscript[].content[] "clear_state | guard { input adynamic | summary adynamic | attribute adynamic; }"
 ilscript[].content[] "clear_state | guard { input abolded | summary abolded | attribute abolded; }"
 ilscript[].content[] "clear_state | guard { input b | summary anotherb | summary b; }"
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java
index 43b403c42c6..aad6df62993 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/AddExtraFieldsToDocumentTest.java
@@ -49,6 +49,29 @@ public class AddExtraFieldsToDocumentTest {
         assertNull(schema.getDocument().getField("my_c"));
     }
 
+    @Test
+    public void testExtraFieldIsAddedWhenBeingASummarySource() throws ParseException {
+        var sd = """
+                search renamed {
+                  document renamed {
+                    field foo type string { }
+                  }
+                  field bar type string {
+                    indexing: input foo | summary
+                    summary baz { }
+                  }
+                  field bar2 type string {
+                    indexing: input foo
+                    summary baz2 { }
+                  }
+                }
+                """;
+        var builder = ApplicationBuilder.createFromString(sd);
+        var schema = builder.getSchema();
+        assertNotNull(schema.getDocument().getDocumentType().getField("bar"));
+        assertNull(schema.getDocument().getDocumentType().getField("bar2"));
+    }
+
     private void assertSummary(Schema schema, String dsName, String name, SummaryTransform transform, String source) {
         var docsum = schema.getSummary(dsName);
         var field = docsum.getSummaryField(name);
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java
index 58956b89a07..76680f4ce9a 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java
@@ -62,4 +62,23 @@ public class IndexingOutputsTestCase {
         }
     }
 
+    @Test
+    void requireThatSummaryFieldSourceIsPopulated() throws ParseException {
+        var sd = """
+                search renamed {
+                  document renamed {
+                    field foo type string { }
+                  }
+                  field bar type string {
+                    indexing: input foo | summary
+                    summary baz { }
+                    summary dyn_baz { dynamic }
+                  }
+                }
+                """;
+        var builder = ApplicationBuilder.createFromString(sd);
+        var schema = builder.getSchema();
+        assertEquals("{ input foo | summary baz | summary dyn_baz | summary bar; }",
+                schema.getConcreteField("bar").getIndexingScript().toString());
+    }
 }
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
index b62fc61f5b7..0b561563421 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java
@@ -48,7 +48,7 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase {
     void testDynamicSummaryRewriting() {
         SDField field = createField("test", DataType.STRING, "{ summary }");
         field.addSummaryField(createDynamicSummaryField(field, "dyn"));
-        assertIndexingScript("{ input test | summary dyn; }", field);
+        assertIndexingScript("{ input test | summary dyn | summary test; }", field);
     }
 
     @Test