Add new summary transform for all 'documentid' summary fields.

This prepares for no longer inserting 'documentid' fields in the docsum blob in the backend.
Instead such fields will be produced using a docsum writer (transform) that uses the document instance to get the id.

7 files changed, 62 insertions, 9 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
index 193c6893203..efc64a5aa40 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java
@@ -59,7 +59,7 @@ public class SummaryClass extends Derived {
     /** MUST be called after all other fields are added */
     private void deriveImplicitFields(DocumentSummary summary, Map<String, SummaryClassField> fields) {
         if (summary.getName().equals("default")) {
-            addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, fields);
+            addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, SummaryTransform.DOCUMENT_ID, fields);
         }
     }
 
@@ -72,10 +72,6 @@ public class SummaryClass extends Derived {
         }
     }
 
-    private void addField(String name, DataType type, Map<String, SummaryClassField> fields) {
-        addField(name, type, null, fields);
-    }
-
     private void addField(String name, DataType type,
                           SummaryTransform transform,
                           Map<String, SummaryClassField> fields) {
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java
index ca81301da73..471d338a5f6 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java
@@ -39,6 +39,7 @@ public class AddExtraFieldsToDocument extends Processor {
                     case DYNAMICBOLDED:
                     case DYNAMICTEASER:
                     case TEXTEXTRACTOR:
+                    case DOCUMENT_ID: // TODO: Adding the 'documentid' field should no longer be needed when the docsum framework in the backend has been simplified and the transform is always used.
                         addSummaryField(schema, document, summaryField, validate);
                         break;
                     default:
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
index 63eca2121c1..8f7e8daeed0 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
@@ -52,6 +52,7 @@ public class Processing {
                 ImplicitSummaries::new,
                 ImplicitSummaryFields::new,
                 AdjustPositionSummaryFields::new,
+                SummaryTransformForDocumentId::new,
                 SummaryConsistency::new,
                 SummaryNamesFieldCollisions::new,
                 SummaryFieldsMustHaveValidSource::new,
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java
new file mode 100644
index 00000000000..99419ecd526
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java
@@ -0,0 +1,32 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.schema.RankProfileRegistry;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.derived.SummaryClass;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Adds the corresponding summary transform for all "documentid" summary fields.
+ *
+ * @author geirst
+ */
+public class SummaryTransformForDocumentId extends Processor {
+
+    public SummaryTransformForDocumentId(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(schema, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process(boolean validate, boolean documentsOnly) {
+        for (var summary : schema.getSummaries().values()) {
+            for (var summaryField : summary.getSummaryFields().values()) {
+                if (summaryField.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) {
+                    summaryField.setTransform(SummaryTransform.DOCUMENT_ID);
+                }
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
index a1ebc9e4bec..5c3aef89edc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java
@@ -23,7 +23,8 @@ public enum SummaryTransform {
     ATTRIBUTECOMBINER("attributecombiner"),
     MATCHED_ELEMENTS_FILTER("matchedelementsfilter"),
     MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"),
-    COPY("copy");
+    COPY("copy"),
+    DOCUMENT_ID("documentid");
 
     private final String name;
 
diff --git a/config-model/src/test/derived/imported_struct_fields/summarymap.cfg b/config-model/src/test/derived/imported_struct_fields/summarymap.cfg
index cd19a75e85f..653b0a26b75 100644
--- a/config-model/src/test/derived/imported_struct_fields/summarymap.cfg
+++ b/config-model/src/test/derived/imported_struct_fields/summarymap.cfg
@@ -1,4 +1,7 @@
 defaultoutputclass -1
+override[].field "documentid"
+override[].command "documentid"
+override[].arguments ""
 override[].field "my_elem_array"
 override[].command "attributecombiner"
 override[].arguments ""
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java
index f0fc58b97e5..2f1b837554d 100644
--- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java
@@ -162,23 +162,42 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase {
         assertFalse(SummaryMap.isDynamicCommand(SummaryTransform.ATTRIBUTE.getName()));
     }
 
+    @Test
+    public void documentid_summary_field_has_corresponding_summary_transform() throws ParseException {
+        var schema = buildSchema("field foo type string { indexing: summary }",
+                joinLines("document-summary bar {",
+                        "    summary documentid type string {}",
+                        "}"));
+        assertOverride(schema, "documentid", SummaryTransform.DOCUMENT_ID.getName(), "");
+    }
+
+    @Test
+    public void documentid_summary_transform_requires_disk_access() {
+        assertFalse(SummaryTransform.DOCUMENT_ID.isInMemory());
+    }
+
     private void assertOverride(String fieldContent, String expFieldName, String expCommand) throws ParseException {
-        var summaryMap = new SummaryMap(buildSearch(fieldContent));
+        assertOverride(buildSchema(fieldContent, ""), expFieldName, expCommand, expFieldName);
+    }
+
+    private void assertOverride(Schema schema, String expFieldName, String expCommand, String expArguments) throws ParseException {
+        var summaryMap = new SummaryMap(schema);
         var cfgBuilder = new SummarymapConfig.Builder();
         summaryMap.getConfig(cfgBuilder);
         var cfg = new SummarymapConfig(cfgBuilder);
         var override = cfg.override(0);
         assertEquals(expFieldName, override.field());
         assertEquals(expCommand, override.command());
-        assertEquals(expFieldName, override.arguments());
+        assertEquals(expArguments, override.arguments());
     }
 
-    private Schema buildSearch(String field) throws ParseException {
+    private Schema buildSchema(String field, String documentSummary) throws ParseException {
         var builder = new ApplicationBuilder(new RankProfileRegistry());
         builder.addSchema(joinLines("search test {",
                                     "  document test {",
                                     field,
                                     "  }",
+                                    documentSummary,
                                     "}"));
         builder.build(true);
         return builder.getSchema();