From a82fcc5b74a6fb788e29103585779bcae2af5ad7 Mon Sep 17 00:00:00 2001 From: Geir Storli Date: Tue, 12 Jul 2022 09:28:45 +0000 Subject: Add new summary transform for all 'documentid' summary fields. This prepares for no longer inserting 'documentid' fields in the docsum blob in the backend. Instead such fields will be produced using a docsum writer (transform) that uses the document instance to get the id. --- .../com/yahoo/schema/derived/SummaryClass.java | 6 +--- .../processing/AddExtraFieldsToDocument.java | 1 + .../com/yahoo/schema/processing/Processing.java | 1 + .../processing/SummaryTransformForDocumentId.java | 32 ++++++++++++++++++++++ .../vespa/documentmodel/SummaryTransform.java | 3 +- .../derived/imported_struct_fields/summarymap.cfg | 3 ++ .../yahoo/schema/derived/SummaryMapTestCase.java | 25 +++++++++++++++-- 7 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 193c6893203..efc64a5aa40 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -59,7 +59,7 @@ public class SummaryClass extends Derived { /** MUST be called after all other fields are added */ private void deriveImplicitFields(DocumentSummary summary, Map fields) { if (summary.getName().equals("default")) { - addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, fields); + addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, SummaryTransform.DOCUMENT_ID, fields); } } @@ -72,10 +72,6 @@ public class SummaryClass extends Derived { } } - private void addField(String name, DataType type, Map fields) { - addField(name, type, null, fields); - } - private void addField(String name, DataType type, SummaryTransform transform, Map fields) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java index ca81301da73..471d338a5f6 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java @@ -39,6 +39,7 @@ public class AddExtraFieldsToDocument extends Processor { case DYNAMICBOLDED: case DYNAMICTEASER: case TEXTEXTRACTOR: + case DOCUMENT_ID: // TODO: Adding the 'documentid' field should no longer be needed when the docsum framework in the backend has been simplified and the transform is always used. addSummaryField(schema, document, summaryField, validate); break; default: diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index 63eca2121c1..8f7e8daeed0 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -52,6 +52,7 @@ public class Processing { ImplicitSummaries::new, ImplicitSummaryFields::new, AdjustPositionSummaryFields::new, + SummaryTransformForDocumentId::new, SummaryConsistency::new, SummaryNamesFieldCollisions::new, SummaryFieldsMustHaveValidSource::new, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java new file mode 100644 index 00000000000..99419ecd526 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryTransformForDocumentId.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Adds the corresponding summary transform for all "documentid" summary fields. + * + * @author geirst + */ +public class SummaryTransformForDocumentId extends Processor { + + public SummaryTransformForDocumentId(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(schema, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + for (var summary : schema.getSummaries().values()) { + for (var summaryField : summary.getSummaryFields().values()) { + if (summaryField.getName().equals(SummaryClass.DOCUMENT_ID_FIELD)) { + summaryField.setTransform(SummaryTransform.DOCUMENT_ID); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index a1ebc9e4bec..5c3aef89edc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -23,7 +23,8 @@ public enum SummaryTransform { ATTRIBUTECOMBINER("attributecombiner"), MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), - COPY("copy"); + COPY("copy"), + DOCUMENT_ID("documentid"); private final String name; diff --git a/config-model/src/test/derived/imported_struct_fields/summarymap.cfg b/config-model/src/test/derived/imported_struct_fields/summarymap.cfg index cd19a75e85f..653b0a26b75 100644 --- a/config-model/src/test/derived/imported_struct_fields/summarymap.cfg +++ b/config-model/src/test/derived/imported_struct_fields/summarymap.cfg @@ -1,4 +1,7 @@ defaultoutputclass -1 +override[].field "documentid" +override[].command "documentid" +override[].arguments "" override[].field "my_elem_array" override[].command "attributecombiner" override[].arguments "" diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java index f0fc58b97e5..2f1b837554d 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java @@ -162,23 +162,42 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { assertFalse(SummaryMap.isDynamicCommand(SummaryTransform.ATTRIBUTE.getName())); } + @Test + public void documentid_summary_field_has_corresponding_summary_transform() throws ParseException { + var schema = buildSchema("field foo type string { indexing: summary }", + joinLines("document-summary bar {", + " summary documentid type string {}", + "}")); + assertOverride(schema, "documentid", SummaryTransform.DOCUMENT_ID.getName(), ""); + } + + @Test + public void documentid_summary_transform_requires_disk_access() { + assertFalse(SummaryTransform.DOCUMENT_ID.isInMemory()); + } + private void assertOverride(String fieldContent, String expFieldName, String expCommand) throws ParseException { - var summaryMap = new SummaryMap(buildSearch(fieldContent)); + assertOverride(buildSchema(fieldContent, ""), expFieldName, expCommand, expFieldName); + } + + private void assertOverride(Schema schema, String expFieldName, String expCommand, String expArguments) throws ParseException { + var summaryMap = new SummaryMap(schema); var cfgBuilder = new SummarymapConfig.Builder(); summaryMap.getConfig(cfgBuilder); var cfg = new SummarymapConfig(cfgBuilder); var override = cfg.override(0); assertEquals(expFieldName, override.field()); assertEquals(expCommand, override.command()); - assertEquals(expFieldName, override.arguments()); + assertEquals(expArguments, override.arguments()); } - private Schema buildSearch(String field) throws ParseException { + private Schema buildSchema(String field, String documentSummary) throws ParseException { var builder = new ApplicationBuilder(new RankProfileRegistry()); builder.addSchema(joinLines("search test {", " document test {", field, " }", + documentSummary, "}")); builder.build(true); return builder.getSchema(); -- cgit v1.2.3