From 4eff6c24b1db63eca7c5a43f82768eda043cfedf Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 29 Jun 2022 10:57:40 +0200 Subject: Use CopyDFW for struct field summaries with explicit source. --- .../main/java/com/yahoo/schema/derived/SummaryMap.java | 2 ++ .../com/yahoo/schema/processing/SummaryConsistency.java | 14 ++++++++++++++ .../com/yahoo/vespa/documentmodel/SummaryField.java | 12 ++++++++++++ .../com/yahoo/vespa/documentmodel/SummaryTransform.java | 3 ++- .../test/derived/array_of_struct_attribute/summary.cfg | 9 +++++++++ .../derived/array_of_struct_attribute/summarymap.cfg | 7 +++++-- .../src/test/derived/array_of_struct_attribute/test.sd | 3 +++ .../test/derived/map_of_struct_attribute/summary.cfg | 17 +++++++++++++++-- .../test/derived/map_of_struct_attribute/summarymap.cfg | 10 ++++++++-- .../src/test/derived/map_of_struct_attribute/test.sd | 4 ++++ 10 files changed, 74 insertions(+), 7 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java index 40d763eb897..35bb147ccf5 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java @@ -42,6 +42,8 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { if (summaryField.getTransform()== SummaryTransform.NONE) continue; if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE || + (summaryField.getTransform()==SummaryTransform.ATTRIBUTECOMBINER && summaryField.hasExplicitSingleSource()) || + summaryField.getTransform()==SummaryTransform.COPY || summaryField.getTransform()==SummaryTransform.DISTANCE || summaryField.getTransform()==SummaryTransform.GEOPOS || summaryField.getTransform()==SummaryTransform.POSITIONS || diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java index 4fb45c3c68f..bcb0917b61a 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java @@ -37,6 +37,7 @@ public class SummaryConsistency extends Processor { assertConsistency(summaryField, schema, validate); makeAttributeTransformIfAppropriate(summaryField, schema); makeAttributeCombinerTransformIfAppropriate(summaryField, schema); + makeCopyTransformIfAppropriate(summaryField, schema); } } } @@ -78,6 +79,19 @@ public class SummaryConsistency extends Processor { } } + /* + * This function must be called after makeAttributeCombinerIfAppropriate(). + */ + private void makeCopyTransformIfAppropriate(SummaryField summaryField, Schema schema) { + if (summaryField.getTransform() == SummaryTransform.NONE) { + String source_field_name = summaryField.getSingleSource(); + ImmutableSDField source = schema.getField(source_field_name); + if (source != null && source.usesStructOrMap() && summaryField.hasExplicitSingleSource()) { + summaryField.setTransform(SummaryTransform.COPY); + } + } + } + private void assertConsistentTypes(SummaryField existing, SummaryField seen) { if (existing.getDataType() instanceof WeightedSetDataType && seen.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)existing.getDataType()).getNestedType().equals(((WeightedSetDataType)seen.getDataType()).getNestedType())) diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index ed6668f0d0d..61889eb36e2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -263,6 +263,18 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } } + public boolean hasExplicitSingleSource() { + String fieldName = getName(); + String sourceName = getSingleSource(); + if (fieldName.equals(sourceName)) { + return false; + } + if (sourceName.contains(".")) { + return false; + } + return true; + } + public VsmCommand getVsmCommand() { return vsmCommand; } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index c50766a2585..a1ebc9e4bec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -22,7 +22,8 @@ public enum SummaryTransform { GEOPOS("geopos"), ATTRIBUTECOMBINER("attributecombiner"), MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), - MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"); + MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), + COPY("copy"); private final String name; diff --git a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg index 3f0994cf2bd..38298feaa0c 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summary.cfg @@ -11,3 +11,12 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].id 659145226 +classes[].name "rename" +classes[].omitsummaryfeatures false +classes[].fields[].name "new_elem_array" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg index bb99d6ced39..9dd63a3f316 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/summarymap.cfg @@ -1,10 +1,13 @@ defaultoutputclass -1 -override[].field "elem_array" +override[].field "new_elem_array" override[].command "attributecombiner" -override[].arguments "" +override[].arguments "elem_array" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" override[].field "summaryfeatures" override[].command "summaryfeatures" override[].arguments "" +override[].field "elem_array" +override[].command "attributecombiner" +override[].arguments "" diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd index 2f7c7755ca5..969dba96bf8 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/array_of_struct_attribute/test.sd @@ -16,4 +16,7 @@ schema test { } } } + document-summary rename { + summary new_elem_array type array { source: elem_array } + } } diff --git a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg index dadf7b5cfe7..997743389c6 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summary.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summary.cfg @@ -1,6 +1,6 @@ -defaultsummaryid 1131098132 +defaultsummaryid 1048168773 usev8geopositions true -classes[].id 1131098132 +classes[].id 1048168773 classes[].name "default" classes[].omitsummaryfeatures false classes[].fields[].name "str_elem_map" @@ -11,5 +11,18 @@ classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" +classes[].fields[].name "new_int_elem_map" +classes[].fields[].type "jsonstring" classes[].fields[].name "documentid" classes[].fields[].type "longstring" +classes[].id 1424421039 +classes[].name "rename" +classes[].omitsummaryfeatures false +classes[].fields[].name "new_str_elem_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "new_int_elem_map" +classes[].fields[].type "jsonstring" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" diff --git a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg index 1540b821ae1..291dbb91542 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/summarymap.cfg @@ -1,10 +1,16 @@ defaultoutputclass -1 -override[].field "str_elem_map" +override[].field "new_str_elem_map" override[].command "attributecombiner" -override[].arguments "" +override[].arguments "str_elem_map" +override[].field "new_int_elem_map" +override[].command "copy" +override[].arguments "int_elem_map" override[].field "rankfeatures" override[].command "rankfeatures" override[].arguments "" override[].field "summaryfeatures" override[].command "summaryfeatures" override[].arguments "" +override[].field "str_elem_map" +override[].command "attributecombiner" +override[].arguments "" diff --git a/config-model/src/test/derived/map_of_struct_attribute/test.sd b/config-model/src/test/derived/map_of_struct_attribute/test.sd index 7806d49392e..2e71678148b 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/test.sd +++ b/config-model/src/test/derived/map_of_struct_attribute/test.sd @@ -29,4 +29,8 @@ schema test { } } } + document-summary rename { + summary new_str_elem_map type map { source: str_elem_map } + summary new_int_elem_map type map { source: int_elem_map } + } } -- cgit v1.2.3