Prepare for emitting warning if summary field type is specified.

Improve resolving of summary field type from source field type.

7 files changed, 164 insertions, 37 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
index ffb86f8ecf2..bc0e16abbe3 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java
@@ -20,6 +20,7 @@ import com.yahoo.vespa.documentmodel.SummaryTransform;
 
 import java.util.Locale;
 import java.util.Map;
+import java.util.logging.Level;
 
 /**
  * Helper for converting ParsedField etc to SDField with settings
@@ -137,7 +138,7 @@ public class ConvertParsedFields {
     }
 
     // from grammar, things that can be inside struct-field block
-    private void convertCommonFieldSettings(SDField field, ParsedField parsed) {
+    private void convertCommonFieldSettings(Schema schema, SDField field, ParsedField parsed) {
         convertMatchSettings(field, parsed.matchSettings());
         var indexing = parsed.getIndexing();
         if (indexing.isPresent()) {
@@ -152,7 +153,12 @@ public class ConvertParsedFields {
         for (var summaryField : parsed.getSummaryFields()) {
             var dataType = field.getDataType();
             var otherType = summaryField.getType();
-            if (otherType != null) {
+            if (otherType != null && summaryField.getHasExplicitType()) {
+                schema.getDeployLogger().log(Level.FINE, () -> "For " + schema.getName() +
+                        ", field '" + field.getName() +
+                        "', summary '" + summaryField.name() +
+                        "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." +
+                        " Remove the type specification to silence this warning.");
                 dataType = context.resolveType(otherType);
             }
             convertSummaryField(field, summaryField, dataType);
@@ -161,7 +167,7 @@ public class ConvertParsedFields {
             field.addQueryCommand(command);
         }
         for (var structField : parsed.getStructFields()) {
-            convertStructField(field, structField);
+            convertStructField(schema, field, structField);
         }
         if (parsed.hasLiteral()) {
             field.getRanking().setLiteral(true);
@@ -174,13 +180,13 @@ public class ConvertParsedFields {
         }
     }
 
-    private void convertStructField(SDField field, ParsedField parsed) {
+    private void convertStructField(Schema schema, SDField field, ParsedField parsed) {
         SDField structField = field.getStructField(parsed.name());
         if (structField == null ) {
             throw new IllegalArgumentException("Struct field '" + parsed.name() + "' has not been defined in struct " +
                                                "for field '" + field.getName() + "'.");
         }
-        convertCommonFieldSettings(structField, parsed);
+        convertCommonFieldSettings(schema, structField, parsed);
     }
 
     private void convertExtraFieldSettings(SDField field, ParsedField parsed) {
@@ -280,7 +286,7 @@ public class ConvertParsedFields {
         String name = parsed.name();
         DataType dataType = context.resolveType(parsed.getType());
         var field = new SDField(document, name, dataType);
-        convertCommonFieldSettings(field, parsed);
+        convertCommonFieldSettings(schema, field, parsed);
         convertExtraFieldSettings(field, parsed);
         document.addField(field);
         return field;
@@ -290,7 +296,7 @@ public class ConvertParsedFields {
         String name = parsed.name();
         DataType dataType = context.resolveType(parsed.getType());
         var field = new SDField(schema.getDocument(), name, dataType);
-        convertCommonFieldSettings(field, parsed);
+        convertCommonFieldSettings(schema, field, parsed);
         convertExtraFieldSettings(field, parsed);
         schema.addExtraField(field);
     }
@@ -307,7 +313,7 @@ public class ConvertParsedFields {
         for (var parsedField : parsed.getFields()) {
             var fieldType = context.resolveType(parsedField.getType());
             var field = new SDField(document, parsedField.name(), fieldType);
-            convertCommonFieldSettings(field, parsedField);
+            convertCommonFieldSettings(schema, field, parsedField);
             structProxy.addField(field);
             if (parsedField.hasIdOverride()) {
                 structProxy.setFieldId(field, parsedField.idOverride());
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java
index 6ca7537205c..689353319c0 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java
@@ -11,11 +11,13 @@ import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.document.DataType;
 import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.PositionDataType;
 import com.yahoo.schema.DefaultRankProfile;
 import com.yahoo.schema.DocumentOnlySchema;
 import com.yahoo.schema.RankProfileRegistry;
 import com.yahoo.schema.Schema;
 import com.yahoo.schema.UnrankedRankProfile;
+import com.yahoo.schema.derived.SummaryClass;
 import com.yahoo.schema.document.SDDocumentType;
 import com.yahoo.schema.document.SDField;
 import com.yahoo.schema.document.TemporaryImportedField;
@@ -25,7 +27,9 @@ import com.yahoo.vespa.documentmodel.SummaryField;
 
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.logging.Level;
 import java.util.Map;
 import java.util.Optional;
 
@@ -137,7 +141,82 @@ public class ConvertParsedSchemas {
         schema.addDocument(document);
     }
 
-    private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) {
+    /*
+     * Helper class for resolving data type for a document summary. Summary type is still
+     * used internally in config model when generating and processing indexing scripts.
+     * See DynamicSummaryTransformUtils class comment for more details.
+     */
+    private class SummaryFieldTypeResolver {
+
+        private final Schema schema;
+        private final Map<String, ParsedSummaryField> summaryFields = new LinkedHashMap<String, ParsedSummaryField>();
+        private static final String zCurveSuffix = new String("_zcurve");
+
+        public SummaryFieldTypeResolver(Schema schema, List<ParsedDocumentSummary> parsed) {
+            this.schema = schema;
+            for (var docsum : parsed) {
+                for (var field : docsum.getSummaryFields()) {
+                    summaryFields.put(field.name(), field);
+                }
+            }
+        }
+
+        private boolean isPositionAttribute(Schema schema, String sourceFieldName) {
+            if (!sourceFieldName.endsWith(zCurveSuffix)) {
+                return false;
+            }
+            var name = sourceFieldName.substring(0, sourceFieldName.length() - zCurveSuffix.length());
+            var field = schema.getField(name);
+            return (field.getDataType().equals(PositionDataType.INSTANCE));
+        }
+
+
+        private String getSingleSource(ParsedSummaryField parsedField) {
+            if (parsedField.getSources().size() == 1) {
+                return parsedField.getSources().get(0);
+            }
+            return parsedField.name();
+        }
+
+        public DataType resolve(ParsedDocumentSummary docsum, ParsedSummaryField parsedField) {
+            var seen = new LinkedHashSet<String>();
+            var origName = parsedField.name();
+            while (true) {
+                if (seen.contains(parsedField.name())) {
+                    throw new IllegalArgumentException("For schema '" + schema.getName() +
+                            "' summary class '" + docsum.name() +
+                            "' summary field '" + origName +
+                            "': Source loop detected for summary field '" + parsedField.name() + "'");
+                }
+                seen.add(parsedField.name());
+                if (parsedField.getSources().size() >= 2) {
+                    return DataType.STRING; // Flattening, streaming search
+                }
+                var source = getSingleSource(parsedField);
+                if (source.equals(SummaryClass.DOCUMENT_ID_FIELD)) {
+                    return DataType.STRING; // Reserved source field name
+                } else if (isPositionAttribute(schema, source)) {
+                    return DataType.LONG;   // Extra field with suffix is added later for positions
+                }
+                var field = schema.getField(source);
+                if (field != null) {
+                    return field.getDataType();
+                } else if (schema.temporaryImportedFields().isPresent() &&
+                        schema.temporaryImportedFields().get().hasField(source)) {
+                    return null; // Imported field, cannot resolve now
+                } else if (source.equals(parsedField.name()) || !summaryFields.containsKey(source)) {
+                    throw new IllegalArgumentException("For schema '" + schema.getName() +
+                            "', summary class '" + docsum.name() +
+                            "', summary field '" + parsedField.name() +
+                            "': there is no valid source '" + source + "'.");
+                }
+                parsedField = summaryFields.get(source);
+            }
+        }
+    }
+
+    private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext,
+                                        SummaryFieldTypeResolver sfResolver) {
         var docsum = new DocumentSummary(parsed.name(), schema);
         parsed.getInherited().forEach(inherited -> docsum.addInherited(inherited));
         if (parsed.getFromDisk()) {
@@ -148,16 +227,17 @@ public class ConvertParsedSchemas {
         }
         for (var parsedField : parsed.getSummaryFields()) {
             var parsedType = parsedField.getType();
-            DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null;
-            var existingField = schema.getField(parsedField.name());
-            if (existingField == null && parsedField.getSources().size() == 1) {
-                var sourceName = parsedField.getSources().get(0);
-                if (!sourceName.equals(parsedField.name())) {
-                    existingField = schema.getField(sourceName);
-                }
+            if (parsedType != null) {
+                var log = schema.getDeployLogger();
+                log.log(Level.FINE, () -> "For " + schema.getName() +
+                        ", document-summary '" + parsed.name() +
+                        "', summary field '" + parsedField.name() +
+                        "': Specifying the type is deprecated, ignored and will be an error in Vespa 9." +
+                        " Remove the type specification to silence this warning.");
             }
-            if (existingField != null) {
-                var existingType = existingField.getDataType();
+            DataType dataType = (parsedType != null) ? typeContext.resolveType(parsedType) : null;
+            DataType existingType = sfResolver.resolve(parsed, parsedField);
+            if (existingType != null) {
                 if (dataType == null) {
                     dataType = existingType;
                 } else if (!dataType.equals(existingType)) {
@@ -167,10 +247,9 @@ public class ConvertParsedSchemas {
                     }
                 }
             }
-            if (dataType == null) {
-                throw new IllegalArgumentException("Missing data-type for summary field " + parsedField.name() + " in document-summary " + parsed.name());
-            }
-            var summaryField = new SummaryField(parsedField.name(), dataType);
+            var summaryField = (dataType == null) ?
+                    SummaryField.createWithUnresolvedType(parsedField.name()) :
+                    new SummaryField(parsedField.name(), dataType);
             // XXX does not belong here:
             summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE);
             ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField);
@@ -212,6 +291,7 @@ public class ConvertParsedSchemas {
         }
         parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value));
         var typeContext = typeConverter.makeContext(parsed.getDocument());
+        var sfResolver = new SummaryFieldTypeResolver(schema, parsed.getDocumentSummaries());
         var fieldConverter = new ConvertParsedFields(typeContext, convertedStructs);
         convertDocument(schema, parsed.getDocument(), fieldConverter);
         for (var field : parsed.getFields()) {
@@ -220,12 +300,12 @@ public class ConvertParsedSchemas {
         for (var index : parsed.getIndexes()) {
             fieldConverter.convertExtraIndex(schema, index);
         }
-        for (var docsum : parsed.getDocumentSummaries()) {
-            convertDocumentSummary(schema, docsum, typeContext);
-        }
         for (var importedField : parsed.getImportedFields()) {
             convertImportField(schema, importedField);
         }
+        for (var docsum : parsed.getDocumentSummaries()) {
+            convertDocumentSummary(schema, docsum, typeContext, sfResolver);
+        }
         for (var fieldSet : parsed.getFieldSets()) {
             convertFieldSet(schema, fieldSet);
         }
diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
index 8b732c358f5..8f9733d2595 100644
--- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
+++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java
@@ -19,6 +19,7 @@ class ParsedSummaryField extends ParsedBlock {
     private boolean isFull = false;
     private boolean isBold = false;
     private boolean isTokens = false;
+    private boolean hasExplicitType = false;
     private final List<String> sources = new ArrayList<>();
     private final List<String> destinations = new ArrayList<>();
 
@@ -39,6 +40,7 @@ class ParsedSummaryField extends ParsedBlock {
     boolean getFull() { return isFull; }
     boolean getMatchedElementsOnly() { return isMEO; }
     boolean getTokens() { return isTokens; }
+    boolean getHasExplicitType() { return hasExplicitType; }
 
     void addDestination(String dst) { destinations.add(dst); }
     void addSource(String src) { sources.add(src); }
@@ -47,6 +49,7 @@ class ParsedSummaryField extends ParsedBlock {
     void setFull() { this.isFull = true; }
     void setMatchedElementsOnly() { this.isMEO = true; }
     void setTokens() { this.isTokens = true; }
+    void setHasExplicitType() { this.hasExplicitType = true; }
     void setType(ParsedType value) {
         verifyThat(type == null, "Cannot change type from ", type, "to", value);
         this.type = value;
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java
index 5b72381bfb1..762279e3871 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java
@@ -2,6 +2,8 @@
 package com.yahoo.schema.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
 import com.yahoo.schema.RankProfileRegistry;
 import com.yahoo.schema.Schema;
 import com.yahoo.schema.document.ImmutableImportedComplexSDField;
@@ -13,17 +15,17 @@ import com.yahoo.vespa.model.container.search.QueryProfiles;
 import java.util.stream.Stream;
 
 /**
- * Adds the attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported
+ * Adds the data type and attribute summary transform ({@link SummaryTransform#ATTRIBUTE} to all {@link SummaryField} having an imported
  * field as source.
  *
  * @author bjorncs
  */
-public class AddAttributeTransformToSummaryOfImportedFields extends Processor {
+public class AddDataTypeAndTransformToSummaryOfImportedFields extends Processor {
 
-    public AddAttributeTransformToSummaryOfImportedFields(Schema schema,
-                                                          DeployLogger deployLogger,
-                                                          RankProfileRegistry rankProfileRegistry,
-                                                          QueryProfiles queryProfiles) {
+    public AddDataTypeAndTransformToSummaryOfImportedFields(Schema schema,
+                                                            DeployLogger deployLogger,
+                                                            RankProfileRegistry rankProfileRegistry,
+                                                            QueryProfiles queryProfiles) {
         super(schema, deployLogger, rankProfileRegistry, queryProfiles);
     }
 
@@ -39,19 +41,29 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor {
 
     private void setTransform(ImmutableSDField field) {
         if (field instanceof ImmutableImportedComplexSDField) {
-            getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeCombinerTransform);
+            getSummaryFieldsForImportedField(field).forEach(summaryField -> setAttributeCombinerTransform(field, summaryField));
         } else {
-            getSummaryFieldsForImportedField(field).forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform);
+            getSummaryFieldsForImportedField(field).forEach(summaryField -> setAttributeTransform(field, summaryField));
         }
     }
 
-    private static void setAttributeTransform(SummaryField summaryField) {
+    private static void setAttributeTransform(ImmutableSDField field, SummaryField summaryField) {
+        if (summaryField.hasUnresolvedType()) {
+            if (field.getDataType().equals(DataType.LONG) && summaryField.getTransform().equals(SummaryTransform.GEOPOS)) {
+                summaryField.setResolvedDataType(PositionDataType.INSTANCE);
+            } else {
+                summaryField.setResolvedDataType(field.getDataType());
+            }
+        }
         if (summaryField.getTransform() == SummaryTransform.NONE) {
             summaryField.setTransform(SummaryTransform.ATTRIBUTE);
         }
     }
 
-    private static void setAttributeCombinerTransform(SummaryField summaryField) {
+    private static void setAttributeCombinerTransform(ImmutableSDField field, SummaryField summaryField) {
+        if (summaryField.hasUnresolvedType()) {
+            summaryField.setResolvedDataType(field.getDataType());
+        }
         if (summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) {
             // This field already has the correct transform.
             return;
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
index 89e6a1533d0..d9b90ad661d 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
@@ -49,6 +49,7 @@ public class Processing {
                 DictionaryProcessor::new,
                 WordMatch::new,
                 ImportedFieldsResolver::new,
+                AddDataTypeAndTransformToSummaryOfImportedFields::new,
                 ImplicitSummaries::new,
                 ImplicitSummaryFields::new,
                 AdjustPositionSummaryFields::new,
@@ -57,7 +58,6 @@ public class Processing {
                 SummaryNamesFieldCollisions::new,
                 SummaryFieldsMustHaveValidSource::new,
                 MatchedElementsOnlyResolver::new,
-                AddAttributeTransformToSummaryOfImportedFields::new,
                 MakeDefaultSummaryTheSuperSet::new,
                 Bolding::new,
                 AttributeProperties::new,
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java
index dd2fd72b280..662f3fc970b 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java
@@ -49,7 +49,9 @@ public class ValidateFieldTypes extends Processor {
     final protected void verifySummaryFields(String searchName, Map<String, DataType> seenFields) {
         for (DocumentSummary summary : schema.getSummaries().values()) {
             for (SummaryField field : summary.getSummaryFields().values()) {
-                checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields);
+                if (!field.hasUnresolvedType()) {
+                    checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields);
+                }
             }
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java
index 2a316c8af60..1c53ee36497 100644
--- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java
+++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java
@@ -66,6 +66,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey {
 
     /** True if this field was defined implicitly */
     private boolean implicit = false;
+    private boolean unresolvedType = false;
 
     /** Creates a summary field with NONE as transform */
     public SummaryField(String name, DataType type) {
@@ -87,10 +88,24 @@ public class SummaryField extends Field implements Cloneable, TypedKey {
         this.transform=transform;
     }
 
+    public static SummaryField createWithUnresolvedType(String name) {
+        /*
+         * Data type is not available during conversion of
+         * parsed schema to schema. Use a placeholder data type and tag the summary
+         * field as having an unresolved type.
+         */
+        var summaryField = new SummaryField(name, DataType.NONE);
+        summaryField.unresolvedType = true;
+        return summaryField;
+    }
+
+
     public void setImplicit(boolean implicit) { this.implicit=implicit; }
 
     public boolean isImplicit() { return implicit; }
 
+    public boolean hasUnresolvedType() { return unresolvedType; }
+
     public void setTransform(SummaryTransform transform) {
         this.transform = transform;
         if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) {
@@ -246,6 +261,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey {
                 clone.sources = new LinkedHashSet<>(this.sources);
             if (this.destinations != null)
                 clone.destinations = new LinkedHashSet<>(destinations);
+            clone.unresolvedType = unresolvedType;
             return clone;
         }
         catch (CloneNotSupportedException e) {
@@ -272,6 +288,14 @@ public class SummaryField extends Field implements Cloneable, TypedKey {
         return true;
     }
 
+    public void setResolvedDataType(DataType type) {
+        this.dataType = type;
+        if (!hasForcedId()) {
+            this.fieldId = calculateIdV7(null);
+        }
+        unresolvedType = false;
+    }
+
     public VsmCommand getVsmCommand() {
         return vsmCommand;
     }