Merge pull request #21539 from vespa-engine/arnej/conversion-types-3

split out conversion of types

2 files changed, 297 insertions, 224 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java
new file mode 100644
index 00000000000..58a52a51fa5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java
@@ -0,0 +1,294 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.parser;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.document.ReferenceDataType;
+import com.yahoo.document.StructDataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Helper class for converting ParsedType instances to DataType
+ *
+ * @author arnej27959
+ **/
+public class ConvertParsedTypes {
+
+    private final List<ParsedSchema> orderedInput;
+    private final DocumentTypeManager docMan;
+
+    ConvertParsedTypes(List<ParsedSchema> input) {
+        this.orderedInput = input;
+        this.docMan = new DocumentTypeManager();
+    }
+
+    public ConvertParsedTypes(List<ParsedSchema> input, DocumentTypeManager docMan) {
+        this.orderedInput = input;
+        this.docMan = docMan;
+    }
+
+    public void convert() {
+        startDataTypes();
+        fillDataTypes();
+        registerDataTypes();
+    }
+
+    private Map<String, DocumentType> documentsFromSchemas = new HashMap<>();
+    private Map<String, StructDataType> structsFromSchemas = new HashMap<>();
+    private Map<String, AnnotationType> annotationsFromSchemas = new HashMap<>();
+
+    private void startDataTypes() {
+        for (var schema : orderedInput) {
+            String name = schema.getDocument().name();
+            documentsFromSchemas.put(name, new DocumentType(name));
+        }
+        for (var schema : orderedInput) {
+            var doc = schema.getDocument();
+            for (var struct : doc.getStructs()) {
+                var dt = new StructDataType(struct.name());
+                String structId = doc.name() + "->" + struct.name();
+                structsFromSchemas.put(structId, dt);
+            }
+            for (var annotation : doc.getAnnotations()) {
+                String annId = doc.name() + "->" + annotation.name();
+                var at = new AnnotationType(annotation.name());
+                annotationsFromSchemas.put(annId, at);
+                var withStruct = annotation.getStruct();
+                if (withStruct.isPresent()) {
+                    var sn = withStruct.get().name();
+                    var dt = new StructDataType(sn);
+                    String structId = doc.name() + "->" + sn;
+                    structsFromSchemas.put(structId, dt);
+                }
+            }
+        }
+    }
+
+    private void fillDataTypes() {
+        for (var schema : orderedInput) {
+            var doc = schema.getDocument();
+            for (var struct : doc.getStructs()) {
+                String structId = doc.name() + "->" + struct.name();
+                var toFill = structsFromSchemas.get(structId);
+                for (ParsedField field : struct.getFields()) {
+                    var t = resolveFromContext(field.getType(), doc);
+                    var f = new com.yahoo.document.Field(field.name(), t);
+                    toFill.addField(f);
+                }
+                for (String inherit : struct.getInherited()) {
+                    var parent = findStructFromSchemas(inherit, doc);
+                    toFill.inherit(parent);
+                }
+            }
+            for (var annotation : doc.getAnnotations()) {
+                String annId = doc.name() + "->" + annotation.name();
+                var at = annotationsFromSchemas.get(annId);
+                var withStruct = annotation.getStruct();
+                if (withStruct.isPresent()) {
+                    ParsedStruct struct = withStruct.get();
+                    String structId = doc.name() + "->" + struct.name();
+                    var toFill = structsFromSchemas.get(structId);
+                    for (ParsedField field : struct.getFields()) {
+                        var t = resolveFromContext(field.getType(), doc);
+                        var f = new com.yahoo.document.Field(field.name(), t);
+                        toFill.addField(f);
+                    }
+                    at.setDataType(toFill);
+                }
+                for (String inherit : annotation.getInherited()) {
+                    var parent = findAnnotationFromSchemas(inherit, doc);
+                    at.inherit(parent);
+                }
+            }
+            var docToFill = documentsFromSchemas.get(doc.name());
+            Map<String, Collection<String>> fieldSets = new HashMap<>();
+            List<String> inDocFields = new ArrayList<>();
+            for (var docField : doc.getFields()) {
+                String name = docField.name();
+                var t = resolveFromContext(docField.getType(), doc);
+                var f = new com.yahoo.document.Field(name, t);
+                docToFill.addField(f);
+                inDocFields.add(name);
+            }
+            fieldSets.put("[document]", inDocFields);
+            for (var extraField : schema.getFields()) {
+                String name = extraField.name();
+                var t = resolveFromContext(extraField.getType(), doc);
+                var f = new com.yahoo.document.Field(name, t);
+                docToFill.addField(f);
+            }
+            for (var fieldset : schema.getFieldSets()) {
+                fieldSets.put(fieldset.name(), fieldset.getFieldNames());
+            }
+            docToFill.addFieldSets(fieldSets);
+            for (String inherit : doc.getInherited()) {
+                docToFill.inherit(findDocFromSchemas(inherit));
+            }
+        }
+    }
+
+    private StructDataType findStructFromSchemas(String name, ParsedDocument context) {
+        var resolved = findParsedStruct(context, name);
+        if (resolved == null) {
+            throw new IllegalArgumentException("no struct named " + name + " in context " + context);
+        }
+        String structId = resolved.getOwner() + "->" + resolved.name();
+        var struct = structsFromSchemas.get(structId);
+        assert(struct != null);
+        return struct;
+    }
+
+    private AnnotationType findAnnotationFromSchemas(String name, ParsedDocument context) {
+        var resolved = findParsedAnnotation(context, name);
+        String annotationId = resolved.getOwner() + "->" + resolved.name();
+        var annotation = annotationsFromSchemas.get(annotationId);
+        if (annotation == null) {
+            throw new IllegalArgumentException("no annotation named " + name + " in context " + context);
+        }
+        return annotation;
+    }
+
+    private ParsedStruct findParsedStruct(ParsedDocument doc, String name) {
+        ParsedStruct found = doc.getStruct(name);
+        if (found != null) return found;
+        for (var parent : doc.getResolvedInherits()) {
+            var fromParent = findParsedStruct(parent, name);
+            if (fromParent == null) continue;
+            if (fromParent == found) continue;
+            if (found == null) {
+                found = fromParent;
+            } else {
+                throw new IllegalArgumentException("conflicting values for struct " + name + " in " +doc);
+            }
+        }
+        if (found == null) {
+            // TODO: be more restrictive here, but we need something
+            // for imported fields. For now, fall back to looking for
+            // struct in any schema.
+            for (var schema : orderedInput) {
+                for (var struct : schema.getDocument().getStructs()) {
+                    if (struct.name().equals(name)) {
+                        return struct;
+                    }
+                }
+            }
+        }
+        return found;
+    }
+
+    private ParsedAnnotation findParsedAnnotation(ParsedDocument doc, String name) {
+        ParsedAnnotation found = doc.getAnnotation(name);
+        if (found != null) return found;
+        for (var parent : doc.getResolvedInherits()) {
+            var fromParent = findParsedAnnotation(parent, name);
+            if (fromParent == null) continue;
+            if (fromParent == found) continue;
+            if (found == null) {
+                found = fromParent;
+            } else {
+                throw new IllegalArgumentException("conflicting values for annotation " + name + " in " +doc);
+            }
+        }
+        return found;
+    }
+
+    private DataType createArray(ParsedType pType, ParsedDocument context) {
+        DataType nested = resolveFromContext(pType.nestedType(), context);
+        return DataType.getArray(nested);
+    }
+
+    private DataType createWset(ParsedType pType, ParsedDocument context) {
+        DataType nested = resolveFromContext(pType.nestedType(), context);
+        boolean cine = pType.getCreateIfNonExistent();
+        boolean riz = pType.getRemoveIfZero();
+        return new WeightedSetDataType(nested, cine, riz);
+    }
+
+    private DataType createMap(ParsedType pType, ParsedDocument context) {
+        DataType kt = resolveFromContext(pType.mapKeyType(), context);
+        DataType vt = resolveFromContext(pType.mapValueType(), context);
+        return DataType.getMap(kt, vt);
+    }
+
+    private DocumentType findDocFromSchemas(String name) {
+        var dt = documentsFromSchemas.get(name);
+        if (dt == null) {
+            throw new IllegalArgumentException("missing document type for: " + name);
+        }
+        return dt;
+    }
+
+    private DataType createAnnRef(ParsedType pType, ParsedDocument context) {
+        AnnotationType annotation = findAnnotationFromSchemas(pType.getNameOfReferencedAnnotation(), context);
+        return new AnnotationReferenceDataType(annotation);
+    }
+
+    private DataType createDocRef(ParsedType pType) {
+        var ref = pType.getReferencedDocumentType();
+        assert(ref.getVariant() == ParsedType.Variant.DOCUMENT);
+        return ReferenceDataType.createWithInferredId(findDocFromSchemas(ref.name()));
+    }
+
+    private DataType resolveFromContext(ParsedType pType, ParsedDocument context) {
+        String name = pType.name();
+        switch (pType.getVariant()) {
+        case NONE:     return DataType.NONE;
+        case BUILTIN:  return docMan.getDataType(name);
+        case POSITION: return PositionDataType.INSTANCE;
+        case ARRAY:    return createArray(pType, context);
+        case WSET:     return createWset(pType, context);
+        case MAP:      return createMap(pType, context);
+        case TENSOR:   return DataType.getTensor(pType.getTensorType());
+        case DOC_REFERENCE:  return createDocRef(pType);
+        case ANN_REFERENCE:  return createAnnRef(pType, context);
+        case DOCUMENT: return findDocFromSchemas(name);
+        case STRUCT:   return findStructFromSchemas(name, context);
+        case UNKNOWN:
+            // fallthrough
+        }
+        // unknown is probably struct
+        var found = findParsedStruct(context, name);
+        if (found != null) {
+            pType.setVariant(ParsedType.Variant.STRUCT);
+            return findStructFromSchemas(name, context);
+        }
+        if (documentsFromSchemas.containsKey(name)) {
+            pType.setVariant(ParsedType.Variant.DOCUMENT);
+            return findDocFromSchemas(name);
+        }
+        throw new IllegalArgumentException("unknown type named '" + name + "' in context "+context);
+    }
+
+    private void registerDataTypes() {
+        for (DataType t : structsFromSchemas.values()) {
+            docMan.register(t);
+        }
+        for (DocumentType t : documentsFromSchemas.values()) {
+            docMan.register(t);
+        }
+        for (AnnotationType t : annotationsFromSchemas.values()) {
+            docMan.getAnnotationTypeRegistry().register(t);
+        }
+    }
+
+    public class TypeResolver {
+        private final ParsedDocument context;
+        public DataType resolveType(ParsedType parsed) {
+            return resolveFromContext(parsed, context);
+        }
+        public TypeResolver(ParsedDocument context) {
+            this.context = context;
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
index 9b69a82a8ff..fb0003ca4f9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java
@@ -38,11 +38,6 @@ public class ConvertSchemaCollection {
         pushTypesToDocuments();
     }
 
-    public void convertTypes() {
-        convertDataTypes();
-        registerDataTypes();
-    }
-
     void order() {
         var map = input.getParsedSchemas();
         for (var schema : map.values()) {
@@ -69,224 +64,8 @@ public class ConvertSchemaCollection {
         }
     }
 
-    Map<String, DocumentType> documentsInProgress = new HashMap<>();
-    Map<String, StructDataType> structsInProgress = new HashMap<>();
-    Map<String, AnnotationType> annotationsInProgress = new HashMap<>();
-
-    StructDataType findStructInProgress(String name, ParsedDocument context) {
-        var resolved = findStructFrom(context, name);
-        if (resolved == null) {
-            throw new IllegalArgumentException("no struct named " + name + " in context " + context);
-        }
-        String structId = resolved.getOwner() + "->" + resolved.name();
-        var struct = structsInProgress.get(structId);
-        assert(struct != null);
-        return struct;
-    }
-
-    AnnotationType findAnnotationInProgress(String name, ParsedDocument context) {
-        var resolved = findAnnotationFrom(context, name);
-        String annotationId = resolved.getOwner() + "->" + resolved.name();
-        var annotation = annotationsInProgress.get(annotationId);
-        if (annotation == null) {
-            throw new IllegalArgumentException("no annotation named " + name + " in context " + context);
-        }
-        return annotation;
-    }
-
-    ParsedStruct findStructFrom(ParsedDocument doc, String name) {
-        ParsedStruct found = doc.getStruct(name);
-        if (found != null) return found;
-        for (var parent : doc.getResolvedInherits()) {
-            var fromParent = findStructFrom(parent, name);
-            if (fromParent == null) continue;
-            if (fromParent == found) continue;
-            if (found == null) {
-                found = fromParent;
-            } else {
-                throw new IllegalArgumentException("conflicting values for struct " + name + " in " +doc);
-            }
-        }
-        return found;
-    }
-
-    ParsedAnnotation findAnnotationFrom(ParsedDocument doc, String name) {
-        ParsedAnnotation found = doc.getAnnotation(name);
-        if (found != null) return found;
-        for (var parent : doc.getResolvedInherits()) {
-            var fromParent = findAnnotationFrom(parent, name);
-            if (fromParent == null) continue;
-            if (fromParent == found) continue;
-            if (found == null) {
-                found = fromParent;
-            } else {
-                throw new IllegalArgumentException("conflicting values for annotation " + name + " in " +doc);
-            }
-        }
-        return found;
-    }
-
-    private DataType createArray(ParsedType pType, ParsedDocument context) {
-        DataType nested = resolveType(pType.nestedType(), context);
-        return DataType.getArray(nested);
-    }
-
-    private DataType createWset(ParsedType pType, ParsedDocument context) {
-        DataType nested = resolveType(pType.nestedType(), context);
-        boolean cine = pType.getCreateIfNonExistent();
-        boolean riz = pType.getRemoveIfZero();
-        return new WeightedSetDataType(nested, cine, riz);
-    }
-
-    private DataType createMap(ParsedType pType, ParsedDocument context) {
-        DataType kt = resolveType(pType.mapKeyType(), context);
-        DataType vt = resolveType(pType.mapValueType(), context);
-        return DataType.getMap(kt, vt);
-    }
-
-    private DocumentType findDocInProgress(String name) {
-        var dt = documentsInProgress.get(name);
-        if (dt == null) {
-            throw new IllegalArgumentException("missing document type for: " + name);
-        }
-        return dt;
-    }
-
-    private DataType createAnnRef(ParsedType pType, ParsedDocument context) {
-        AnnotationType annotation = findAnnotationInProgress(pType.getNameOfReferencedAnnotation(), context);
-        return new AnnotationReferenceDataType(annotation);
-    }
-
-    private DataType createDocRef(ParsedType pType) {
-        var ref = pType.getReferencedDocumentType();
-        assert(ref.getVariant() == ParsedType.Variant.DOCUMENT);
-        return ReferenceDataType.createWithInferredId(findDocInProgress(ref.name()));
-    }
-
-    DataType resolveType(ParsedType pType, ParsedDocument context) {
-        switch (pType.getVariant()) {
-        case NONE:     return DataType.NONE;
-        case BUILTIN:  return docMan.getDataType(pType.name());
-        case POSITION: return PositionDataType.INSTANCE;
-        case ARRAY:    return createArray(pType, context);
-        case WSET:     return createWset(pType, context);
-        case MAP:      return createMap(pType, context);
-        case TENSOR:   return DataType.getTensor(pType.getTensorType());
-        case DOC_REFERENCE:  return createDocRef(pType);
-        case ANN_REFERENCE:  return createAnnRef(pType, context);
-        case DOCUMENT: return findDocInProgress(pType.name());
-        case STRUCT:   return findStructInProgress(pType.name(), context);
-        case UNKNOWN:
-            // fallthrough
-        }
-        // unknown is probably struct, but could be document:
-        if (documentsInProgress.containsKey(pType.name())) {
-            pType.setVariant(ParsedType.Variant.DOCUMENT);
-            return findDocInProgress(pType.name());
-        }
-        var struct = findStructInProgress(pType.name(), context);
-        pType.setVariant(ParsedType.Variant.STRUCT);
-        return struct;
-    }
-
-    void convertDataTypes() {
-        for (var schema : orderedInput) {
-            String name = schema.getDocument().name();
-            documentsInProgress.put(name, new DocumentType(name));
-        }
-        for (var schema : orderedInput) {
-            var doc = schema.getDocument();
-            for (var struct : doc.getStructs()) {
-                var dt = new StructDataType(struct.name());
-                String structId = doc.name() + "->" + struct.name();
-                structsInProgress.put(structId, dt);
-            }
-            for (var annotation : doc.getAnnotations()) {
-                String annId = doc.name() + "->" + annotation.name();
-                var at = new AnnotationType(annotation.name());
-                annotationsInProgress.put(annId, at);
-                var withStruct = annotation.getStruct();
-                if (withStruct.isPresent()) {
-                    var sn = withStruct.get().name();
-                    var dt = new StructDataType(sn);
-                    String structId = doc.name() + "->" + sn;
-                    structsInProgress.put(structId, dt);
-                }
-            }
-        }
-        for (var schema : orderedInput) {
-            var doc = schema.getDocument();
-            for (var struct : doc.getStructs()) {
-                String structId = doc.name() + "->" + struct.name();
-                var toFill = structsInProgress.get(structId);
-                for (String inherit : struct.getInherited()) {
-                    var parent = findStructInProgress(inherit, doc);
-                    toFill.inherit(parent);
-                }
-                for (ParsedField field : struct.getFields()) {
-                    var t = resolveType(field.getType(), doc);
-                    var f = new com.yahoo.document.Field(field.name(), t);
-                    toFill.addField(f);
-                }
-            }
-            for (var annotation : doc.getAnnotations()) {
-                String annId = doc.name() + "->" + annotation.name();
-                var at = annotationsInProgress.get(annId);
-                var withStruct = annotation.getStruct();
-                if (withStruct.isPresent()) {
-                    ParsedStruct struct = withStruct.get();
-                    String structId = doc.name() + "->" + struct.name();
-                    var toFill = structsInProgress.get(structId);
-                    for (ParsedField field : struct.getFields()) {
-                        var t = resolveType(field.getType(), doc);
-                        var f = new com.yahoo.document.Field(field.name(), t);
-                        toFill.addField(f);
-                    }
-                    at.setDataType(toFill);
-                }
-                for (String inherit : annotation.getInherited()) {
-                    var parent = findAnnotationInProgress(inherit, doc);
-                    at.inherit(parent);
-                }
-            }
-
-            var docToFill = documentsInProgress.get(doc.name());
-            Map<String, Collection<String>> fieldSets = new HashMap<>();
-            List<String> inDocFields = new ArrayList<>();
-            for (var docField : doc.getFields()) {
-                String name = docField.name();
-                var t = resolveType(docField.getType(), doc);
-                var f = new com.yahoo.document.Field(name, t);
-                docToFill.addField(f);
-                inDocFields.add(name);
-            }
-            fieldSets.put("[document]", inDocFields);
-            for (var extraField : schema.getFields()) {
-                String name = extraField.name();
-                var t = resolveType(extraField.getType(), doc);
-                var f = new com.yahoo.document.Field(name, t);
-                docToFill.addField(f);
-            }
-            for (var fieldset : schema.getFieldSets()) {
-                fieldSets.put(fieldset.name(), fieldset.getFieldNames());
-            }
-            docToFill.addFieldSets(fieldSets);
-            for (String inherit : doc.getInherited()) {
-                docToFill.inherit(findDocInProgress(inherit));
-            }
-        }
-    }
-
-    void registerDataTypes() {
-        for (DataType t : structsInProgress.values()) {
-            docMan.register(t);
-        }
-        for (DocumentType t : documentsInProgress.values()) {
-            docMan.register(t);
-        }
-        for (AnnotationType t : annotationsInProgress.values()) {
-            docMan.getAnnotationTypeRegistry().register(t);
-        }
+    public void convertTypes() {
+        var converter = new ConvertParsedTypes(orderedInput, docMan);
+        converter.convert();
     }
-
 }