add another Apply implentation

* if there is no config at all in the old arrays,
  read new per-doctype config instead

1 files changed, 261 insertions, 13 deletions

diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index 1f9e494aa29..c950f6b5163 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -11,15 +11,19 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.function.Supplier;
+import com.yahoo.tensor.TensorType;
 
 /**
  * Configures the Vespa document manager from a config id.
  *
  * @author Einar M R Rosenvinge
  */
-public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{
+public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig> {
 
     private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName());
 
@@ -65,15 +69,16 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
             return;
         }
         new Apply(config, manager);
+        if (config.datatype().size() == 0 && config.annotationtype().size() == 0) {
+            new ApplyNewDoctypeConfig(config, manager);
+        }
     }
 
     private static class Apply {
         public Apply(DocumentmanagerConfig config, DocumentTypeManager manager) {
             this.manager = manager;
-            this.usev8geopositions = (config == null) ? false : config.usev8geopositions();
-            if (config != null) {
-                apply(config);
-            }
+            this.usev8geopositions = config.usev8geopositions();
+            apply(config);
         }
 
         private Map<Integer, DataType> typesById = new HashMap<>();
@@ -109,15 +114,12 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                         .collect(Collectors.toUnmodifiableSet());
                     DocumentType type = new DocumentType(doc.name(), header, importedFields);
                     if (id != type.getId()) {
+                        typesById.put(id, type);
                         // really old stuff, should rewrite tests using this:
                         int alt = (doc.name()+"."+doc.version()).hashCode();
-                        if (id == alt) {
-                            typesById.put(id, type);
-                        } else {
-                            throw new IllegalArgumentException("Document type "+doc.name()+
-                                                               " wanted id "+id+" but got "+
-                                                               type.getId()+", alternative id was: "+alt);
-                        }
+                        log.warning("Document type "+doc.name()+
+                                    " wanted id "+id+" but got "+
+                                    type.getId()+", alternative id was: "+alt);
                     }
                     inProgress(type);
                     configMap.remove(id);
@@ -314,9 +316,255 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         private final DocumentTypeManager manager;
     }
 
+
+    private static class ApplyNewDoctypeConfig {
+
+        public ApplyNewDoctypeConfig(DocumentmanagerConfig config, DocumentTypeManager manager) {
+            this.manager = manager;
+            this.usev8geopositions = config.usev8geopositions();
+            apply(config);
+        }
+
+        Map<Integer, DataType> typesByIdx = new HashMap<>();
+
+        DataType addNewType(int id, DataType type) {
+            if (type == null) {
+                throw new IllegalArgumentException("Type to add for idx "+id+" cannot be null");
+            }
+            var old = typesByIdx.put(id, type);
+            if (old != null) {
+                throw new IllegalArgumentException("Type "+type+" for idx "+id+" conflict: "+old+" present");
+            }
+            return type;
+        }
+
+        Map<Integer, Supplier<DataType>> factoryByIdx = new HashMap<>();
+
+        ArrayList<Integer> proxyRefs = new ArrayList<>();
+
+        private DataType getOrCreateType(int id) {
+            if (typesByIdx.containsKey(id)) {
+                return typesByIdx.get(id);
+            }
+            var factory = factoryByIdx.remove(id);
+            if (factory != null) {
+                DataType type = factory.get();
+                return addNewType(id, type);
+            }
+            throw new IllegalArgumentException("No type or factory found for idx: "+id);
+        }
+
+        void createComplexTypes() {
+            var toCreate = new ArrayList<Integer>(factoryByIdx.keySet());
+            for (var dataTypeId : toCreate) {
+                var type = getOrCreateType(dataTypeId);
+                assert(type != null);
+            }
+        }
+
+        class PerDocTypeData {
+            DocumentmanagerConfig.Doctype docTypeConfig;
+
+            DocumentType docType = null;
+
+            PerDocTypeData(DocumentmanagerConfig.Doctype config) {
+                this.docTypeConfig = config;
+            }
+
+            void createSimpleTypes() {
+                for (var typeconf : docTypeConfig.primitivetype()) {
+                    DataType type = manager.getDataType(typeconf.name());
+                    if (! (type instanceof PrimitiveDataType)) {
+                        throw new IllegalArgumentException("Needed primitive type for idx "+typeconf.idx()+" but got: "+type);
+                    }
+                    addNewType(typeconf.idx(), type);
+                }
+                for (var typeconf : docTypeConfig.tensortype()) {
+                    var type = new TensorDataType(TensorType.fromSpec(typeconf.detailedtype()));
+                    addNewType(typeconf.idx(), type);
+                }
+            }
+
+            void createFactories() {
+                for (var typeconf : docTypeConfig.arraytype()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new ArrayDataType(getOrCreateType(typeconf.elementtype())));
+                }
+                for (var typeconf : docTypeConfig.maptype()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new MapDataType(getOrCreateType(typeconf.keytype()),
+                                                                           getOrCreateType(typeconf.valuetype())));
+                }
+                for (var typeconf : docTypeConfig.wsettype()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new WeightedSetDataType(getOrCreateType(typeconf.elementtype()),
+                                                                                   typeconf.createifnonexistent(),
+                                                                                   typeconf.removeifzero()));
+                }
+                for (var typeconf : docTypeConfig.documentref()) {
+                    factoryByIdx.put(typeconf.idx(), () -> ReferenceDataType.createWithInferredId(inProgressById.get(typeconf.targettype()).docType));
+                }
+                for (var typeconf : docTypeConfig.annotationref()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new AnnotationReferenceDataType
+                                     (annTypeFromIdx(typeconf.annotationtype())));
+                }
+            }
+
+            void createEmptyStructs() {
+                String docName = docTypeConfig.name();
+                for (var typeconf : docTypeConfig.structtype()) {
+                    addNewType(typeconf.idx(), new StructDataType(typeconf.name()));
+                }
+            }
+
+            void initializeDocType() {
+                Set<String> importedFields = new HashSet<>();
+                for (var imported : docTypeConfig.importedfield()) {
+                    importedFields.add(imported.name());
+                }
+                int contentIdx = docTypeConfig.contentstruct();
+                DataType contentStruct = typesByIdx.get(contentIdx);
+                if (! (contentStruct instanceof StructDataType)) {
+                    throw new IllegalArgumentException("Content struct for document type "+docTypeConfig.name()+
+                                                       " should be a struct, but was: "+contentStruct);
+                }
+                if (docTypeConfig.name().equals(DataType.DOCUMENT.getName())) {
+                    this.docType = DataType.DOCUMENT;
+                } else {
+                    this.docType = new DocumentType(docTypeConfig.name(), (StructDataType)contentStruct, importedFields);
+                }
+                addNewType(docTypeConfig.idx(), docType);
+            }
+
+            void createEmptyAnnotationTypes() {
+                for (var typeconf : docTypeConfig.annotationtype()) {
+                    AnnotationType annType = manager.getAnnotationTypeRegistry().getType(typeconf.name());
+                    if (typeconf.internalid() != -1) {
+                        if (annType == null) {
+                            annType = new AnnotationType(typeconf.name(), typeconf.internalid());
+                        } else {
+                            if (annType.getId() != typeconf.internalid()) {
+                                throw new IllegalArgumentException("Wrong internalid for annotation type "+annType+
+                                                                   " (wanted "+typeconf.internalid()+", got "+annType.getId()+")");
+                            }
+                        }
+                    } else if (annType == null) {
+                        annType = new AnnotationType(typeconf.name());
+                    }
+                    manager.getAnnotationTypeRegistry().register(annType);
+                    // because AnnotationType is not a DataType, make a proxy
+                    var proxy = new AnnotationReferenceDataType(annType);
+                    proxyRefs.add(typeconf.idx());
+                    addNewType(typeconf.idx(), proxy);
+                }
+            }
+
+            AnnotationType annTypeFromIdx(int idx) {
+                var proxy = (AnnotationReferenceDataType) typesByIdx.get(idx);
+                if (proxy == null) {
+                    throw new IllegalArgumentException("Needed AnnotationType for idx "+idx+", found: "+typesByIdx.get(idx));
+                }
+                return proxy.getAnnotationType();
+            }
+
+            void fillAnnotationTypes() {
+                for (var typeConf : docTypeConfig.annotationtype()) {
+                    var annType = annTypeFromIdx(typeConf.idx());
+                    int pIdx = typeConf.datatype();
+                    if (pIdx != -1) {
+                        DataType payload = getOrCreateType(pIdx);
+                        annType.setDataType(payload);
+                    }
+                    for (var inherit : typeConf.inherits()) {
+                        var inheritedType = annTypeFromIdx(inherit.idx());
+                        if (! annType.inherits(inheritedType)) {
+                            annType.inherit(inheritedType);
+                        }
+                    }
+                }
+            }
+            void fillStructs() {
+                for (var structCfg : docTypeConfig.structtype()) {
+                    int idx = structCfg.idx();
+                    StructDataType type = (StructDataType) typesByIdx.get(idx);
+                    for (var parent : structCfg.inherits()) {
+                        var parentStruct = (StructDataType) typesByIdx.get(parent.type());
+                        type.inherit(parentStruct);
+                    }
+                    for (var fieldCfg : structCfg.field()) {
+                        if (fieldCfg.type() == idx) {
+                            log.fine("Self-referencing struct "+structCfg.name()+" field: "+fieldCfg);
+                        }
+                        DataType fieldType = getOrCreateType(fieldCfg.type());
+                        type.addField(new Field(fieldCfg.name(), fieldCfg.internalid(), fieldType));
+                    }
+                }
+            }
+            void fillDocument() {
+                for (var inherit : docTypeConfig.inherits()) {
+                    var data = inProgressById.get(inherit.idx());
+                    if (data == null) {
+                        throw new IllegalArgumentException("Missing doctype for inherit idx: "+inherit.idx());
+                    } else {
+                        docType.inherit(data.docType);
+                    }
+                }
+                Map<String, Collection<String>> fieldSets = new HashMap<>();
+                for (var entry : docTypeConfig.fieldsets().entrySet()) {
+                    fieldSets.put(entry.getKey(), entry.getValue().fields());
+                }
+                Set<String> importedFields = new HashSet<>();
+                for (var imported : docTypeConfig.importedfield()) {
+                    importedFields.add(imported.name());
+                }
+                docType.addFieldSets(fieldSets);
+            }
+        }
+
+        private Map<String, PerDocTypeData> inProgressByName = new HashMap<>();
+        private Map<Integer, PerDocTypeData> inProgressById = new HashMap<>();
+
+        private void apply(DocumentmanagerConfig config) {
+            for (var docType : config.doctype()) {
+                int idx = docType.idx();
+                String name = docType.name();
+                var data = new PerDocTypeData(docType);
+                var old = inProgressById.put(idx, data);
+                if (old != null) {
+                    throw new IllegalArgumentException("Multiple document types with id: "+idx);
+                }
+                old = inProgressByName.put(name, data);
+                if (old != null) {
+                    throw new IllegalArgumentException("Multiple document types with name: "+name);
+                }
+            }
+            for (var docType : config.doctype()) {
+                var docTypeData = inProgressById.get(docType.idx());
+                docTypeData.createEmptyStructs();
+                docTypeData.initializeDocType();
+                docTypeData.createEmptyAnnotationTypes();
+                docTypeData.createFactories();
+                docTypeData.createSimpleTypes();
+            }
+            createComplexTypes();
+            for (var docType : config.doctype()) {
+                var docTypeData = inProgressById.get(docType.idx());
+                docTypeData.fillStructs();
+                docTypeData.fillDocument();
+                docTypeData.fillAnnotationTypes();
+            }
+            for (int idx : proxyRefs) {
+                typesByIdx.remove(idx);
+            }
+            for (DataType type : typesByIdx.values()) {
+                manager.register(type);
+            }
+        }
+
+        private final boolean usev8geopositions;
+        private final DocumentTypeManager manager;
+    }
+
     public static DocumentTypeManager configureNewManager(DocumentmanagerConfig config) {
         DocumentTypeManager manager = new DocumentTypeManager();
-        new Apply(config, manager);
+        configureNewManager(config, manager);
         return manager;
     }