Merge pull request #20384 from vespa-engine/arnej/proposed-model-for-document-manager-config

proposal for new documentmanager config

5 files changed, 518 insertions, 122 deletions

diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index 1f9e494aa29..c950f6b5163 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -11,15 +11,19 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.function.Supplier;
+import com.yahoo.tensor.TensorType;
 
 /**
  * Configures the Vespa document manager from a config id.
  *
  * @author Einar M R Rosenvinge
  */
-public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig>{
+public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig> {
 
     private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName());
 
@@ -65,15 +69,16 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
             return;
         }
         new Apply(config, manager);
+        if (config.datatype().size() == 0 && config.annotationtype().size() == 0) {
+            new ApplyNewDoctypeConfig(config, manager);
+        }
     }
 
     private static class Apply {
         public Apply(DocumentmanagerConfig config, DocumentTypeManager manager) {
             this.manager = manager;
-            this.usev8geopositions = (config == null) ? false : config.usev8geopositions();
-            if (config != null) {
-                apply(config);
-            }
+            this.usev8geopositions = config.usev8geopositions();
+            apply(config);
         }
 
         private Map<Integer, DataType> typesById = new HashMap<>();
@@ -109,15 +114,12 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                         .collect(Collectors.toUnmodifiableSet());
                     DocumentType type = new DocumentType(doc.name(), header, importedFields);
                     if (id != type.getId()) {
+                        typesById.put(id, type);
                         // really old stuff, should rewrite tests using this:
                         int alt = (doc.name()+"."+doc.version()).hashCode();
-                        if (id == alt) {
-                            typesById.put(id, type);
-                        } else {
-                            throw new IllegalArgumentException("Document type "+doc.name()+
-                                                               " wanted id "+id+" but got "+
-                                                               type.getId()+", alternative id was: "+alt);
-                        }
+                        log.warning("Document type "+doc.name()+
+                                    " wanted id "+id+" but got "+
+                                    type.getId()+", alternative id was: "+alt);
                     }
                     inProgress(type);
                     configMap.remove(id);
@@ -314,9 +316,255 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         private final DocumentTypeManager manager;
     }
 
+
+    private static class ApplyNewDoctypeConfig {
+
+        public ApplyNewDoctypeConfig(DocumentmanagerConfig config, DocumentTypeManager manager) {
+            this.manager = manager;
+            this.usev8geopositions = config.usev8geopositions();
+            apply(config);
+        }
+
+        Map<Integer, DataType> typesByIdx = new HashMap<>();
+
+        DataType addNewType(int id, DataType type) {
+            if (type == null) {
+                throw new IllegalArgumentException("Type to add for idx "+id+" cannot be null");
+            }
+            var old = typesByIdx.put(id, type);
+            if (old != null) {
+                throw new IllegalArgumentException("Type "+type+" for idx "+id+" conflict: "+old+" present");
+            }
+            return type;
+        }
+
+        Map<Integer, Supplier<DataType>> factoryByIdx = new HashMap<>();
+
+        ArrayList<Integer> proxyRefs = new ArrayList<>();
+
+        private DataType getOrCreateType(int id) {
+            if (typesByIdx.containsKey(id)) {
+                return typesByIdx.get(id);
+            }
+            var factory = factoryByIdx.remove(id);
+            if (factory != null) {
+                DataType type = factory.get();
+                return addNewType(id, type);
+            }
+            throw new IllegalArgumentException("No type or factory found for idx: "+id);
+        }
+
+        void createComplexTypes() {
+            var toCreate = new ArrayList<Integer>(factoryByIdx.keySet());
+            for (var dataTypeId : toCreate) {
+                var type = getOrCreateType(dataTypeId);
+                assert(type != null);
+            }
+        }
+
+        class PerDocTypeData {
+            DocumentmanagerConfig.Doctype docTypeConfig;
+
+            DocumentType docType = null;
+
+            PerDocTypeData(DocumentmanagerConfig.Doctype config) {
+                this.docTypeConfig = config;
+            }
+
+            void createSimpleTypes() {
+                for (var typeconf : docTypeConfig.primitivetype()) {
+                    DataType type = manager.getDataType(typeconf.name());
+                    if (! (type instanceof PrimitiveDataType)) {
+                        throw new IllegalArgumentException("Needed primitive type for idx "+typeconf.idx()+" but got: "+type);
+                    }
+                    addNewType(typeconf.idx(), type);
+                }
+                for (var typeconf : docTypeConfig.tensortype()) {
+                    var type = new TensorDataType(TensorType.fromSpec(typeconf.detailedtype()));
+                    addNewType(typeconf.idx(), type);
+                }
+            }
+
+            void createFactories() {
+                for (var typeconf : docTypeConfig.arraytype()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new ArrayDataType(getOrCreateType(typeconf.elementtype())));
+                }
+                for (var typeconf : docTypeConfig.maptype()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new MapDataType(getOrCreateType(typeconf.keytype()),
+                                                                           getOrCreateType(typeconf.valuetype())));
+                }
+                for (var typeconf : docTypeConfig.wsettype()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new WeightedSetDataType(getOrCreateType(typeconf.elementtype()),
+                                                                                   typeconf.createifnonexistent(),
+                                                                                   typeconf.removeifzero()));
+                }
+                for (var typeconf : docTypeConfig.documentref()) {
+                    factoryByIdx.put(typeconf.idx(), () -> ReferenceDataType.createWithInferredId(inProgressById.get(typeconf.targettype()).docType));
+                }
+                for (var typeconf : docTypeConfig.annotationref()) {
+                    factoryByIdx.put(typeconf.idx(), () -> new AnnotationReferenceDataType
+                                     (annTypeFromIdx(typeconf.annotationtype())));
+                }
+            }
+
+            void createEmptyStructs() {
+                String docName = docTypeConfig.name();
+                for (var typeconf : docTypeConfig.structtype()) {
+                    addNewType(typeconf.idx(), new StructDataType(typeconf.name()));
+                }
+            }
+
+            void initializeDocType() {
+                Set<String> importedFields = new HashSet<>();
+                for (var imported : docTypeConfig.importedfield()) {
+                    importedFields.add(imported.name());
+                }
+                int contentIdx = docTypeConfig.contentstruct();
+                DataType contentStruct = typesByIdx.get(contentIdx);
+                if (! (contentStruct instanceof StructDataType)) {
+                    throw new IllegalArgumentException("Content struct for document type "+docTypeConfig.name()+
+                                                       " should be a struct, but was: "+contentStruct);
+                }
+                if (docTypeConfig.name().equals(DataType.DOCUMENT.getName())) {
+                    this.docType = DataType.DOCUMENT;
+                } else {
+                    this.docType = new DocumentType(docTypeConfig.name(), (StructDataType)contentStruct, importedFields);
+                }
+                addNewType(docTypeConfig.idx(), docType);
+            }
+
+            void createEmptyAnnotationTypes() {
+                for (var typeconf : docTypeConfig.annotationtype()) {
+                    AnnotationType annType = manager.getAnnotationTypeRegistry().getType(typeconf.name());
+                    if (typeconf.internalid() != -1) {
+                        if (annType == null) {
+                            annType = new AnnotationType(typeconf.name(), typeconf.internalid());
+                        } else {
+                            if (annType.getId() != typeconf.internalid()) {
+                                throw new IllegalArgumentException("Wrong internalid for annotation type "+annType+
+                                                                   " (wanted "+typeconf.internalid()+", got "+annType.getId()+")");
+                            }
+                        }
+                    } else if (annType == null) {
+                        annType = new AnnotationType(typeconf.name());
+                    }
+                    manager.getAnnotationTypeRegistry().register(annType);
+                    // because AnnotationType is not a DataType, make a proxy
+                    var proxy = new AnnotationReferenceDataType(annType);
+                    proxyRefs.add(typeconf.idx());
+                    addNewType(typeconf.idx(), proxy);
+                }
+            }
+
+            AnnotationType annTypeFromIdx(int idx) {
+                var proxy = (AnnotationReferenceDataType) typesByIdx.get(idx);
+                if (proxy == null) {
+                    throw new IllegalArgumentException("Needed AnnotationType for idx "+idx+", found: "+typesByIdx.get(idx));
+                }
+                return proxy.getAnnotationType();
+            }
+
+            void fillAnnotationTypes() {
+                for (var typeConf : docTypeConfig.annotationtype()) {
+                    var annType = annTypeFromIdx(typeConf.idx());
+                    int pIdx = typeConf.datatype();
+                    if (pIdx != -1) {
+                        DataType payload = getOrCreateType(pIdx);
+                        annType.setDataType(payload);
+                    }
+                    for (var inherit : typeConf.inherits()) {
+                        var inheritedType = annTypeFromIdx(inherit.idx());
+                        if (! annType.inherits(inheritedType)) {
+                            annType.inherit(inheritedType);
+                        }
+                    }
+                }
+            }
+            void fillStructs() {
+                for (var structCfg : docTypeConfig.structtype()) {
+                    int idx = structCfg.idx();
+                    StructDataType type = (StructDataType) typesByIdx.get(idx);
+                    for (var parent : structCfg.inherits()) {
+                        var parentStruct = (StructDataType) typesByIdx.get(parent.type());
+                        type.inherit(parentStruct);
+                    }
+                    for (var fieldCfg : structCfg.field()) {
+                        if (fieldCfg.type() == idx) {
+                            log.fine("Self-referencing struct "+structCfg.name()+" field: "+fieldCfg);
+                        }
+                        DataType fieldType = getOrCreateType(fieldCfg.type());
+                        type.addField(new Field(fieldCfg.name(), fieldCfg.internalid(), fieldType));
+                    }
+                }
+            }
+            void fillDocument() {
+                for (var inherit : docTypeConfig.inherits()) {
+                    var data = inProgressById.get(inherit.idx());
+                    if (data == null) {
+                        throw new IllegalArgumentException("Missing doctype for inherit idx: "+inherit.idx());
+                    } else {
+                        docType.inherit(data.docType);
+                    }
+                }
+                Map<String, Collection<String>> fieldSets = new HashMap<>();
+                for (var entry : docTypeConfig.fieldsets().entrySet()) {
+                    fieldSets.put(entry.getKey(), entry.getValue().fields());
+                }
+                Set<String> importedFields = new HashSet<>();
+                for (var imported : docTypeConfig.importedfield()) {
+                    importedFields.add(imported.name());
+                }
+                docType.addFieldSets(fieldSets);
+            }
+        }
+
+        private Map<String, PerDocTypeData> inProgressByName = new HashMap<>();
+        private Map<Integer, PerDocTypeData> inProgressById = new HashMap<>();
+
+        private void apply(DocumentmanagerConfig config) {
+            for (var docType : config.doctype()) {
+                int idx = docType.idx();
+                String name = docType.name();
+                var data = new PerDocTypeData(docType);
+                var old = inProgressById.put(idx, data);
+                if (old != null) {
+                    throw new IllegalArgumentException("Multiple document types with id: "+idx);
+                }
+                old = inProgressByName.put(name, data);
+                if (old != null) {
+                    throw new IllegalArgumentException("Multiple document types with name: "+name);
+                }
+            }
+            for (var docType : config.doctype()) {
+                var docTypeData = inProgressById.get(docType.idx());
+                docTypeData.createEmptyStructs();
+                docTypeData.initializeDocType();
+                docTypeData.createEmptyAnnotationTypes();
+                docTypeData.createFactories();
+                docTypeData.createSimpleTypes();
+            }
+            createComplexTypes();
+            for (var docType : config.doctype()) {
+                var docTypeData = inProgressById.get(docType.idx());
+                docTypeData.fillStructs();
+                docTypeData.fillDocument();
+                docTypeData.fillAnnotationTypes();
+            }
+            for (int idx : proxyRefs) {
+                typesByIdx.remove(idx);
+            }
+            for (DataType type : typesByIdx.values()) {
+                manager.register(type);
+            }
+        }
+
+        private final boolean usev8geopositions;
+        private final DocumentTypeManager manager;
+    }
+
     public static DocumentTypeManager configureNewManager(DocumentmanagerConfig config) {
         DocumentTypeManager manager = new DocumentTypeManager();
-        new Apply(config, manager);
+        configureNewManager(config, manager);
         return manager;
     }
 
diff --git a/document/src/main/java/com/yahoo/document/annotation/Annotation.java b/document/src/main/java/com/yahoo/document/annotation/Annotation.java
index a5f70c2b9e3..2ee2d0baaa7 100644
--- a/document/src/main/java/com/yahoo/document/annotation/Annotation.java
+++ b/document/src/main/java/com/yahoo/document/annotation/Annotation.java
@@ -223,6 +223,7 @@ public class Annotation implements Comparable<Annotation> {
     public String toString() {
         String retval = "annotation of type " + type;
         retval += ((value == null) ? " (no value)" : " (with value)");
+        retval += ((spanNode == null) ? " (no span)" : (" with span "+spanNode));
         return retval;
     }
 
diff --git a/document/src/test/document/documentmanager.cfg b/document/src/test/document/documentmanager.cfg
index e4c581304ce..6ceda63e606 100644
--- a/document/src/test/document/documentmanager.cfg
+++ b/document/src/test/document/documentmanager.cfg
@@ -1,105 +1,96 @@
-datatype[11]
-datatype[0].id -1365874599
-datatype[0].arraytype[0]
-datatype[0].weightedsettype[0]
-datatype[0].structtype[1]
-datatype[0].structtype[0].name foobar.header
-datatype[0].structtype[0].version 9
-datatype[0].structtype[0].field[2]
-datatype[0].structtype[0].field[0].name foobarfield1
-datatype[0].structtype[0].field[0].id[0]
-datatype[0].structtype[0].field[0].datatype 4
-datatype[0].structtype[0].field[1].name foobarfield0
-datatype[0].structtype[0].field[1].id[0]
-datatype[0].structtype[0].field[1].datatype 2
-datatype[0].documenttype[0]
-datatype[1].id 278604398
-datatype[1].arraytype[0]
-datatype[1].weightedsettype[0]
-datatype[1].structtype[1]
-datatype[1].structtype[0].name foobar.body
-datatype[1].structtype[0].version 9
-datatype[1].documenttype[0]
-datatype[2].id 378030104
-datatype[2].arraytype[0]
-datatype[2].weightedsettype[0]
-datatype[2].structtype[0]
-datatype[2].documenttype[1]
-datatype[2].documenttype[0].name foobar
-datatype[2].documenttype[0].version 9
-datatype[2].documenttype[0].inherits[0]
-datatype[2].documenttype[0].headerstruct -1365874599
-datatype[2].documenttype[0].bodystruct 278604398
-datatype[3].id 673066331
-datatype[3].arraytype[0]
-datatype[3].weightedsettype[0]
-datatype[3].structtype[1]
-datatype[3].structtype[0].name banana.header
-datatype[3].structtype[0].version 234
-datatype[3].structtype[0].field[1]
-datatype[3].structtype[0].field[0].name bananafield0
-datatype[3].structtype[0].field[0].id[0]
-datatype[3].structtype[0].field[0].datatype 16
-datatype[3].documenttype[0]
-datatype[4].id -176986064
-datatype[4].arraytype[0]
-datatype[4].weightedsettype[0]
-datatype[4].structtype[1]
-datatype[4].structtype[0].name banana.body
-datatype[4].structtype[0].version 234
-datatype[4].documenttype[0]
-datatype[5].id 556449802
-datatype[5].arraytype[0]
-datatype[5].weightedsettype[0]
-datatype[5].structtype[0]
-datatype[5].documenttype[1]
-datatype[5].documenttype[0].name banana
-datatype[5].documenttype[0].version 234
-datatype[5].documenttype[0].inherits[1]
-datatype[5].documenttype[0].inherits[0].name foobar
-datatype[5].documenttype[0].inherits[0].version 9
-datatype[5].documenttype[0].headerstruct 673066331
-datatype[5].documenttype[0].bodystruct -176986064
-datatype[6].id -858669928
-datatype[6].arraytype[0]
-datatype[6].weightedsettype[0]
-datatype[6].structtype[1]
-datatype[6].structtype[0].name customtypes.header
-datatype[6].structtype[0].version 3
-datatype[6].structtype[0].field[2]
-datatype[6].structtype[0].field[0].name arrayfloat
-datatype[6].structtype[0].field[0].id[0]
-datatype[6].structtype[0].field[0].datatype 99
-datatype[6].structtype[0].field[1].name arrayarrayfloat
-datatype[6].structtype[0].field[1].id[0]
-datatype[6].structtype[0].field[1].datatype 4003
-datatype[6].documenttype[0]
-datatype[7].id 99
-datatype[7].arraytype[1]
-datatype[7].arraytype[0].datatype 1
-datatype[7].weightedsettype[0]
-datatype[7].structtype[0]
-datatype[7].documenttype[0]
-datatype[8].id 4003
-datatype[8].arraytype[1]
-datatype[8].arraytype[0].datatype 99
-datatype[8].weightedsettype[0]
-datatype[8].structtype[0]
-datatype[8].documenttype[0]
-datatype[9].id 2142817261
-datatype[9].arraytype[0]
-datatype[9].weightedsettype[0]
-datatype[9].structtype[1]
-datatype[9].structtype[0].name customtypes.body
-datatype[9].structtype[0].version 3
-datatype[9].documenttype[0]
-datatype[10].id -1500313747
-datatype[10].arraytype[0]
-datatype[10].weightedsettype[0]
-datatype[10].structtype[0]
-datatype[10].documenttype[1]
-datatype[10].documenttype[0].name customtypes
-datatype[10].documenttype[0].version 3
-datatype[10].documenttype[0].inherits[0]
-datatype[10].documenttype[0].headerstruct -858669928
-datatype[10].documenttype[0].bodystruct 2142817261
+doctype[4]
+doctype[0].name "document"
+doctype[0].idx 1000
+doctype[0].contentstruct 1001
+doctype[0].primitivetype[0].idx 1002
+doctype[0].primitivetype[0].name "int"
+doctype[0].primitivetype[1].idx 1003
+doctype[0].primitivetype[1].name "double"
+doctype[0].primitivetype[2].idx 1004
+doctype[0].primitivetype[2].name "string"
+doctype[0].annotationtype[0].idx 1005
+doctype[0].annotationtype[0].name "proximity_break"
+doctype[0].annotationtype[0].internalid 8
+doctype[0].annotationtype[0].datatype 1003
+doctype[0].annotationtype[1].idx 1006
+doctype[0].annotationtype[1].name "normalized"
+doctype[0].annotationtype[1].internalid 4
+doctype[0].annotationtype[1].datatype 1004
+doctype[0].annotationtype[2].idx 1007
+doctype[0].annotationtype[2].name "reading"
+doctype[0].annotationtype[2].internalid 5
+doctype[0].annotationtype[2].datatype 1004
+doctype[0].annotationtype[3].idx 1008
+doctype[0].annotationtype[3].name "term"
+doctype[0].annotationtype[3].internalid 1
+doctype[0].annotationtype[3].datatype 1004
+doctype[0].annotationtype[4].idx 1009
+doctype[0].annotationtype[4].name "transformed"
+doctype[0].annotationtype[4].internalid 7
+doctype[0].annotationtype[4].datatype 1004
+doctype[0].annotationtype[5].idx 1010
+doctype[0].annotationtype[5].name "canonical"
+doctype[0].annotationtype[5].internalid 3
+doctype[0].annotationtype[5].datatype 1004
+doctype[0].annotationtype[6].idx 1011
+doctype[0].annotationtype[6].name "token_type"
+doctype[0].annotationtype[6].internalid 2
+doctype[0].annotationtype[6].datatype 1002
+doctype[0].annotationtype[7].idx 1012
+doctype[0].annotationtype[7].name "special_token"
+doctype[0].annotationtype[7].internalid 9
+doctype[0].annotationtype[8].idx 1013
+doctype[0].annotationtype[8].name "stem"
+doctype[0].annotationtype[8].internalid 6
+doctype[0].annotationtype[8].datatype 1004
+doctype[0].structtype[0].idx 1001
+doctype[0].structtype[0].name document.header
+doctype[1].name "foobar"
+doctype[1].idx 1014
+doctype[1].inherits[0].idx 1000
+doctype[1].contentstruct 1015
+doctype[1].primitivetype[0].idx 1016
+doctype[1].primitivetype[0].name "long"
+doctype[1].structtype[0].idx 1015
+doctype[1].structtype[0].name foobar.header
+doctype[1].structtype[0].field[0].name "foobarfield1"
+doctype[1].structtype[0].field[0].internalid 1707020592
+doctype[1].structtype[0].field[0].type 1016
+doctype[1].structtype[0].field[1].name "foobarfield0"
+doctype[1].structtype[0].field[1].internalid 1055920092
+doctype[1].structtype[0].field[1].type 1004
+doctype[2].name "banana"
+doctype[2].idx 1017
+doctype[2].inherits[0].idx 1014
+doctype[2].contentstruct 1018
+doctype[2].primitivetype[0].idx 1019
+doctype[2].primitivetype[0].name "byte"
+doctype[2].structtype[0].idx 1018
+doctype[2].structtype[0].name banana.header
+doctype[2].structtype[0].field[0].name "foobarfield1"
+doctype[2].structtype[0].field[0].internalid 1707020592
+doctype[2].structtype[0].field[0].type 1016
+doctype[2].structtype[0].field[1].name "foobarfield0"
+doctype[2].structtype[0].field[1].internalid 1055920092
+doctype[2].structtype[0].field[1].type 1004
+doctype[2].structtype[0].field[2].name "bananafield0"
+doctype[2].structtype[0].field[2].internalid 1294599520
+doctype[2].structtype[0].field[2].type 1019
+doctype[3].name "customtypes"
+doctype[3].idx 1020
+doctype[3].inherits[0].idx 1000
+doctype[3].contentstruct 1021
+doctype[3].primitivetype[0].idx 1023
+doctype[3].primitivetype[0].name "float"
+doctype[3].arraytype[0].idx 1022
+doctype[3].arraytype[0].elementtype 1024
+doctype[3].arraytype[1].idx 1024
+doctype[3].arraytype[1].elementtype 1023
+doctype[3].structtype[0].idx 1021
+doctype[3].structtype[0].name customtypes.header
+doctype[3].structtype[0].field[0].name "arrayfloat"
+doctype[3].structtype[0].field[0].internalid 1493411963
+doctype[3].structtype[0].field[0].type 1024
+doctype[3].structtype[0].field[1].name "arrayarrayfloat"
+doctype[3].structtype[0].field[1].internalid 890649191
+doctype[3].structtype[0].field[1].type 1022
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
index 4040f3455da..eb5249227be 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
@@ -190,7 +190,7 @@ public class DocumentTypeManagerTestCase {
 
         Field arrayfloat = type.getField("arrayfloat");
         ArrayDataType dataType = (ArrayDataType) arrayfloat.getDataType();
-        assertTrue(dataType.getCode() == 99);
+        // assertTrue(dataType.getCode() == 99);
         assertTrue(dataType.getValueClass().equals(Array.class));
         assertTrue(dataType.getNestedType().getCode() == 1);
         assertTrue(dataType.getNestedType().getValueClass().equals(FloatFieldValue.class));
@@ -198,9 +198,9 @@ public class DocumentTypeManagerTestCase {
 
         Field arrayarrayfloat = type.getField("arrayarrayfloat");
         ArrayDataType subType = (ArrayDataType) arrayarrayfloat.getDataType();
-        assertTrue(subType.getCode() == 4003);
+        // assertTrue(subType.getCode() == 4003);
         assertTrue(subType.getValueClass().equals(Array.class));
-        assertTrue(subType.getNestedType().getCode() == 99);
+        // assertTrue(subType.getNestedType().getCode() == 99);
         assertTrue(subType.getNestedType().getValueClass().equals(Array.class));
         ArrayDataType subSubType = (ArrayDataType) subType.getNestedType();
         assertTrue(subSubType.getNestedType().getCode() == 1);
@@ -215,7 +215,7 @@ public class DocumentTypeManagerTestCase {
         DocumentType customtypes = manager.getDocumentType(new DataTypeName("customtypes"));
 
         assertNull(banana.getField("newfield"));
-        assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99)), customtypes.getField("arrayfloat"));
+        assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT)), customtypes.getField("arrayfloat"));
 
         var sub = DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.updated.cfg");
         sub.close();
diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def
index b9e7cc0f0d1..ec19ba8d802 100644
--- a/document/src/vespa/document/config/documentmanager.def
+++ b/document/src/vespa/document/config/documentmanager.def
@@ -107,3 +107,159 @@ annotationtype[].id int
 annotationtype[].name string
 annotationtype[].datatype int default=-1
 annotationtype[].inherits[].id int
+
+
+# Here starts a new model for how datatypes are configured, where
+# everything is per document-type, and each documenttype contains the
+# datatypes it defines.  Will be used (only?) if the arrays above
+# (datatype[] and annotationtype[]) are empty.
+
+
+# Note: we will include the built-in "document" document
+# type that all other doctypes inherit from also, in order
+# to get all the primitive and built-in types declared
+# with an idx we can refer to.
+
+## Name of the document type. Must be unique.
+doctype[].name string
+
+# Note: indexes are only meaningful as internal references in this
+# config; they will typically be sequential (1,2,3,...)  in the order
+# that they are generated (but nothing should depend on that).
+
+## Index of this type (as a datatype which can be referred to).
+doctype[].idx int
+
+# Could also use name here?
+## Specify document types to inherit
+doctype[].inherits[].idx int
+
+## Index of struct defining document fields
+doctype[].contentstruct int
+
+## Field sets available for this document type
+doctype[].fieldsets{}.fields[] string
+
+## Imported fields (specified outside the document block in the schema)
+doctype[].importedfield[].name string
+
+# Everything below here is configuration of data types defined by
+# this document type.
+
+# Primitive types must be present as built-in static members.
+
+## Index of primitive type
+doctype[].primitivetype[].idx int
+
+## The name of this primitive type
+doctype[].primitivetype[].name string
+
+# Arrays are the simplest collection type:
+
+## Index of this array type
+doctype[].arraytype[].idx int
+
+## Index of the element type this array type contains
+doctype[].arraytype[].elementtype int
+
+
+# Maps are another collection type:
+
+## Index of this map type
+doctype[].maptype[].idx int
+
+## Index of the key type used by this map type
+doctype[].maptype[].keytype int
+
+## Index of the key type used by this map type
+doctype[].maptype[].valuetype int
+
+
+# Weighted sets are more complicated;
+# they can be considered as an collection
+# of unique elements where each element has
+# an associated weight:
+
+## Index of this weighted set type
+doctype[].wsettype[].idx int
+
+## Index of the element types contained in this weighted set type
+doctype[].wsettype[].elementtype int
+
+## Should an update to a nonexistent element cause it to be created 
+doctype[].wsettype[].createifnonexistent bool default=false
+
+## Should an element in a weighted set be removed if an update changes the weight to 0
+doctype[].wsettype[].removeifzero bool default=false
+
+
+# Tensors have their own type system
+
+## Index of this tensor type
+doctype[].tensortype[].idx int
+
+## Description of the type of the actual tensors contained
+doctype[].tensortype[].detailedtype string
+
+
+# Document references refer to parent documents that a document can
+# import fields from:
+
+## Index of this reference data type:
+doctype[].documentref[].idx int
+
+# Could also use name?
+## Index of the document type this reference type refers to:
+doctype[].documentref[].targettype int
+
+
+# Annotation types are another world, but are modeled here
+# as if they were also datatypes contained inside document types:
+
+## Index of an annotation type.
+doctype[].annotationtype[].idx int
+
+## Name of the annotation type.
+doctype[].annotationtype[].name string
+
+# Could we somehow avoid this?
+## Internal id of this annotation type
+doctype[].annotationtype[].internalid int default=-1
+
+## Index of contained datatype of the annotation type, if any
+doctype[].annotationtype[].datatype int default=-1
+
+## Index of annotation type that this type inherits.
+doctype[].annotationtype[].inherits[].idx int
+
+
+# Annotation references are field values referring to
+# an annotation of a certain annotation type.
+
+## Index of this annotation reference type
+doctype[].annotationref[].idx int
+
+## Index of the annotation type this annotation reference type refers to
+doctype[].annotationref[].annotationtype int
+
+
+# A struct is just a named collections of fields:
+
+## Index of this struct type
+doctype[].structtype[].idx int
+
+## Name of the struct type. Must be unique within documenttype.
+doctype[].structtype[].name string
+
+## Index of another struct type to inherit
+doctype[].structtype[].inherits[].type int
+
+## Name of a struct field. Must be unique within the struct type.
+doctype[].structtype[].field[].name string
+
+## The "field id" - used in serialized format!
+doctype[].structtype[].field[].internalid int
+
+## Index of the type of this field
+doctype[].structtype[].field[].type int
+