Merge pull request #20422 from vespa-engine/arnej/resolve-more-temporaries

Arnej/resolve more temporaries

19 files changed, 260 insertions, 221 deletions

diff --git a/config-model/.gitignore b/config-model/.gitignore
index b0f358e8113..4cf50da0853 100644
--- a/config-model/.gitignore
+++ b/config-model/.gitignore
@@ -4,3 +4,4 @@
 /target
 /src/test/integration/*/copy/
 /src/test/integration/*/models.generated/
+*.cfg.actual
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
index 118714ca2b1..8848759b415 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java
@@ -48,9 +48,14 @@ public class DataTypeRepo implements DataTypeCollection {
         {
             throw new IllegalStateException("Data type '" + type.getName() + "' is not registered.");
         }
-        typeByName.remove(type.getName());
+        var oldByName = typeByName.remove(type.getName());
+        var oldById = typeById.remove(type.getId());
+        if (oldByName != oldById) {
+            throw new IllegalStateException("Data type '" + type.getName() +
+                                            "' inconsistent replace, by name: " + oldByName
+                                            + " but by id: " + oldById);
+        }
         typeByName.put(type.getName(), type);
-        typeById.remove(type.getId());
         typeById.put(type.getId(), type);
         return this;
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index 8809cdeacc8..170753a6ff1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -26,6 +26,7 @@ import com.yahoo.vespa.documentmodel.FieldView;
 import com.yahoo.vespa.documentmodel.SearchDef;
 import com.yahoo.vespa.documentmodel.SearchField;
 
+import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -184,57 +185,94 @@ public class DocumentModelBuilder {
             }
         }
     }
+
+    // This is how you make a "Pair" class in java....
+    private static class TypeReplacement extends AbstractMap.SimpleEntry<DataType,DataType> {
+        DataType oldType() { return getKey(); }
+        DataType newType() { return getValue(); }
+        public TypeReplacement(DataType oldType, DataType newType) {
+            super(oldType, newType);
+        }
+    }
+    
     private void addDocumentTypes(List<SDDocumentType> docList) {
         LinkedList<NewDocumentType> lst = new LinkedList<>();
         for (SDDocumentType doc : docList) {
             lst.add(convert(doc));
             model.getDocumentManager().add(lst.getLast());
         }
+        Set<TypeReplacement> replacements = new HashSet<>();
+        for(NewDocumentType doc : lst) {
+            resolveTemporaries(doc.getAllTypes(), lst, replacements);
+        }
         for(NewDocumentType doc : lst) {
-            resolveTemporaries(doc.getAllTypes(), lst);
+            for (var entry : replacements) {
+                var old = entry.oldType();
+                if (doc.getDataType(old.getId()) == old) {
+                    doc.replace(entry.newType());
+                }
+            }
         }
     }
-    private static void resolveTemporaries(DataTypeCollection dtc, Collection<NewDocumentType> docs) {
+
+    private static void resolveTemporaries(DataTypeCollection dtc,
+                                           Collection<NewDocumentType> docs,
+                                           Set<TypeReplacement> replacements)
+    {
         for (DataType type : dtc.getTypes()) {
-            resolveTemporariesRecurse(type, dtc, docs);
+            resolveTemporariesRecurse(type, dtc, docs, replacements);
         }
     }
 
     @SuppressWarnings("deprecation")
     private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo,
-                                                      Collection<NewDocumentType> docs) {
+                                                      Collection<NewDocumentType> docs,
+                                                      Set<TypeReplacement> replacements)
+    {
+        DataType original = type;
         if (type instanceof TemporaryStructuredDataType) {
-            DataType struct = repo.getDataType(type.getId());
-            if (struct != null)
-                type = struct;
-            else
-                type = getDocumentType(docs, type.getId());
-        }
-        else if (type instanceof StructDataType) {
+            DataType other = repo.getDataType(type.getId());
+            if (other == null || other == type) {
+                other = getDocumentType(docs, type.getId());
+            }
+            // maybe warning if null here?
+            if (other != null) {
+                type = other;
+            }
+        } else if (type instanceof DocumentType || type instanceof NewDocumentType) {
+            DataType other = getDocumentType(docs, type.getId());
+            // maybe warning if null here?
+            if (other != null) {
+                type = other;
+            }
+        } else if (type instanceof StructDataType) {
             StructDataType dt = (StructDataType) type;
             for (com.yahoo.document.Field field : dt.getFields()) {
                 if (field.getDataType() != type) {
                     // XXX deprecated:
-                    field.setDataType(resolveTemporariesRecurse(field.getDataType(), repo, docs));
+                    field.setDataType(resolveTemporariesRecurse(field.getDataType(), repo, docs, replacements));
                 }
             }
         }
         else if (type instanceof MapDataType) {
             MapDataType t = (MapDataType) type;
-            t.setKeyType(resolveTemporariesRecurse(t.getKeyType(), repo, docs));
-            t.setValueType(resolveTemporariesRecurse(t.getValueType(), repo, docs));
+            t.setKeyType(resolveTemporariesRecurse(t.getKeyType(), repo, docs, replacements));
+            t.setValueType(resolveTemporariesRecurse(t.getValueType(), repo, docs, replacements));
         }
         else if (type instanceof CollectionDataType) {
             CollectionDataType t = (CollectionDataType) type;
-            t.setNestedType(resolveTemporariesRecurse(t.getNestedType(), repo, docs));
+            t.setNestedType(resolveTemporariesRecurse(t.getNestedType(), repo, docs, replacements));
         }
         else if (type instanceof ReferenceDataType) {
             ReferenceDataType t = (ReferenceDataType) type;
             if (t.getTargetType() instanceof TemporaryStructuredDataType) {
-                DataType targetType = resolveTemporariesRecurse(t.getTargetType(), repo, docs);
+                DataType targetType = resolveTemporariesRecurse(t.getTargetType(), repo, docs, replacements);
                 t.setTargetType((StructuredDataType) targetType);
             }
         }
+        if (type != original) {
+            replacements.add(new TypeReplacement(original, type));
+        }
         return type;
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index fdbb1d8c8e0..4cfd5c84550 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -11,8 +11,11 @@ import com.yahoo.documentmodel.NewDocumentType;
 import com.yahoo.documentmodel.VespaDocumentType;
 import com.yahoo.searchdefinition.document.FieldSet;
 import com.yahoo.vespa.documentmodel.DocumentModel;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 /**
@@ -35,7 +38,8 @@ public class DocumentManager {
         for(NewDocumentType documentType : model.getDocumentManager().getTypes()) {
             buildConfig(documentType, documentConfigBuilder, handled);
             buildConfig(documentType.getAnnotations(), documentConfigBuilder);
-            if ( documentType != VespaDocumentType.INSTANCE) {
+            if (documentType != VespaDocumentType.INSTANCE && ! handled.contains(documentType)) {
+                handled.add(documentType);
                 DocumentmanagerConfig.Datatype.Builder dataTypeBuilder = new DocumentmanagerConfig.Datatype.Builder();
                 documentConfigBuilder.datatype(dataTypeBuilder);
                 buildConfig(documentType, dataTypeBuilder);
@@ -46,10 +50,16 @@ public class DocumentManager {
 
     @SuppressWarnings("deprecation")
     private void buildConfig(DataTypeCollection type, DocumentmanagerConfig.Builder documentConfigBuilder, Set<DataType> built) {
-        for (DataType dataType : type.getTypes()) {
+        List<DataType> todo = new ArrayList<>(type.getTypes());
+        Collections.sort(todo, (a, b) -> (a.getName().equals(b.getName())
+                                          ? a.getId() - b.getId()
+                                          : a.getName().compareTo(b.getName())));
+        for (DataType dataType : todo) {
             if (built.contains(dataType)) continue;
             built.add(dataType);
-            if (dataType instanceof TemporaryStructuredDataType) continue;
+            if (dataType instanceof TemporaryStructuredDataType) {
+                throw new IllegalArgumentException("Can not create config for temporary data type: " + dataType.getName());
+            }
             if ((dataType.getId() < 0) || (dataType.getId()> DataType.lastPredefinedDataTypeId())) {
                 Datatype.Builder dataTypeBuilder = new Datatype.Builder();
                 documentConfigBuilder.datatype(dataTypeBuilder);
@@ -98,15 +108,7 @@ public class DocumentManager {
                     keytype(mtype.getKeyType().getId()).
                     valtype(mtype.getValueType().getId()));
         } else if (type instanceof DocumentType) {
-            DocumentType dt = (DocumentType) type;
-            Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
-            builder.documenttype(doc);
-            doc.
-                name(dt.getName()).
-                headerstruct(dt.contentStruct().getId());
-            for (DocumentType inherited : dt.getInheritedTypes()) {
-                doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
-            }
+            throw new IllegalArgumentException("Can not create config for unadorned document type: " + type.getName());
         } else if (type instanceof NewDocumentType) {
             NewDocumentType dt = (NewDocumentType) type;
             Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder();
@@ -120,7 +122,7 @@ public class DocumentManager {
             buildConfig(dt.getFieldSets(), doc);
             buildImportedFieldsConfig(dt.getImportedFieldNames(), doc);
         } else if (type instanceof TemporaryStructuredDataType) {
-            //Ignored
+            throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName());
         } else if (type instanceof StructDataType) {
             StructDataType structType = (StructDataType) type;
             Datatype.Structtype.Builder structBuilder = new Datatype.Structtype.Builder();
diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg
index 66002968586..8b93e3a4665 100644
--- a/config-model/src/test/configmodel/types/documentmanager.cfg
+++ b/config-model/src/test/configmodel/types/documentmanager.cfg
@@ -13,117 +13,117 @@ datatype[0].structtype[0].field[0].detailedtype ""
 datatype[0].structtype[0].field[1].name "y"
 datatype[0].structtype[0].field[1].datatype 0
 datatype[0].structtype[0].field[1].detailedtype ""
-datatype[1].id -1865479609
-datatype[1].maptype[0].keytype 2
-datatype[1].maptype[0].valtype 4
-datatype[2].id 294108848
-datatype[2].structtype[0].name "folder"
-datatype[2].structtype[0].version 0
-datatype[2].structtype[0].compresstype NONE
-datatype[2].structtype[0].compresslevel 0
-datatype[2].structtype[0].compressthreshold 95
-datatype[2].structtype[0].compressminsize 800
-datatype[2].structtype[0].field[0].name "Version"
-datatype[2].structtype[0].field[0].datatype 0
-datatype[2].structtype[0].field[0].detailedtype ""
-datatype[2].structtype[0].field[1].name "Name"
-datatype[2].structtype[0].field[1].datatype 2
-datatype[2].structtype[0].field[1].detailedtype ""
-datatype[2].structtype[0].field[2].name "FlagsCounter"
-datatype[2].structtype[0].field[2].datatype -1865479609
-datatype[2].structtype[0].field[2].detailedtype ""
-datatype[2].structtype[0].field[3].name "anotherfolder"
-datatype[2].structtype[0].field[3].datatype 294108848
-datatype[2].structtype[0].field[3].detailedtype ""
-datatype[3].id 109267174
-datatype[3].structtype[0].name "sct"
-datatype[3].structtype[0].version 0
-datatype[3].structtype[0].compresstype NONE
-datatype[3].structtype[0].compresslevel 0
-datatype[3].structtype[0].compressthreshold 95
-datatype[3].structtype[0].compressminsize 800
-datatype[3].structtype[0].field[0].name "s1"
-datatype[3].structtype[0].field[0].datatype 2
-datatype[3].structtype[0].field[0].detailedtype ""
-datatype[3].structtype[0].field[1].name "s2"
-datatype[3].structtype[0].field[1].datatype 2
-datatype[3].structtype[0].field[1].detailedtype ""
-datatype[4].id 49942803
-datatype[4].arraytype[0].datatype 16
-datatype[5].id 339965458
-datatype[5].maptype[0].keytype 2
-datatype[5].maptype[0].valtype 2
-datatype[6].id -2092985853
-datatype[6].structtype[0].name "mystruct"
-datatype[6].structtype[0].version 0
-datatype[6].structtype[0].compresstype NONE
-datatype[6].structtype[0].compresslevel 0
-datatype[6].structtype[0].compressthreshold 95
-datatype[6].structtype[0].compressminsize 800
-datatype[6].structtype[0].field[0].name "bytearr"
-datatype[6].structtype[0].field[0].datatype 49942803
-datatype[6].structtype[0].field[0].detailedtype ""
-datatype[6].structtype[0].field[1].name "mymap"
-datatype[6].structtype[0].field[1].datatype 339965458
-datatype[6].structtype[0].field[1].detailedtype ""
-datatype[6].structtype[0].field[2].name "title"
-datatype[6].structtype[0].field[2].datatype 2
-datatype[6].structtype[0].field[2].detailedtype ""
-datatype[6].structtype[0].field[3].name "structfield"
-datatype[6].structtype[0].field[3].datatype 2
-datatype[6].structtype[0].field[3].detailedtype ""
-datatype[7].id -1245117006
-datatype[7].arraytype[0].datatype 0
-datatype[8].id 1328286588
-datatype[8].weightedsettype[0].datatype 2
-datatype[8].weightedsettype[0].createifnonexistant false
-datatype[8].weightedsettype[0].removeifzero false
-datatype[9].id 2125328771
-datatype[9].weightedsettype[0].datatype 2
-datatype[9].weightedsettype[0].createifnonexistant false
-datatype[9].weightedsettype[0].removeifzero true
-datatype[10].id 2065577986
-datatype[10].weightedsettype[0].datatype 2
-datatype[10].weightedsettype[0].createifnonexistant true
-datatype[10].weightedsettype[0].removeifzero false
-datatype[11].id -1244829667
-datatype[11].arraytype[0].datatype 109267174
-datatype[12].id -1584287606
-datatype[12].maptype[0].keytype 2
-datatype[12].maptype[0].valtype 0
-datatype[13].id 2125154557
-datatype[13].maptype[0].keytype 2
-datatype[13].maptype[0].valtype 1
-datatype[14].id -1715531035
+datatype[1].id -794985308
+datatype[1].arraytype[0].datatype 1707615575
+datatype[2].id 1707615575
+datatype[2].arraytype[0].datatype -1486737430
+datatype[3].id 1416345047
+datatype[3].arraytype[0].datatype -372512406
+datatype[4].id 69621385
+datatype[4].arraytype[0].datatype 339965458
+datatype[5].id 49942803
+datatype[5].arraytype[0].datatype 16
+datatype[6].id -1245117006
+datatype[6].arraytype[0].datatype 0
+datatype[7].id 759956026
+datatype[7].arraytype[0].datatype -2092985853
+datatype[8].id -1244829667
+datatype[8].arraytype[0].datatype 109267174
+datatype[9].id -1486737430
+datatype[9].arraytype[0].datatype 2
+datatype[10].id -372512406
+datatype[10].maptype[0].keytype 0
+datatype[10].maptype[0].valtype 1707615575
+datatype[11].id 2138385264
+datatype[11].maptype[0].keytype 0
+datatype[11].maptype[0].valtype 5
+datatype[12].id -389833101
+datatype[12].maptype[0].keytype 0
+datatype[12].maptype[0].valtype 294108848
+datatype[13].id -1715531035
+datatype[13].maptype[0].keytype 0
+datatype[13].maptype[0].valtype 4
+datatype[14].id 1901258752
 datatype[14].maptype[0].keytype 0
-datatype[14].maptype[0].valtype 4
-datatype[15].id 2138385264
-datatype[15].maptype[0].keytype 0
-datatype[15].maptype[0].valtype 5
-datatype[16].id 435886609
+datatype[14].maptype[0].valtype -2092985853
+datatype[15].id 435886609
+datatype[15].maptype[0].keytype 2
+datatype[15].maptype[0].valtype -1245117006
+datatype[16].id 2125154557
 datatype[16].maptype[0].keytype 2
-datatype[16].maptype[0].valtype -1245117006
-datatype[17].id -1486737430
-datatype[17].arraytype[0].datatype 2
-datatype[18].id 1707615575
-datatype[18].arraytype[0].datatype -1486737430
-datatype[19].id -794985308
-datatype[19].arraytype[0].datatype 1707615575
-datatype[20].id 69621385
-datatype[20].arraytype[0].datatype 339965458
-datatype[21].id -372512406
-datatype[21].maptype[0].keytype 0
-datatype[21].maptype[0].valtype 1707615575
-datatype[22].id 1416345047
-datatype[22].arraytype[0].datatype -372512406
-datatype[23].id 1901258752
-datatype[23].maptype[0].keytype 0
-datatype[23].maptype[0].valtype -2092985853
-datatype[24].id 759956026
-datatype[24].arraytype[0].datatype -2092985853
-datatype[25].id -389833101
-datatype[25].maptype[0].keytype 0
-datatype[25].maptype[0].valtype 294108848
+datatype[16].maptype[0].valtype 1
+datatype[17].id -1584287606
+datatype[17].maptype[0].keytype 2
+datatype[17].maptype[0].valtype 0
+datatype[18].id -1865479609
+datatype[18].maptype[0].keytype 2
+datatype[18].maptype[0].valtype 4
+datatype[19].id 339965458
+datatype[19].maptype[0].keytype 2
+datatype[19].maptype[0].valtype 2
+datatype[20].id 1328286588
+datatype[20].weightedsettype[0].datatype 2
+datatype[20].weightedsettype[0].createifnonexistant false
+datatype[20].weightedsettype[0].removeifzero false
+datatype[21].id 2065577986
+datatype[21].weightedsettype[0].datatype 2
+datatype[21].weightedsettype[0].createifnonexistant true
+datatype[21].weightedsettype[0].removeifzero false
+datatype[22].id 2125328771
+datatype[22].weightedsettype[0].datatype 2
+datatype[22].weightedsettype[0].createifnonexistant false
+datatype[22].weightedsettype[0].removeifzero true
+datatype[23].id 294108848
+datatype[23].structtype[0].name "folder"
+datatype[23].structtype[0].version 0
+datatype[23].structtype[0].compresstype NONE
+datatype[23].structtype[0].compresslevel 0
+datatype[23].structtype[0].compressthreshold 95
+datatype[23].structtype[0].compressminsize 800
+datatype[23].structtype[0].field[0].name "Version"
+datatype[23].structtype[0].field[0].datatype 0
+datatype[23].structtype[0].field[0].detailedtype ""
+datatype[23].structtype[0].field[1].name "Name"
+datatype[23].structtype[0].field[1].datatype 2
+datatype[23].structtype[0].field[1].detailedtype ""
+datatype[23].structtype[0].field[2].name "FlagsCounter"
+datatype[23].structtype[0].field[2].datatype -1865479609
+datatype[23].structtype[0].field[2].detailedtype ""
+datatype[23].structtype[0].field[3].name "anotherfolder"
+datatype[23].structtype[0].field[3].datatype 294108848
+datatype[23].structtype[0].field[3].detailedtype ""
+datatype[24].id -2092985853
+datatype[24].structtype[0].name "mystruct"
+datatype[24].structtype[0].version 0
+datatype[24].structtype[0].compresstype NONE
+datatype[24].structtype[0].compresslevel 0
+datatype[24].structtype[0].compressthreshold 95
+datatype[24].structtype[0].compressminsize 800
+datatype[24].structtype[0].field[0].name "bytearr"
+datatype[24].structtype[0].field[0].datatype 49942803
+datatype[24].structtype[0].field[0].detailedtype ""
+datatype[24].structtype[0].field[1].name "mymap"
+datatype[24].structtype[0].field[1].datatype 339965458
+datatype[24].structtype[0].field[1].detailedtype ""
+datatype[24].structtype[0].field[2].name "title"
+datatype[24].structtype[0].field[2].datatype 2
+datatype[24].structtype[0].field[2].detailedtype ""
+datatype[24].structtype[0].field[3].name "structfield"
+datatype[24].structtype[0].field[3].datatype 2
+datatype[24].structtype[0].field[3].detailedtype ""
+datatype[25].id 109267174
+datatype[25].structtype[0].name "sct"
+datatype[25].structtype[0].version 0
+datatype[25].structtype[0].compresstype NONE
+datatype[25].structtype[0].compresslevel 0
+datatype[25].structtype[0].compressthreshold 95
+datatype[25].structtype[0].compressminsize 800
+datatype[25].structtype[0].field[0].name "s1"
+datatype[25].structtype[0].field[0].datatype 2
+datatype[25].structtype[0].field[0].detailedtype ""
+datatype[25].structtype[0].field[1].name "s2"
+datatype[25].structtype[0].field[1].datatype 2
+datatype[25].structtype[0].field[1].detailedtype ""
 datatype[26].id 1328581348
 datatype[26].structtype[0].name "types.header"
 datatype[26].structtype[0].version 0
diff --git a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg
index 9be843c89aa..61c92eee8d1 100644
--- a/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg
+++ b/config-model/src/test/configmodel/types/documenttypes_with_doc_field.cfg
@@ -27,7 +27,7 @@ documenttype[1].version 0
 documenttype[1].headerstruct 1328581348
 documenttype[1].bodystruct 0
 documenttype[1].inherits[0].id 8
-documenttype[1].datatype[0].id -1368624373
+documenttype[1].datatype[0].id 1328581348
 documenttype[1].datatype[0].type STRUCT
 documenttype[1].datatype[0].array.element.id 0
 documenttype[1].datatype[0].map.key.id 0
@@ -36,29 +36,14 @@ documenttype[1].datatype[0].wset.key.id 0
 documenttype[1].datatype[0].wset.createifnonexistent false
 documenttype[1].datatype[0].wset.removeifzero false
 documenttype[1].datatype[0].annotationref.annotation.id 0
-documenttype[1].datatype[0].sstruct.name "other_doc"
+documenttype[1].datatype[0].sstruct.name "types.header"
 documenttype[1].datatype[0].sstruct.version 0
 documenttype[1].datatype[0].sstruct.compression.type NONE
 documenttype[1].datatype[0].sstruct.compression.level 0
 documenttype[1].datatype[0].sstruct.compression.threshold 95
 documenttype[1].datatype[0].sstruct.compression.minsize 200
-documenttype[1].datatype[1].id 1328581348
-documenttype[1].datatype[1].type STRUCT
-documenttype[1].datatype[1].array.element.id 0
-documenttype[1].datatype[1].map.key.id 0
-documenttype[1].datatype[1].map.value.id 0
-documenttype[1].datatype[1].wset.key.id 0
-documenttype[1].datatype[1].wset.createifnonexistent false
-documenttype[1].datatype[1].wset.removeifzero false
-documenttype[1].datatype[1].annotationref.annotation.id 0
-documenttype[1].datatype[1].sstruct.name "types.header"
-documenttype[1].datatype[1].sstruct.version 0
-documenttype[1].datatype[1].sstruct.compression.type NONE
-documenttype[1].datatype[1].sstruct.compression.level 0
-documenttype[1].datatype[1].sstruct.compression.threshold 95
-documenttype[1].datatype[1].sstruct.compression.minsize 200
-documenttype[1].datatype[1].sstruct.field[0].name "doc_field"
-documenttype[1].datatype[1].sstruct.field[0].id 819293364
-documenttype[1].datatype[1].sstruct.field[0].datatype -1368624373
-documenttype[1].datatype[1].sstruct.field[0].detailedtype ""
+documenttype[1].datatype[0].sstruct.field[0].name "doc_field"
+documenttype[1].datatype[0].sstruct.field[0].id 819293364
+documenttype[1].datatype[0].sstruct.field[0].datatype -1368624373
+documenttype[1].datatype[0].sstruct.field[0].detailedtype ""
 documenttype[1].fieldsets{[document]}.fields[0] "doc_field"
diff --git a/config-model/src/test/examples/fieldoftypedocument.cfg b/config-model/src/test/examples/fieldoftypedocument.cfg
index 5753ae556a6..82a30012a07 100644
--- a/config-model/src/test/examples/fieldoftypedocument.cfg
+++ b/config-model/src/test/examples/fieldoftypedocument.cfg
@@ -23,37 +23,37 @@ datatype[1].structtype[0].compressminsize 800
 datatype[1].structtype[0].field[0].name "soundtrack"
 datatype[1].structtype[0].field[0].datatype 1412693671
 datatype[1].structtype[0].field[0].detailedtype ""
-datatype[2].id -1383388565
-datatype[2].documenttype[0].name "book"
+datatype[2].id 1412693671
+datatype[2].documenttype[0].name "music"
 datatype[2].documenttype[0].version 0
 datatype[2].documenttype[0].inherits[0].name "document"
 datatype[2].documenttype[0].inherits[0].version 0
-datatype[2].documenttype[0].headerstruct -1344444812
+datatype[2].documenttype[0].headerstruct -1910204744
 datatype[2].documenttype[0].bodystruct 0
-datatype[2].documenttype[0].fieldsets{[document]}.fields[0] "soundtrack"
-datatype[3].id -1910204744
-datatype[3].structtype[0].name "music.header"
-datatype[3].structtype[0].version 0
-datatype[3].structtype[0].compresstype NONE
-datatype[3].structtype[0].compresslevel 0
-datatype[3].structtype[0].compressthreshold 95
-datatype[3].structtype[0].compressminsize 800
-datatype[3].structtype[0].field[0].name "intfield"
-datatype[3].structtype[0].field[0].datatype 0
-datatype[3].structtype[0].field[0].detailedtype ""
-datatype[3].structtype[0].field[1].name "stringfield"
-datatype[3].structtype[0].field[1].datatype 2
-datatype[3].structtype[0].field[1].detailedtype ""
-datatype[3].structtype[0].field[2].name "longfield"
-datatype[3].structtype[0].field[2].datatype 4
-datatype[3].structtype[0].field[2].detailedtype ""
-datatype[4].id 1412693671
-datatype[4].documenttype[0].name "music"
-datatype[4].documenttype[0].version 0
-datatype[4].documenttype[0].inherits[0].name "document"
-datatype[4].documenttype[0].inherits[0].version 0
-datatype[4].documenttype[0].headerstruct -1910204744
-datatype[4].documenttype[0].bodystruct 0
-datatype[4].documenttype[0].fieldsets{[document]}.fields[0] "intfield"
-datatype[4].documenttype[0].fieldsets{[document]}.fields[1] "longfield"
-datatype[4].documenttype[0].fieldsets{[document]}.fields[2] "stringfield"
+datatype[2].documenttype[0].fieldsets{[document]}.fields[0] "intfield"
+datatype[2].documenttype[0].fieldsets{[document]}.fields[1] "longfield"
+datatype[2].documenttype[0].fieldsets{[document]}.fields[2] "stringfield"
+datatype[3].id -1383388565
+datatype[3].documenttype[0].name "book"
+datatype[3].documenttype[0].version 0
+datatype[3].documenttype[0].inherits[0].name "document"
+datatype[3].documenttype[0].inherits[0].version 0
+datatype[3].documenttype[0].headerstruct -1344444812
+datatype[3].documenttype[0].bodystruct 0
+datatype[3].documenttype[0].fieldsets{[document]}.fields[0] "soundtrack"
+datatype[4].id -1910204744
+datatype[4].structtype[0].name "music.header"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].compresstype NONE
+datatype[4].structtype[0].compresslevel 0
+datatype[4].structtype[0].compressthreshold 95
+datatype[4].structtype[0].compressminsize 800
+datatype[4].structtype[0].field[0].name "intfield"
+datatype[4].structtype[0].field[0].datatype 0
+datatype[4].structtype[0].field[0].detailedtype ""
+datatype[4].structtype[0].field[1].name "stringfield"
+datatype[4].structtype[0].field[1].datatype 2
+datatype[4].structtype[0].field[1].detailedtype ""
+datatype[4].structtype[0].field[2].name "longfield"
+datatype[4].structtype[0].field[2].datatype 4
+datatype[4].structtype[0].field[2].detailedtype ""
diff --git a/config-model/src/test/examples/structresult.cfg b/config-model/src/test/examples/structresult.cfg
index 639d91c892d..b5b90245858 100755..100644
--- a/config-model/src/test/examples/structresult.cfg
+++ b/config-model/src/test/examples/structresult.cfg
@@ -13,32 +13,32 @@ datatype[0].structtype[0].field[0].detailedtype ""
 datatype[0].structtype[0].field[1].name "y"
 datatype[0].structtype[0].field[1].datatype 0
 datatype[0].structtype[0].field[1].detailedtype ""
-datatype[1].id 93505813
-datatype[1].structtype[0].name "bar"
-datatype[1].structtype[0].version 0
-datatype[1].structtype[0].compresstype NONE
-datatype[1].structtype[0].compresslevel 0
-datatype[1].structtype[0].compressthreshold 95
-datatype[1].structtype[0].compressminsize 800
-datatype[1].structtype[0].field[0].name "humbe"
-datatype[1].structtype[0].field[0].datatype 97614088
-datatype[1].structtype[0].field[0].detailedtype ""
-datatype[2].id 97614088
-datatype[2].structtype[0].name "foo"
+datatype[1].id -1245205573
+datatype[1].arraytype[0].datatype 97614088
+datatype[2].id 93505813
+datatype[2].structtype[0].name "bar"
 datatype[2].structtype[0].version 0
 datatype[2].structtype[0].compresstype NONE
 datatype[2].structtype[0].compresslevel 0
 datatype[2].structtype[0].compressthreshold 95
 datatype[2].structtype[0].compressminsize 800
-datatype[2].structtype[0].field[0].name "fubar"
-datatype[2].structtype[0].field[0].datatype 0
+datatype[2].structtype[0].field[0].name "humbe"
+datatype[2].structtype[0].field[0].datatype 97614088
 datatype[2].structtype[0].field[0].detailedtype ""
-datatype[2].structtype[0].field[1].name "bar"
-datatype[2].structtype[0].field[1].id[0].id 1
-datatype[2].structtype[0].field[1].datatype 2
-datatype[2].structtype[0].field[1].detailedtype ""
-datatype[3].id -1245205573
-datatype[3].arraytype[0].datatype 97614088
+datatype[3].id 97614088
+datatype[3].structtype[0].name "foo"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[3].structtype[0].field[0].name "fubar"
+datatype[3].structtype[0].field[0].datatype 0
+datatype[3].structtype[0].field[0].detailedtype ""
+datatype[3].structtype[0].field[1].name "bar"
+datatype[3].structtype[0].field[1].id[0].id 1
+datatype[3].structtype[0].field[1].datatype 2
+datatype[3].structtype[0].field[1].detailedtype ""
 datatype[4].id -1910204744
 datatype[4].structtype[0].name "music.header"
 datatype[4].structtype[0].version 0
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java
index a08ec110219..8ece5cd0fe4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java
@@ -13,6 +13,10 @@ import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals;
 public abstract class AbstractSchemaTestCase {
 
     protected static void assertConfigFile(String filename, String cfg) throws IOException {
+        IOUtils.writeFile(filename + ".actual", cfg, false);
+        if (! cfg.endsWith("\n")) {
+            IOUtils.writeFile(filename + ".actual", "\n", true);
+        }
         assertSerializedConfigFileEquals(filename, cfg);
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/empty.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/empty.cfg
index 08b0e6809ce..26aab134699 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/empty.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/empty.cfg
@@ -1 +1 @@
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/explicit-reference-override.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/explicit-reference-override.cfg
index fc77c5d82fa..5f48b7b75c2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/explicit-reference-override.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/explicit-reference-override.cfg
@@ -11,4 +11,4 @@ queryprofile[1].property[0].overridable ""
 queryprofile[1].reference[0].name "a"
 queryprofile[1].reference[0].value "a1"
 queryprofile[1].reference[0].overridable ""
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbe-query-profiles-simple.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbe-query-profiles-simple.cfg
index 337623bc448..954a6b8d68a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbe-query-profiles-simple.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsbe-query-profiles-simple.cfg
@@ -18,4 +18,4 @@ queryprofile[0].queryprofilevariant[0].fordimensionvalues[2] "sc"
 queryprofile[0].queryprofilevariant[0].property[0].name "scthumbnail.sourcecountry"
 queryprofile[0].queryprofilevariant[0].property[0].value "uk"
 queryprofile[0].queryprofilevariant[0].property[0].overridable ""
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfe-query-profiles-simple.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfe-query-profiles-simple.cfg
index b3e41d88233..33ef2610d1d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfe-query-profiles-simple.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/newsfe-query-profiles-simple.cfg
@@ -27,4 +27,4 @@ queryprofile[1].type ""
 queryprofile[1].reference[0].name "source.news"
 queryprofile[1].reference[0].value "backend/news"
 queryprofile[1].reference[0].overridable ""
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants-configuration.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants-configuration.cfg
index 8cf8385f397..c10e0b93560 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants-configuration.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants-configuration.cfg
@@ -46,4 +46,4 @@ queryprofile[2].type ""
 queryprofile[2].property[0].name "a"
 queryprofile[2].property[0].value "a1"
 queryprofile[2].property[0].overridable ""
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants2-configuration.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants2-configuration.cfg
index 0bce407e8ef..2f9879dc721 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants2-configuration.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profile-variants2-configuration.cfg
@@ -33,8 +33,8 @@ queryprofile[1].queryprofilevariant[0].property[0].value "default"
 queryprofile[1].queryprofilevariant[0].property[0].overridable ""
 queryprofile[1].queryprofilevariant[0].reference[0].name "model"
 queryprofile[1].queryprofilevariant[0].reference[0].value "querylove"
-queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "*"
 queryprofile[1].queryprofilevariant[0].reference[0].overridable ""
+queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "*"
 queryprofile[1].queryprofilevariant[1].fordimensionvalues[1] "default"
 queryprofile[1].queryprofilevariant[1].property[0].name "model.defaultIndex"
 queryprofile[1].queryprofilevariant[1].property[0].value "default"
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profiles.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profiles.cfg
index 54997e152f3..18fc48fc7c9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profiles.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/query-profiles.cfg
@@ -103,4 +103,4 @@ queryprofiletype[2].field[0].type "string"
 queryprofiletype[2].field[0].overridable false
 queryprofiletype[2].field[0].mandatory false
 queryprofiletype[2].field[0].alias ""
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound-with-reference.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound-with-reference.cfg
index fdfd1955491..bb125065671 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound-with-reference.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound-with-reference.cfg
@@ -20,9 +20,9 @@ queryprofile[2].queryprofilevariant[0].property[0].value "a.b.x1"
 queryprofile[2].queryprofilevariant[0].property[0].overridable ""
 queryprofile[2].queryprofilevariant[0].reference[0].name "a"
 queryprofile[2].queryprofilevariant[0].reference[0].value "a2"
-queryprofile[2].queryprofilevariant[1].fordimensionvalues[0] "x2"
 queryprofile[2].queryprofilevariant[0].reference[0].overridable ""
+queryprofile[2].queryprofilevariant[1].fordimensionvalues[0] "x2"
 queryprofile[2].queryprofilevariant[1].property[0].name "a.b"
 queryprofile[2].queryprofilevariant[1].property[0].value "a.b.x2"
 queryprofile[2].queryprofilevariant[1].property[0].overridable ""
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound.cfg b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound.cfg
index 6f66a3bd441..f867ca9a56b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound.cfg
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/variants-of-explicit-compound.cfg
@@ -17,4 +17,4 @@ queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "x2"
 queryprofile[1].queryprofilevariant[1].property[0].name "a.b"
 queryprofile[1].queryprofilevariant[1].property[0].value "a.b.x2"
 queryprofile[1].queryprofilevariant[1].property[0].overridable ""
-enableGroupingSessionCache true
\ No newline at end of file
+enableGroupingSessionCache true
diff --git a/config-model/src/test/java/helpers/CompareConfigTestHelper.java b/config-model/src/test/java/helpers/CompareConfigTestHelper.java
index 18c0723c6a2..ba06ecc9397 100644
--- a/config-model/src/test/java/helpers/CompareConfigTestHelper.java
+++ b/config-model/src/test/java/helpers/CompareConfigTestHelper.java
@@ -19,6 +19,10 @@ import static org.junit.Assert.assertEquals;
 public class CompareConfigTestHelper {
 
     public static void assertSerializedConfigFileEquals(String filename, String actual) throws IOException {
+        IOUtils.writeFile(filename + ".actual", actual, false);
+        if (! actual.endsWith("\n")) {
+            IOUtils.writeFile(filename + ".actual", "\n", true);
+        }
         assertSerializedConfigEquals(IOUtils.readFile(new File(filename)), actual, false);
     }