Move any single set_language operation to the top

22 files changed, 226 insertions, 148 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 1fab30f9ea4..779ae54c242 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -396,23 +396,25 @@ public class SearchBuilder {
         return builder;
     }
 
-    public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException {
-        return createFromDirectory(dir, new RankProfileRegistry());
+    public static SearchBuilder createFromDirectory(String dir, DeployLogger logger) throws IOException, ParseException {
+        return createFromDirectory(dir, new RankProfileRegistry(), logger);
     }
     public static SearchBuilder createFromDirectory(String dir,
-                                                    RankProfileRegistry rankProfileRegistry) throws IOException, ParseException {
-        return createFromDirectory(dir, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir));
+                                                    RankProfileRegistry rankProfileRegistry,
+                                                    DeployLogger logger) throws IOException, ParseException {
+        return createFromDirectory(dir, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir), logger);
     }
     public static SearchBuilder createFromDirectory(String dir,
                                                     RankProfileRegistry rankProfileRegistry,
-                                                    QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException {
+                                                    QueryProfileRegistry queryProfileRegistry,
+                                                    DeployLogger logger) throws IOException, ParseException {
         SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir),
                                                   rankProfileRegistry,
                                                   queryProfileRegistry);
         for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) {
             builder.importFile(i.next());
         }
-        builder.build(true, new BaseDeployLogger());
+        builder.build(true, logger);
         return builder;
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
index 7cfcf422e80..d6065a07656 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
@@ -39,14 +39,11 @@ public abstract class Derived implements Exportable {
     protected void derive(Search search) {
         setName(search.getName());
         derive(search.getDocument(), search);
-        for (Index index : search.getExplicitIndices()) {
+        for (Index index : search.getExplicitIndices())
             derive(index, search);
-        }
-        for (SDField field : search.allExtraFields() ) {
+        for (SDField field : search.allExtraFields())
             derive(field, search);
-        }
-        search.allImportedFields()
-                .forEach(importedField -> derive(importedField, search));
+        search.allImportedFields().forEach(importedField -> derive(importedField, search));
     }
 
 
@@ -54,10 +51,10 @@ public abstract class Derived implements Exportable {
      * Derives the content of this configuration. This
      * default calls derive(SDField) for each document field
      */
-    protected void derive(SDDocumentType document,Search search) {
+    protected void derive(SDDocumentType document, Search search) {
         for (Field field : document.fieldSet()) {
             SDField sdField = (SDField) field;
-            if (!sdField.isExtraField()) {
+            if ( ! sdField.isExtraField()) {
                 derive(sdField, search);
             }
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
index 94d475cf519..8a12bb92dae 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
 
@@ -24,6 +25,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * An indexing language script derived from a search definition. An indexing script contains a set of indexing
@@ -35,30 +37,49 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
 
     private final List<String> docFields = new ArrayList<>();
     private final List<Expression> expressions = new ArrayList<>();
+    private List<ImmutableSDField> fieldsSettingLanguage;
 
     public IndexingScript(Search search) {
         derive(search);
     }
 
     @Override
+    protected void derive(Search search) {
+        fieldsSettingLanguage = fieldsSettingLanguage(search);
+        if (fieldsSettingLanguage.size() == 1) // Assume this language should be used for all fields
+            addExpression(fieldsSettingLanguage.get(0).getIndexingScript());
+        super.derive(search);
+    }
+
+    @Override
     protected void derive(ImmutableSDField field, Search search) {
-        if (field.isImportedField()) {
-            return;
-        }
-        if (field.hasFullIndexingDocprocRights()) {
+        if (field.isImportedField()) return;
+
+        if (field.hasFullIndexingDocprocRights())
             docFields.add(field.getName());
-        }
+
         if (field.usesStructOrMap() &&
-            !field.getDataType().equals(PositionDataType.INSTANCE) &&
-            !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
-        {
+            ! field.getDataType().equals(PositionDataType.INSTANCE) &&
+            ! field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) {
             return; // unsupported
         }
-        ScriptExpression script = field.getIndexingScript();
-        if (!script.isEmpty()) {
-            expressions.add(new StatementExpression(new ClearStateExpression(),
-                                                    new GuardExpression(script)));
-        }
+
+        if (fieldsSettingLanguage.size() == 1 && fieldsSettingLanguage.get(0).equals(field))
+            return; // Already added
+
+        addExpression(field.getIndexingScript());
+    }
+
+    private void addExpression(ScriptExpression expression) {
+        if ( expression.isEmpty()) return;
+        expressions.add(new StatementExpression(new ClearStateExpression(), new GuardExpression(expression)));
+    }
+
+    private List<ImmutableSDField> fieldsSettingLanguage(Search search) {
+        return search.allFieldsList().stream()
+                                     .filter(field -> ! field.isImportedField())
+                                     .filter(field -> field.containsExpression(SetLanguageExpression.class))
+                                     .collect(Collectors.toList());
     }
 
     public Iterable<Expression> expressions() {
@@ -81,20 +102,19 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
 
     private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) {
         ArrayList<Expression> later = new ArrayList<>();
-        Set<String> touchedFields = new HashSet<String>();
-        for (Expression exp : expressions) {
+        Set<String> touchedFields = new HashSet<>();
+        for (Expression expression : expressions) {
+            if (modifiesSelf(expression) && ! setsLanguage(expression))
+                later.add(expression);
+            else
+                ilscriptBuilder.content(expression.toString());
+
             FieldScanVisitor fieldFetcher = new FieldScanVisitor();
-            if (modifiesSelf(exp)) {
-                later.add(exp);
-            } else {
-                ilscriptBuilder.content(exp.toString());
-            }
-            fieldFetcher.visit(exp);
+            fieldFetcher.visit(expression);
             touchedFields.addAll(fieldFetcher.touchedFields());
         }
-        for (Expression exp : later) {
+        for (Expression exp : later)
             ilscriptBuilder.content(exp.toString());
-        }
         generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields);
     }
 
@@ -110,13 +130,20 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
         }
     }
 
-    private boolean modifiesSelf(Expression exp) {
-        MyExpVisitor visitor = new MyExpVisitor();
-        visitor.visit(exp);
+    private boolean setsLanguage(Expression expression) {
+        SetsLanguageVisitor visitor = new SetsLanguageVisitor();
+        visitor.visit(expression);
+        return visitor.setsLanguage;
+    }
+
+    private boolean modifiesSelf(Expression expression) {
+        ModifiesSelfVisitor visitor = new ModifiesSelfVisitor();
+        visitor.visit(expression);
         return visitor.modifiesSelf();
     }
 
-    private class MyExpVisitor extends ExpressionVisitor {
+    private static class ModifiesSelfVisitor extends ExpressionVisitor {
+
         private String inputField = null;
         private String outputField = null;
 
@@ -124,9 +151,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
 
         @Override
         protected void doVisit(Expression expression) {
-            if (modifiesSelf()) {
-                return;
-            }
+            if (modifiesSelf()) return;
+
             if (expression instanceof InputExpression) {
                 inputField = ((InputExpression) expression).getFieldName();
             }
@@ -136,6 +162,18 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
         }
     }
 
+    private static class SetsLanguageVisitor extends ExpressionVisitor {
+
+        boolean setsLanguage = false;
+
+        @Override
+        protected void doVisit(Expression expression) {
+            if (expression instanceof SetLanguageExpression)
+                setsLanguage = true;
+        }
+
+    }
+
     private static class FieldScanVisitor extends ExpressionVisitor {
         List<String> touchedFields = new ArrayList<String>();
         List<String> candidates = new ArrayList<String>();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index e36635ba6b8..c3a0abc892c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -465,9 +465,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
     }
 
     @Override
-    public ScriptExpression getIndexingScript() {
-        return indexingScript;
-    }
+    public ScriptExpression getIndexingScript() { return indexingScript; }
 
     @SuppressWarnings("deprecation")
     @Override
@@ -477,7 +475,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
             stemming=Stemming.NONE;
         }
         this.dataType = type;
-        if (!idOverride) {
+        if ( ! idOverride) {
             this.fieldId = calculateIdV7(null);
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
index 8f2a29abcb6..416d956ac21 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
@@ -29,7 +29,7 @@ public class OptimizeIlscript extends Processor {
 
             field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script));
             if ( ! field.getIndexingScript().toString().equals(script.toString())) {
-                warn(search, field, "Rewrote ilscript from:\n" + script.toString() +
+                info(search, field, "Rewrote ilscript from:\n" + script.toString() +
                                     "\nto\n" + field.getIndexingScript().toString());
             }
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 3f225b00277..e8594c2a87f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -27,7 +27,6 @@ public class Processing {
                 IndexFieldNames::new,
                 IntegerIndex2Attribute::new,
                 MakeAliases::new,
-                SetLanguage::new,
                 UriHack::new,
                 LiteralBoost::new,
                 TagType::new,
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
index e15e17817a2..3744af7cc2c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
@@ -128,12 +128,22 @@ public abstract class Processor {
         throw newProcessException(search, field, msg);
     }
 
-    protected void warn(String searchName, String fieldName, String msg) {
-        String fullMsg = formatError(searchName, fieldName, msg);
+    protected void warn(String searchName, String fieldName, String message) {
+        String fullMsg = formatError(searchName, fieldName, message);
         deployLogger.log(Level.WARNING, fullMsg);
     }
 
-    protected void warn(Search search, Field field, String msg) {
-        warn(search.getName(), field.getName(), msg);
+    protected void warn(Search search, Field field, String message) {
+        warn(search.getName(), field.getName(), message);
     }
+
+    protected void info(String searchName, String fieldName, String message) {
+        String fullMsg = formatError(searchName, fieldName, message);
+        deployLogger.log(Level.INFO, fullMsg);
+    }
+
+    protected void info(Search search, Field field, String message) {
+        warn(search.getName(), field.getName(), message);
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
deleted file mode 100644
index 8a4795c4dd2..00000000000
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchdefinition.processing;
-
-import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.searchdefinition.RankProfileRegistry;
-import com.yahoo.document.DataType;
-import com.yahoo.searchdefinition.document.SDField;
-import com.yahoo.searchdefinition.Search;
-import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression;
-import com.yahoo.vespa.model.container.search.QueryProfiles;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Check that no text field appears before a field that sets language.
- *
- * @author Gunnar Gauslaa Bergem
- */
-public class SetLanguage extends Processor {
-
-    public SetLanguage(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
-        super(search, deployLogger, rankProfileRegistry, queryProfiles);
-    }
-
-    @Override
-    public void process(boolean validate, boolean documentsOnly) {
-        if ( ! validate) return;
-        List<String> textFieldsWithoutLanguage = new ArrayList<>();
-
-        for (SDField field : search.allConcreteFields()) {
-            if (fieldMustComeAfterLanguageSettingField(field)) {
-                textFieldsWithoutLanguage.add(field.getName());
-            }
-            if (field.containsExpression(SetLanguageExpression.class) && !textFieldsWithoutLanguage.isEmpty()) {
-                StringBuilder fieldString = new StringBuilder();
-                for (String fieldName : textFieldsWithoutLanguage) {
-                    fieldString.append(fieldName).append(" ");
-                }
-                warn(search, field, "Field '" + field.getName() + "' sets the language for this document, " +
-                                    "and should be defined as the first field in the searchdefinition." +
-                                    "Preceding text fields that will not have their language set: " +
-                                    fieldString.toString() +
-                                    " (This warning is omitted for any subsequent fields that also do set_language.)");
-                return;
-            }
-        }
-    }
-
-    private boolean fieldMustComeAfterLanguageSettingField(SDField field) {
-        return (!field.containsExpression(SetLanguageExpression.class) &&
-                (field.getDataType() == DataType.STRING));
-    }
-
-}
diff --git a/config-model/src/test/derived/language/ilscripts.cfg b/config-model/src/test/derived/language/ilscripts.cfg
new file mode 100644
index 00000000000..cafc7feafb8
--- /dev/null
+++ b/config-model/src/test/derived/language/ilscripts.cfg
@@ -0,0 +1,9 @@
+maxtermoccurrences 100
+fieldmatchmaxlength 1000000
+ilscript[].doctype "language"
+ilscript[].docfield[] "language"
+ilscript[].docfield[] "title"
+ilscript[].content[] "clear_state | guard { input language | tokenize normalize stem:\"BEST\" | summary language | index language | set_language; }"
+ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | index titlebest; }"
+ilscript[].content[] "clear_state | guard { input title | tokenize normalize | index titlenone; }"
+ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }"
\ No newline at end of file
diff --git a/config-model/src/test/derived/language/language.sd b/config-model/src/test/derived/language/language.sd
new file mode 100644
index 00000000000..a859438d885
--- /dev/null
+++ b/config-model/src/test/derived/language/language.sd
@@ -0,0 +1,19 @@
+schema language {
+    document language {
+        field language type string {
+            indexing: summary | index |set_language
+        }
+        field title type string {
+            indexing: summary | index
+        }
+    }
+    field titlenone type string {
+        indexing: input title | index
+        stemming: none
+    }
+    field titlebest type string {
+        indexing: input title | index
+        stemming: best
+    }
+
+}
\ No newline at end of file
diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg
index 3bcdd16e3d6..b3da5f8e727 100644
--- a/config-model/src/test/derived/types/ilscripts.cfg
+++ b/config-model/src/test/derived/types/ilscripts.cfg
@@ -21,6 +21,7 @@ ilscript[].docfield[] "doublemapfield"
 ilscript[].docfield[] "arraymapfield"
 ilscript[].docfield[] "arrarr"
 ilscript[].docfield[] "maparr"
+ilscript[].docfield[] "complexarray"
 ilscript[].docfield[] "mystructfield"
 ilscript[].docfield[] "mystructmap"
 ilscript[].docfield[] "mystructarr"
@@ -28,7 +29,6 @@ ilscript[].docfield[] "Folders"
 ilscript[].docfield[] "juletre"
 ilscript[].docfield[] "album0"
 ilscript[].docfield[] "album1"
-ilscript[].docfield[] "complexarray"
 ilscript[].content[] "clear_state | guard { input along | attribute other; }"
 ilscript[].content[] "clear_state | guard { input abyte | summary abyte | attribute abyte; }"
 ilscript[].content[] "clear_state | guard { input along | summary along | attribute along; }"
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java
index 7f3ea7d14bc..2e0fc3b4f98 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java
@@ -16,9 +16,12 @@ public abstract class SchemaTestCase {
         assertSerializedConfigFileEquals(filename, cfg);
     }
 
-    protected static void assertConfigFiles(String expectedFile, String cfgFile, boolean updateOnAssert) throws IOException {
+    protected static void assertConfigFiles(String expectedFile,
+                                            String cfgFile,
+                                            boolean orderMatters,
+                                            boolean updateOnAssert) throws IOException {
         try {
-            assertSerializedConfigEquals(readAndCensorIndexes(expectedFile), readAndCensorIndexes(cfgFile));
+            assertSerializedConfigEquals(readAndCensorIndexes(expectedFile), readAndCensorIndexes(cfgFile), orderMatters);
         } catch (AssertionError e) {
             if (updateOnAssert) {
                 BufferedWriter writer = IOUtils.createWriter(expectedFile, false);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
index a345cabe909..6b6af95a8d2 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.document.DocumenttypesConfig;
 import com.yahoo.document.config.DocumentmanagerConfig;
@@ -26,21 +26,25 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase {
     private static final String tempDir = "temp/";
     private static final String searchDefRoot = "src/test/derived/";
 
-    private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
+    private DerivedConfiguration derive(String dirName,
+                                        String searchDefinitionName,
+                                        TestProperties properties,
+                                        DeployLogger logger) throws IOException, ParseException {
         File toDir = new File(tempDir + dirName);
         toDir.mkdirs();
         deleteContent(toDir);
 
-        SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/");
-        return derive(dirName, searchDefinitionName, properties, builder);
+        SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/", logger);
+        return derive(dirName, searchDefinitionName, properties, builder, logger);
     }
 
     private DerivedConfiguration derive(String dirName,
                                         String searchDefinitionName,
                                         TestProperties properties,
-                                        SearchBuilder builder) throws IOException {
+                                        SearchBuilder builder,
+                                        DeployLogger logger) throws IOException {
         DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName),
-                                                               new BaseDeployLogger(),
+                                                               logger,
                                                                properties,
                                                                builder.getRankProfileRegistry(),
                                                                builder.getQueryProfileRegistry(),
@@ -80,15 +84,23 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase {
      * @throws IOException    if file access failed.
      */
     protected DerivedConfiguration assertCorrectDeriving(String dirName) throws IOException, ParseException {
-        return assertCorrectDeriving(dirName, null);
+        return assertCorrectDeriving(dirName, new TestableDeployLogger());
+    }
+    protected DerivedConfiguration assertCorrectDeriving(String dirName, DeployLogger logger) throws IOException, ParseException {
+        return assertCorrectDeriving(dirName, null, logger);
     }
 
-    protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException {
-        return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties());
+    protected DerivedConfiguration assertCorrectDeriving(String dirName,
+                                                         String searchDefinitionName,
+                                                         DeployLogger logger) throws IOException, ParseException {
+        return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger);
     }
 
-    protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
-        DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties);
+    protected DerivedConfiguration assertCorrectDeriving(String dirName,
+                                                         String searchDefinitionName,
+                                                         TestProperties properties,
+                                                         DeployLogger logger) throws IOException, ParseException {
+        DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties, logger);
         assertCorrectConfigFiles(dirName);
         return derived;
     }
@@ -97,9 +109,9 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase {
      * Asserts config is correctly derived given a builder.
      * This will fail if the builder contains multiple search definitions.
      */
-    protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException {
+    protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName, DeployLogger logger) throws IOException {
         builder.build();
-        DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder);
+        DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder, logger);
         assertCorrectConfigFiles(dirName);
         return derived;
     }
@@ -121,13 +133,14 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase {
         if (files == null) return;
         for (File file : files) {
             if ( ! file.getName().endsWith(".cfg")) continue;
-            assertEqualFiles(file.getPath(), tempDir + name + "/" + file.getName());
+            boolean orderMatters = file.getName().equals("ilscripts.cfg");
+            assertEqualFiles(file.getPath(), tempDir + name + "/" + file.getName(), orderMatters);
         }
     }
 
-    static void assertEqualFiles(String correctFileName, String checkFileName) throws IOException {
+    static void assertEqualFiles(String correctFileName, String checkFileName, boolean orderMatters) throws IOException {
         // Set updateOnAssert to true if you want update the files with correct answer.
-        assertConfigFiles(correctFileName, checkFileName, false);
+        assertConfigFiles(correctFileName, checkFileName, orderMatters, false);
     }
 
     void deleteContent(File dir) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index 71cca20f8fa..770bea55af2 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -1,13 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.searchdefinition.SearchBuilder;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
 import java.io.IOException;
 
+import static org.junit.Assert.assertEquals;
+
 /**
  * Tests exporting
  *
@@ -158,7 +159,14 @@ public class ExportingTestCase extends AbstractExportingTestCase {
 
     @Test
     public void testRankProfileInheritance() throws IOException, ParseException {
-        assertCorrectDeriving("rankprofileinheritance", "child");
+        assertCorrectDeriving("rankprofileinheritance", "child", new TestableDeployLogger());
+    }
+
+    @Test
+    public void testLanguage() throws IOException, ParseException {
+        TestableDeployLogger logger = new TestableDeployLogger();
+        assertCorrectDeriving("language", logger);
+        assertEquals(0, logger.warnings.size());
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
index a0dd89229dd..f61143833c9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java
@@ -13,26 +13,26 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase {
 
     @Test
     public void configs_for_imported_fields_are_derived() throws IOException, ParseException {
-        assertCorrectDeriving("importedfields", "child");
+        assertCorrectDeriving("importedfields", "child", new TestableDeployLogger());
     }
 
     @Test
     public void configs_for_imported_struct_fields_are_derived() throws IOException, ParseException {
-        assertCorrectDeriving("imported_struct_fields", "child");
+        assertCorrectDeriving("imported_struct_fields", "child", new TestableDeployLogger());
     }
 
     @Test
     public void configs_for_imported_position_field_are_derived() throws IOException, ParseException {
-        assertCorrectDeriving("imported_position_field", "child");
+        assertCorrectDeriving("imported_position_field", "child", new TestableDeployLogger());
     }
 
     @Test
     public void configs_for_imported_position_field_summary_are_derived() throws IOException, ParseException {
-        assertCorrectDeriving("imported_position_field_summary", "child");
+        assertCorrectDeriving("imported_position_field_summary", "child", new TestableDeployLogger());
     }
 
     @Test
     public void derives_configs_for_imported_fields_when_reference_fields_are_inherited() throws IOException, ParseException {
-        assertCorrectDeriving("imported_fields_inherited_reference", "child_c");
+        assertCorrectDeriving("imported_fields_inherited_reference", "child_c", new TestableDeployLogger());
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java
index 187e766c315..5e45102a626 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java
@@ -18,7 +18,7 @@ public class MailTestCase extends AbstractExportingTestCase {
         String dir = "src/test/derived/mail/";
         SearchBuilder sb = new SearchBuilder();
         sb.importFile(dir + "mail.sd");
-        assertCorrectDeriving(sb, dir);
+        assertCorrectDeriving(sb, dir, new TestableDeployLogger());
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java
index 073a15b0525..1f1c905e0bf 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java
@@ -13,6 +13,6 @@ public class ReferenceFieldsTestCase extends AbstractExportingTestCase {
 
     @Test
     public void configs_related_to_reference_fields_are_derived() throws IOException, ParseException {
-        assertCorrectDeriving("reference_fields", "ad");
+        assertCorrectDeriving("reference_fields", "ad", new TestableDeployLogger());
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java
index 45a6240d3ba..83e11c365f8 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java
@@ -45,10 +45,8 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase {
     private void checkDir(String toDirName, String expectedResultsDirName) throws IOException {
         File[] files = new File(expectedResultsDirName).listFiles();
         for (File file : files) {
-            if (!file.getName().endsWith(".cfg")) {
-                continue;
-            }
-            assertEqualFiles(file.getPath(), toDirName + "/" + file.getName());
+            if ( ! file.getName().endsWith(".cfg")) continue;
+            assertEqualFiles(file.getPath(), toDirName + "/" + file.getName(), false);
         }
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java
new file mode 100644
index 00000000000..a23c8024872
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java
@@ -0,0 +1,30 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.config.application.api.DeployLogger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * @author bratseth
+ */
+public class TestableDeployLogger implements DeployLogger {
+
+    private static final Logger log = Logger.getLogger("DeployLogger");
+
+    public List<String> warnings = new ArrayList<>();
+    public List<String> info = new ArrayList<>();
+
+    @Override
+    public final void log(Level level, String message) {
+        log.log(level, message);
+        if (level.equals(Level.WARNING))
+            warnings.add(message);
+        if (level.equals(Level.INFO))
+            info.add(message);
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
index 1567a4c3b5e..8861432d97b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
@@ -9,6 +9,7 @@ import com.yahoo.searchdefinition.*;
 import com.yahoo.searchdefinition.derived.DerivedConfiguration;
 import com.yahoo.searchdefinition.derived.AttributeFields;
 import com.yahoo.searchdefinition.derived.RawRankProfile;
+import com.yahoo.searchdefinition.derived.TestableDeployLogger;
 import com.yahoo.searchdefinition.parser.ParseException;
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import org.junit.Test;
@@ -25,7 +26,8 @@ public class RankingExpressionsTestCase extends SchemaTestCase {
     public void testFunctions() throws IOException, ParseException {
         RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
         Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction",
-                                                          rankProfileRegistry).getSearch();
+                                                          rankProfileRegistry,
+                                                          new TestableDeployLogger()).getSearch();
         RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros");
         Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions();
         assertEquals(2, functions.get("titlematch$").function().arguments().size());
@@ -63,7 +65,9 @@ public class RankingExpressionsTestCase extends SchemaTestCase {
     @Test(expected = IllegalArgumentException.class)
     public void testThatIncludingFileInSubdirFails() throws IOException, ParseException {
         RankProfileRegistry registry = new RankProfileRegistry();
-        Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", registry).getSearch();
+        Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile",
+                                                          registry,
+                                                          new TestableDeployLogger()).getSearch();
         new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // rank profile parsing happens during deriving
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java
index 31d0084570a..9e7370a933c 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java
@@ -159,7 +159,7 @@ public class RoutingTestCase {
             fail("Expected file '" + fileName + "' not found.");
             return;
         }
-        assertSerializedConfigEquals(IOUtils.readFile(files.get(fileName)), configString);
+        assertSerializedConfigEquals(IOUtils.readFile(files.get(fileName)), configString, false);
     }
 
     /**
diff --git a/config-model/src/test/java/helpers/CompareConfigTestHelper.java b/config-model/src/test/java/helpers/CompareConfigTestHelper.java
index 89b4eeb3c8d..586f82ec379 100644
--- a/config-model/src/test/java/helpers/CompareConfigTestHelper.java
+++ b/config-model/src/test/java/helpers/CompareConfigTestHelper.java
@@ -19,15 +19,20 @@ import static org.junit.Assert.assertEquals;
 public class CompareConfigTestHelper {
 
     public static void assertSerializedConfigFileEquals(String filename, String actual) throws IOException {
-        assertSerializedConfigEquals(IOUtils.readFile(new File(filename)), actual);
+        assertSerializedConfigEquals(IOUtils.readFile(new File(filename)), actual, false);
     }
 
     // Written this way to compare order independently but output error with order preserved
     // Note that this means that if a test fails you'll also see spurious differences in the comparison
     // from lines which are present in both but at different locations.
-    public static void assertSerializedConfigEquals(String expected, String actual) {
-        if ( ! sortLines(expected.trim()).equals(sortLines(actual.trim())))
-            assertEquals(expected, actual);
+    public static void assertSerializedConfigEquals(String expected, String actual, boolean orderMatters) {
+        if (orderMatters) {
+            assertEquals(expected.trim(), actual.trim());
+        }
+        else {
+            if (!sortLines(expected.trim()).equals(sortLines(actual.trim())))
+                assertEquals(expected, actual);
+        }
     }
 
     private static String sortLines(String fileData) {