Process imported fields in config derivers

11 files changed, 93 insertions, 39 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 15a00e155d7..1da7206ef3b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -1,17 +1,23 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.document.*;
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
 import com.yahoo.searchdefinition.document.Ranking;
-import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.document.Sorting;
 import com.yahoo.vespa.config.search.AttributesConfig;
-import com.yahoo.config.subscription.ConfigInstanceUtil;
 import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression;
 
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * The set of all attribute fields defined by a search definition
@@ -34,13 +40,17 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
     /**
      * Derives everything from a field
      */
-    protected void derive(SDField field, Search search) {
+    @Override
+    protected void derive(ImmutableSDField field, Search search) {
         if (field.usesStructOrMap() &&
             !field.getDataType().equals(PositionDataType.INSTANCE) &&
             !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
         {
             return; // Ignore struct fields for indexed search (only implemented for streaming search)
         }
+        if (field.isImportedField()) {
+            return; // Ignore imported fields
+        }
         deriveAttributes(field);
     }
 
@@ -57,8 +67,9 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
 
     /**
      * Derives one attribute. TODO: Support non-default named attributes
+     * @param field
      */
-    private void deriveAttributes(SDField field) {
+    private void deriveAttributes(ImmutableSDField field) {
         for (Attribute fieldAttribute : field.getAttributes().values()) {
             Attribute attribute = getAttribute(fieldAttribute.getName());
             if (attribute == null) {
@@ -97,6 +108,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
         return "attributes " + getName();
     }
 
+    @Override
     protected String getDerivedName() {
         return "attributes";
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
index 520c743d341..469af71f7d9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition.derived;
 
 import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.config.ConfigInstance;
@@ -44,6 +45,8 @@ public abstract class Derived implements Exportable {
         for (SDField field : search.allExtraFields() ) {
             derive(field,search);
         }
+        search.allImportedFields()
+                .forEach(importedField -> derive(importedField, search));
     }
 
 
@@ -64,7 +67,7 @@ public abstract class Derived implements Exportable {
      * Derives the content of this configuration. This
      * default does nothing.
      */
-    protected void derive(SDField field,Search search) {}
+    protected void derive(ImmutableSDField field, Search search) {}
 
     /**
      * Derives the content of this configuration. This
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
index 9c8f3d7d8dc..ad28fa3add6 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
@@ -48,6 +48,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         derive(search);
     }
 
+    @Override
     protected void derive(Search search) {
         super.derive(search); // Derive per field
         this.search = search;
@@ -74,6 +75,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         }
     }
 
+    @Override
     protected void derive(Index index, Search search) {
         if (index.getMatchGroup().size() > 0) {
             addIndexCommand(index.getName(), CMD_MATCH_GROUP + toSpaceSeparated(index.getMatchGroup()));
@@ -91,7 +93,8 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         return b.toString();
     }
 
-    protected void derive(SDField field, Search search) {
+    @Override
+    protected void derive(ImmutableSDField field, Search search) {
         if (field.getDataType().equals(DataType.PREDICATE)) {
             Index index = field.getIndex(field.getName());
             if (index != null) {
@@ -111,7 +114,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
             addIndexAlias(alias, name);
         }
         if (field.usesStructOrMap()) {
-            for (SDField structField : field.getStructFields()) {
+            for (ImmutableSDField structField : field.getStructFields()) {
                 derive(structField, search); // Recursion
             }
         }
@@ -164,25 +167,25 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
 
     }
 
-    static String stemCmd(SDField field, Search search) {
+    static String stemCmd(ImmutableSDField field, Search search) {
         return CMD_STEM + ":" + field.getStemming(search).toStemMode();
     }
 
-    private boolean stemSomehow(SDField field, Search search) {
+    private boolean stemSomehow(ImmutableSDField field, Search search) {
         if (field.getStemming(search).equals(Stemming.NONE)) return false;
         return isTypeOrNested(field, DataType.STRING);
     }
 
-    private boolean normalizeAccents(SDField field) {
+    private boolean normalizeAccents(ImmutableSDField field) {
         return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING);
     }
 
-    private boolean isTypeOrNested(SDField field, DataType type) {
+    private boolean isTypeOrNested(ImmutableSDField field, DataType type) {
         return field.getDataType().equals(type) || field.getDataType().equals(DataType.getArray(type)) ||
                field.getDataType().equals(DataType.getWeightedSet(type));
     }
 
-    private boolean isUriField(Field field) {
+    private boolean isUriField(ImmutableSDField field) {
         DataType fieldType = field.getDataType();
         if (DataType.URI.equals(fieldType)) {
             return true;
@@ -195,7 +198,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         return false;
     }
 
-    private void addUriIndexCommands(SDField field) {
+    private void addUriIndexCommands(ImmutableSDField field) {
         String fieldName = field.getName();
         addIndexCommand(fieldName, CMD_FULLURL);
         addIndexCommand(fieldName, CMD_LOWERCASE);
@@ -225,14 +228,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
     /**
      * Sets a command for all indices of a field
      */
-    private void addIndexCommand(SDField field, String command) {
+    private void addIndexCommand(ImmutableSDField field, String command) {
         addIndexCommand(field, command, null);
     }
 
     /**
      * Sets a command for all indices of a field
      */
-    private void addIndexCommand(SDField field, String command, IndexOverrider overrider) {
+    private void addIndexCommand(ImmutableSDField field, String command, IndexOverrider overrider) {
         if (overrider == null || !overrider.override(field.getName(), command, field)) {
             addIndexCommand(field.getName(), command);
         }
@@ -417,7 +420,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         return false;
     }
 
-    private Stemming getEffectiveStemming(SDField field) {
+    private Stemming getEffectiveStemming(ImmutableSDField field) {
         Stemming active = field.getStemming(search);
         if (field.getIndex(field.getName()) != null) {
             if (field.getIndex(field.getName()).getStemming()!=null) {
@@ -447,7 +450,8 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
         if (m.getType().equals(Matching.Type.WORD)) return true;
         return false;
     }
-    
+
+    @Override
     protected String getDerivedName() {
         return "index-info";
     }
@@ -518,7 +522,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
          * Override the setting of this index for this field, returns true if overriden, false if this index should be
          * set according to the field
          */
-        public abstract boolean override(String indexName, String command, SDField field);
+        public abstract boolean override(String indexName, String command, ImmutableSDField field);
 
     }
 
@@ -531,7 +535,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
             this.search = search;
         }
 
-        public boolean override(String indexName, String command, SDField field) {
+        public boolean override(String indexName, String command, ImmutableSDField field) {
             if (search == null) {
                 return false;
             }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
index 46cd0b59c1b..ceb21b149c8 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
@@ -1,14 +1,23 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.document.*;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
 import com.yahoo.searchdefinition.document.FieldSet;
-import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
 import com.yahoo.vespa.config.search.IndexschemaConfig;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Deriver of indexschema config containing information of all index fields with name and data type.
@@ -35,13 +44,13 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
         super.derive(search);
     }
 
-    private void deriveIndexFields(SDField field, Search search) {
+    private void deriveIndexFields(ImmutableSDField field, Search search) {
         if (!field.doesIndexing() &&
             !field.isIndexStructureField())
         {
             return;
         }
-        List<Field> lst = flattenField(field);
+        List<Field> lst = flattenField(field.asField());
         if (lst.isEmpty()) {
             return;
         }
@@ -83,7 +92,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
     }
 
     @Override
-    protected void derive(SDField field, Search search) {
+    protected void derive(ImmutableSDField field, Search search) {
         if (field.usesStructOrMap()) {
             return; // unsupported
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
index e8890288a7f..5471830cea5 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
@@ -1,15 +1,30 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.document.*;
-import com.yahoo.searchdefinition.*;
-import com.yahoo.searchdefinition.document.SDField;
-import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
-import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder;
-
-import java.util.*;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.GuardExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
 
 /**
  * An indexing language script derived from a search definition. An indexing script contains a set of indexing
@@ -26,7 +41,11 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
         derive(search);
     }
 
-    protected void derive(SDField field, Search search) {
+    @Override
+    protected void derive(ImmutableSDField field, Search search) {
+        if (field.isImportedField()) {
+            return;
+        }
         if (!field.isExtraField()) {
             docFields.add(field.getName());
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java
index 9ef0ddbc723..3fc41b7caba 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java
@@ -40,6 +40,7 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer {
     }
 
     // Inherit doc from Derived.
+    @Override
     protected String getDerivedName() {
         return "juniperrc";
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
index 9523c224394..4fd33a51176 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
@@ -25,6 +25,7 @@ public class Summaries extends Derived implements SummaryConfig.Producer {
         }
     }
 
+    @Override
     protected String getDerivedName() { return "summary"; }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
index 35ec45acba8..60e039c1b98 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
@@ -133,7 +133,8 @@ public class SummaryClass extends Derived {
         }
         return classBuilder;
     }
-    
+
+    @Override
     protected String getDerivedName() { return "summary"; }
 
     public String toString() {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java
index be4c34028fc..824c8504fa7 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java
@@ -1,12 +1,13 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
+import com.yahoo.vespa.config.search.SummarymapConfig;
 import com.yahoo.vespa.documentmodel.DocumentSummary;
 import com.yahoo.vespa.documentmodel.SummaryField;
 import com.yahoo.vespa.documentmodel.SummaryTransform;
-import com.yahoo.vespa.config.search.SummarymapConfig;
+
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.Map;
@@ -34,7 +35,8 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer {
         super.derive(search);
     }
 
-    protected void derive(SDField field, Search search) {
+    @Override
+    protected void derive(ImmutableSDField field, Search search) {
     }
 
     private void derive(DocumentSummary documentSummary) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java
index 385fc34776b..733528a1d26 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java
@@ -30,6 +30,7 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer {
         derive(search);
     }
 
+    @Override
     protected void derive(SDDocumentType document,Search search) {
         super.derive(document, search);
         StreamingDocumentType docType=getDocumentType(document.getName());
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java
index 5b3b64ee686..31fdd3c551d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java
@@ -88,6 +88,7 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer {
         return ret;
     }
 
+    @Override
     public String getDerivedName() {
         return "vsmsummary";
     }