Publish

145 files changed, 13999 insertions, 0 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java
new file mode 100644
index 00000000000..98eb0a4b77c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.NameNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Transforms named references to constant tensors with the rank feature 'constant'.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+class ConstantTensorTransformer extends ExpressionTransformer {
+
+    private final Map<String, Value> constants;
+    private final Map<String, String> rankPropertiesOutput;
+
+    public ConstantTensorTransformer(Map<String, Value> constants,
+                                     Map<String, String> rankPropertiesOutput) {
+        this.constants = constants;
+        this.rankPropertiesOutput = rankPropertiesOutput;
+    }
+
+    @Override
+    public ExpressionNode transform(ExpressionNode node) {
+        if (node instanceof ReferenceNode) {
+            return transformFeature((ReferenceNode) node);
+        } else if (node instanceof CompositeNode) {
+            return transformChildren((CompositeNode) node);
+        } else {
+            return node;
+        }
+    }
+
+    private ExpressionNode transformFeature(ReferenceNode node) {
+        if (!node.getArguments().isEmpty()) {
+            return transformArguments(node);
+        } else {
+            return transformConstantReference(node);
+        }
+    }
+
+    private ExpressionNode transformArguments(ReferenceNode node) {
+        List<ExpressionNode> arguments = node.getArguments().expressions();
+        List<ExpressionNode> transformedArguments = new ArrayList<>(arguments.size());
+        for (ExpressionNode argument : arguments) {
+            transformedArguments.add(transform(argument));
+        }
+        return node.setArguments(transformedArguments);
+    }
+
+    private ExpressionNode transformConstantReference(ReferenceNode node) {
+        Value value = constants.get(node.getName());
+        if (value == null || !(value instanceof TensorValue)) {
+            return node;
+        }
+        TensorValue tensorValue = (TensorValue)value;
+        String featureName = "constant(" + node.getName() + ")";
+        String tensorType = (tensorValue.getType().isPresent() ? tensorValue.getType().get().toString() : "tensor");
+        rankPropertiesOutput.put(featureName + ".value", tensorValue.toString());
+        rankPropertiesOutput.put(featureName + ".type", tensorType);
+        return new ReferenceNode("constant", Arrays.asList(new NameNode(node.getName())), null);
+    }
+
+}
\ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java
new file mode 100644
index 00000000000..c0dcf27c88d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * The rank profile containing default settings. This is derived from the fields
+ * whenever this is accessed.
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class DefaultRankProfile extends RankProfile {
+
+    /**
+     * Creates a new rank profile
+     *
+     * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles.
+     */
+    public DefaultRankProfile(Search search, RankProfileRegistry rankProfileRegistry) {
+        super("default", search, rankProfileRegistry);
+    }
+
+    /**
+     * Does nothing, the default rank profile can not inherit anything
+     */
+    // TODO: Why not? If that's the case, then fail attempts at it
+    public void setInherited(String inheritedName) {
+    }
+
+    /**
+     * Returns null, the default rank profile can not inherit anything
+     */
+    public String getInheritedName() {
+        return null;
+    }
+
+    /**
+     * Returns the rank boost value of the given field
+     */
+    public RankSetting getRankSetting(String fieldOrIndex,RankSetting.Type type) {
+        RankSetting setting=super.getRankSetting(fieldOrIndex,type);
+        if (setting!=null) return setting;
+
+        SDField field=getSearch().getField(fieldOrIndex);
+        if (field!=null) {
+            setting=toRankSetting(field,type);
+            if (setting!=null)
+                return setting;
+        }
+
+        Index index=getSearch().getIndex(fieldOrIndex);
+        if (index!=null) {
+            setting=toRankSetting(index,type);
+            if (setting!=null)
+                return setting;
+        }
+
+        return null;
+    }
+
+    private RankSetting toRankSetting(SDField field,RankSetting.Type type) {
+        if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight()>0 && field.getWeight()!=100)
+            return new RankSetting(field.getName(),type,field.getWeight());
+        if (type.equals(RankSetting.Type.RANKTYPE))
+            return new RankSetting(field.getName(),type,field.getRankType());
+        if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost()>0)
+            return new RankSetting(field.getName(),type,field.getLiteralBoost());
+
+        // Index level setting really
+        if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) {
+            return new RankSetting(field.getName(), type, true);
+        }
+
+        return null;
+    }
+
+    private RankSetting toRankSetting(Index index, RankSetting.Type type) {
+        /* TODO: Add support for indexes by adding a ranking object to the index
+        if (type.equals(RankSetting.Type.PREFERBITVECTOR) && index.isPreferBitVector()) {
+            return new RankSetting(index.getName(), type, new Boolean(true));
+        }
+        */
+        return null;
+    }
+
+    /**
+     * Returns the names of the fields which have a rank boost setting
+     * explicitly in this profile or in fields
+     */
+    public Set<RankSetting> rankSettings() {
+        Set<RankSetting> settings=new LinkedHashSet<>(20);
+        settings.addAll(this.rankSettings);
+        for (SDField field : getSearch().allFieldsList() ) {
+            addSetting(field,RankSetting.Type.WEIGHT,settings);
+            addSetting(field,RankSetting.Type.RANKTYPE,settings);
+            addSetting(field,RankSetting.Type.LITERALBOOST,settings);
+            addSetting(field,RankSetting.Type.PREFERBITVECTOR,settings);
+        }
+
+        // Foer settings that really pertains to indexes do the explicit indexes too
+        for (Index index : getSearch().getExplicitIndices()) {
+            addSetting(index,RankSetting.Type.PREFERBITVECTOR,settings);
+        }
+        return settings;
+    }
+
+    private void addSetting(SDField field,RankSetting.Type type,Set<RankSetting> settings) {
+        if (type.isIndexLevel()) {
+            addIndexSettings(field,type,settings);
+        }
+        else {
+            RankSetting setting=toRankSetting(field,type);
+            if (setting==null) return;
+            settings.add(setting);
+        }
+    }
+
+    private void addIndexSettings(SDField field,RankSetting.Type type,Set<RankSetting> settings) {
+        for (Iterator i = field.getFieldNameAsIterator(); i.hasNext(); ) {
+            String indexName=(String)i.next();
+            Index explicitIndex=field.getIndex(indexName);
+
+            // TODO: Make a ranking object in the index override the field level ranking object
+            if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) {
+                settings.add(new RankSetting(indexName, type, true));
+            }
+        }
+    }
+
+    private void addSetting(Index index,RankSetting.Type type,Set<RankSetting> settings) {
+        RankSetting setting=toRankSetting(index,type);
+        if (setting==null) return;
+        settings.add(setting);
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
new file mode 100644
index 00000000000..4fd7048159e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -0,0 +1,415 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.documentmodel.DataTypeCollection;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.VespaDocumentType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.documentmodel.FieldView;
+import com.yahoo.vespa.documentmodel.SearchDef;
+import com.yahoo.vespa.documentmodel.SearchField;
+
+import java.util.*;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class DocumentModelBuilder {
+
+    public static class RetryLaterException extends IllegalArgumentException {
+        public RetryLaterException(String message) {
+            super(message);
+        }
+    }
+    private DocumentModel model;
+    private final Map<NewDocumentType, List<SDDocumentType>> scratchInheritsMap = new HashMap<>();
+    public DocumentModelBuilder(DocumentModel model) {
+        this.model = model;
+        model.getDocumentManager().add(VespaDocumentType.INSTANCE);
+    }
+    public boolean valid() {
+        return scratchInheritsMap.isEmpty();
+    }
+    public void addToModel(Collection<Search> searchList) {
+        List<SDDocumentType> docList = new LinkedList<>();
+        for (Search search : searchList) {
+            docList.add(search.getDocument());
+        }
+        docList = sortDocumentTypes(docList);
+        addDocumentTypes(docList);
+        for (Collection<Search> toAdd = tryAdd(searchList);
+             !toAdd.isEmpty() && (toAdd.size() < searchList.size()); toAdd = tryAdd(searchList)) {
+            searchList = toAdd;
+        }
+    }
+
+    private List<SDDocumentType> sortDocumentTypes(List<SDDocumentType> docList) {
+        Set<String> doneNames = new HashSet<>();
+        doneNames.add(SDDocumentType.VESPA_DOCUMENT.getName());
+        List<SDDocumentType> doneList = new LinkedList<>();
+        List<SDDocumentType> prevList = null;
+        List<SDDocumentType> nextList = docList;
+        while (prevList == null || nextList.size() < prevList.size()) {
+            prevList = nextList;
+            nextList = new LinkedList<>();
+            for (SDDocumentType doc : prevList) {
+                boolean isDone = true;
+                for (SDDocumentType inherited : doc.getInheritedTypes()) {
+                    if (!doneNames.contains(inherited.getName())) {
+                        isDone = false;
+                        break;
+                    }
+                }
+                if (isDone) {
+                    doneNames.add(doc.getName());
+                    doneList.add(doc);
+                } else {
+                    nextList.add(doc);
+                }
+            }
+        }
+        if (!nextList.isEmpty()) {
+            throw new IllegalArgumentException("Could not resolve inheritance of document types " +
+                                               toString(prevList) + ".");
+        }
+        return doneList;
+    }
+
+    private static String toString(List<SDDocumentType> lst) {
+        StringBuilder out = new StringBuilder();
+        for (int i = 0, len = lst.size(); i < len; ++i) {
+            out.append("'").append(lst.get(i).getName()).append("'");
+            if (i < len - 2) {
+                out.append(", ");
+            } else if (i < len - 1) {
+                out.append(" and ");
+            }
+        }
+        return out.toString();
+    }
+
+    private Collection<Search> tryAdd(Collection<Search> searchList) {
+        Collection<Search> left = new ArrayList<>();
+        for (Search search : searchList) {
+            try {
+                addToModel(search);
+            } catch (RetryLaterException e) {
+                left.add(search);
+            }
+        }
+        return left;
+    }
+    public void addToModel(Search search) {
+        // Then we add the search specific stuff
+        SearchDef searchDef = new SearchDef(search.getName());
+        addSearchFields(search.extraFieldList(), searchDef);
+        for (Field f : search.getDocument().fieldSet()) {
+            addSearchField((SDField) f, searchDef);
+        }
+        for(SDField field : search.allFieldsList()) {
+            for(Attribute attribute : field.getAttributes().values()) {
+                if (!searchDef.getFields().containsKey(attribute.getName())) {
+                    searchDef.add(new SearchField(new Field(attribute.getName(), field), !field.getIndices().isEmpty(), true));
+                }
+            }
+        }
+
+        for (Field f : search.getDocument().fieldSet()) {
+            addAlias((SDField) f, searchDef);
+        }
+        model.getSearchManager().add(searchDef);
+    }
+    private static void addSearchFields(Collection<SDField> fields, SearchDef searchDef) {
+        for (SDField field : fields) {
+            addSearchField(field, searchDef);
+        }
+    }
+    private static void addSearchField(SDField field, SearchDef searchDef) {
+
+        SearchField searchField = 
+                new SearchField(field,
+                                field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), 
+                                field.getAttributes().containsKey(field.getName()));
+        searchDef.add(searchField);
+
+        // Add field to views
+        addToView(field.getIndices().keySet(), searchField, searchDef);
+    }
+
+    private static void addAlias(SDField field, SearchDef searchDef) {
+        for (Map.Entry<String, String> entry : field.getAliasToName().entrySet()) {
+            searchDef.addAlias(entry.getKey(), entry.getValue());
+        }
+    }
+
+    private static void addToView(Collection<String> views, Field field, SearchDef searchDef) {
+        for (String viewName : views) {
+            addToView(viewName, field, searchDef);
+        }
+    }
+
+    private static void addToView(String viewName, Field field, SearchDef searchDef) {
+        if (searchDef.getViews().containsKey(viewName)) {
+            searchDef.getViews().get(viewName).add(field);
+        } else {
+            if (!searchDef.getFields().containsKey(viewName)) {
+                FieldView view = new FieldView(viewName);
+                view.add(field);
+                searchDef.add(view);
+            }
+        }
+    }
+    private void addDocumentTypes(List<SDDocumentType> docList) {
+        LinkedList<NewDocumentType> lst = new LinkedList<>();
+        for (SDDocumentType doc : docList) {
+            lst.add(convert(doc));
+            model.getDocumentManager().add(lst.getLast());
+        }
+        for(NewDocumentType doc : lst) {
+            resolveTemporaries(doc.getAllTypes(), lst);
+        }
+    }
+    private static void resolveTemporaries(DataTypeCollection dtc, Collection<NewDocumentType> docs) {
+        for (DataType type : dtc.getTypes()) {
+            resolveTemporariesRecurse(type, dtc, docs);
+        }
+    }
+    private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo,
+                                                      Collection<NewDocumentType> docs) {
+        if (type instanceof TemporaryStructuredDataType) {
+            NewDocumentType docType = getDocumentType(docs, type.getId());
+            if (docType != null) {
+                type = docType;
+                return type;
+            }
+            DataType real = repo.getDataType(type.getId());
+            if (real == null) {
+                throw new NullPointerException("Can not find type '" + type.toString() + "', impossible.");
+            }
+            type = real;
+        } else if (type instanceof StructDataType) {
+            StructDataType dt = (StructDataType) type;
+            for (com.yahoo.document.Field field : dt.getFields()) {
+                if (field.getDataType() != type) {
+                    field.setDataType(resolveTemporariesRecurse(field.getDataType(), repo, docs));
+                }
+            }
+        } else if (type instanceof MapDataType) {
+            MapDataType t = (MapDataType) type;
+            t.setKeyType(resolveTemporariesRecurse(t.getKeyType(), repo, docs));
+            t.setValueType(resolveTemporariesRecurse(t.getValueType(), repo, docs));
+        } else if (type instanceof CollectionDataType) {
+            CollectionDataType t = (CollectionDataType) type;
+            t.setNestedType(resolveTemporariesRecurse(t.getNestedType(), repo, docs));
+        }
+        return type;
+    }
+
+    private static NewDocumentType getDocumentType(Collection<NewDocumentType> docs, int id) {
+        for (NewDocumentType doc : docs) {
+            if (doc.getId() == id) {
+                return doc;
+            }
+        }
+        return null;
+    }
+
+    private static void specialHandleAnnotationReference(NewDocumentType docType, Field field) {
+        DataType fieldType = specialHandleAnnotationReferenceRecurse(docType, field.getName(), field.getDataType());
+        if (fieldType == null) {
+            return;
+        }
+        field.setDataType(fieldType);
+    }
+
+    private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName,
+                                                                    DataType dataType)
+    {
+        if (dataType instanceof TemporaryAnnotationReferenceDataType) {
+            TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType;
+            if (refType.getId() != 0) {
+                return null;
+            }
+            AnnotationType target = docType.getAnnotationType(refType.getTarget());
+            if (target == null) {
+                throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName +
+                                              "' does not exist.");
+            }
+            dataType = new AnnotationReferenceDataType(target);
+            addType(docType, dataType);
+            return dataType;
+        } else if (dataType instanceof MapDataType) {
+            MapDataType mapType = (MapDataType)dataType;
+            DataType valueType = specialHandleAnnotationReferenceRecurse(docType, fieldName, mapType.getValueType());
+            if (valueType == null) {
+                return null;
+            }
+            mapType = mapType.clone();
+            mapType.setValueType(valueType);
+            addType(docType, mapType);
+            return mapType;
+        } else if (dataType instanceof CollectionDataType) {
+            CollectionDataType lstType = (CollectionDataType)dataType;
+            DataType nestedType = specialHandleAnnotationReferenceRecurse(docType, fieldName, lstType.getNestedType());
+            if (nestedType == null) {
+                return null;
+            }
+            lstType = lstType.clone();
+            lstType.setNestedType(nestedType);
+            addType(docType, lstType);
+            return lstType;
+        }
+        return null;
+    }
+
+    private static StructDataType handleStruct(NewDocumentType dt, SDDocumentType type) {
+        StructDataType s = new StructDataType(type.getName());
+        for (Field f : type.getDocumentType().getHeaderType().getFieldsThisTypeOnly()) {
+            specialHandleAnnotationReference(dt, f);
+            s.addField(f);
+        }
+        for (StructDataType inherited : type.getDocumentType().getHeaderType().getInheritedTypes()) {
+            s.inherit(inherited);
+        }
+        extractNestedTypes(dt, s);
+        addType(dt, s);
+        return s;
+    }
+
+    private static StructDataType handleStruct(NewDocumentType dt, StructDataType s) {
+        for (Field f : s.getFieldsThisTypeOnly()) {
+            specialHandleAnnotationReference(dt, f);
+        }
+        extractNestedTypes(dt, s);
+        addType(dt, s);
+        return s;
+    }
+    private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) {
+        if (sa.getInherits() != null) {
+            AnnotationType tmp = sdoc.findAnnotation(sa.getInherits());
+            SDAnnotationType inherited = (SDAnnotationType) tmp;
+            return ((inherited.getSdDocType() != null) || anyParentsHavePayLoad(inherited, sdoc));
+        }
+        return false;
+    }
+    private NewDocumentType convert(SDDocumentType sdoc) {
+        Map<AnnotationType, String> annotationInheritance = new HashMap<>();
+        Map<StructDataType, String> structInheritance = new HashMap<>();
+        NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()),
+                                                 sdoc.getDocumentType().getHeaderType(),
+                                                 sdoc.getDocumentType().getBodyType(),
+                                                 sdoc.getFieldSets());
+        for (SDDocumentType n : sdoc.getInheritedTypes()) {
+            NewDocumentType.Name name = new NewDocumentType.Name(n.getName());
+                NewDocumentType inherited =  model.getDocumentManager().getDocumentType(name);
+                if (inherited != null) {
+                    dt.inherit(inherited);
+                }
+        }
+        for (SDDocumentType type : sdoc.getTypes()) {
+            if (type.isStruct()) {
+                handleStruct(dt, type);
+            } else {
+                throw new IllegalArgumentException("Data type '" + sdoc.getName() + "' is not a struct => tostring='" + sdoc.toString() + "'.");
+            }
+        }
+        for (AnnotationType annotation : sdoc.getAnnotations()) {
+            dt.add(annotation);
+        }
+        for (AnnotationType annotation : sdoc.getAnnotations()) {
+            SDAnnotationType sa = (SDAnnotationType) annotation;
+            if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) {
+                annotationInheritance.put(annotation, sa.getInherits());
+            }
+            if (annotation.getDataType() == null) {
+                if (sa.getSdDocType() != null) {
+                    StructDataType s = handleStruct(dt, sa.getSdDocType());
+                    annotation.setDataType(s);
+                    if ((sa.getInherits() != null)) {
+                        structInheritance.put(s, "annotation."+sa.getInherits());
+                    }
+                } else if (sa.getInherits() != null) {
+                    StructDataType s = new StructDataType("annotation."+annotation.getName());
+                    if (anyParentsHavePayLoad(sa, sdoc)) {
+                        annotation.setDataType(s);
+                        addType(dt, s);
+                    }
+                    structInheritance.put(s, "annotation."+sa.getInherits());
+                }
+            }
+        }
+        for (Map.Entry<AnnotationType, String> e : annotationInheritance.entrySet()) {
+            e.getKey().inherit(dt.getAnnotationType(e.getValue()));
+        }
+        for (Map.Entry<StructDataType, String> e : structInheritance.entrySet()) {
+            StructDataType s = (StructDataType)dt.getDataType(e.getValue());
+            if (s != null) {
+                e.getKey().inherit(s);
+            }
+        }
+        handleStruct(dt, sdoc.getDocumentType().getHeaderType());
+        handleStruct(dt, sdoc.getDocumentType().getBodyType());
+
+        extractDataTypesFromFields(dt, sdoc.fieldSet());
+        return dt;
+    }
+    private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) {
+        for (Field f : fields) {
+            DataType type = f.getDataType();
+            if (testAddType(dt, type)) {
+                extractNestedTypes(dt, type);
+                addType(dt, type);
+            }
+        }
+    }
+    private static void extractNestedTypes(NewDocumentType dt, DataType type) {
+        if (type instanceof StructDataType) {
+            StructDataType tmp = (StructDataType) type;
+            extractDataTypesFromFields(dt, tmp.getFieldsThisTypeOnly());
+        } else if (type instanceof DocumentType) {
+            throw new IllegalArgumentException("Can not handle nested document definitions. In document type '" + dt.getName().toString() +
+                                               "', we can not define document type '" + type.toString());
+        } else if (type instanceof CollectionDataType) {
+            CollectionDataType tmp = (CollectionDataType) type;
+            extractNestedTypes(dt, tmp.getNestedType());
+            addType(dt, tmp.getNestedType());
+        } else if (type instanceof MapDataType) {
+            MapDataType tmp = (MapDataType) type;
+            extractNestedTypes(dt, tmp.getKeyType());
+            extractNestedTypes(dt, tmp.getValueType());
+            addType(dt, tmp.getKeyType());
+            addType(dt, tmp.getValueType());
+        } else if (type instanceof TemporaryAnnotationReferenceDataType) {
+            throw new IllegalArgumentException(type.toString());
+        }
+    }
+    private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); }
+    private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); }
+    private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) {
+        DataType oldType = dt.getDataTypeRecursive(type.getId());
+        if (oldType == null) {
+            if ( ! dryRun) {
+                dt.add(type);
+            }
+            return true;
+        } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) {
+            StructDataType s = (StructDataType) type;
+            StructDataType os = (StructDataType) oldType;
+            if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) {
+                if ( ! dryRun) {
+                    dt.replace(type);
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java
new file mode 100644
index 00000000000..9cb3dea36a8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldOperationApplier {
+    public void process(SDDocumentType sdoc) {
+        if (!sdoc.isStruct()) {
+            apply(sdoc);
+        }
+    }
+
+    protected void apply(SDDocumentType type) {
+        for (Field field : type.fieldSet()) {
+            apply(field);
+        }
+    }
+
+    protected void apply(Field field) {
+        if (field instanceof SDField) {
+            SDField sdField = (SDField) field;
+            sdField.applyOperations();
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java
new file mode 100644
index 00000000000..301d78bbaec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldOperationApplierForSearch extends FieldOperationApplier {
+    @Override
+    public void process(SDDocumentType sdoc) {
+        //Do nothing
+    }
+
+    public void process(Search search) {
+        for (Field field : search.extraFieldList()) {
+            apply(field);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
new file mode 100644
index 00000000000..5bef71ac920
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class FieldOperationApplierForStructs extends FieldOperationApplier {
+    @Override
+    public void process(SDDocumentType sdoc) {
+        for (SDDocumentType type : sdoc.getAllTypes()) {
+            if (type.isStruct()) {
+                apply(type);
+                copyFields(type, sdoc);
+            }
+        }
+    }
+
+    private void copyFields(SDDocumentType structType, SDDocumentType sdoc) {
+        //find all fields in OTHER types that have this type:
+        List<SDDocumentType> list = new ArrayList<>();
+        list.add(sdoc);
+        list.addAll(sdoc.getTypes());
+        for (SDDocumentType anyType : list) {
+            Iterator<Field> fields = anyType.fieldIterator();
+            while (fields.hasNext()) {
+                SDField field = (SDField) fields.next();
+                DataType structUsedByField = field.getFirstStructRecursive();
+                if (structUsedByField == null) {
+                    continue;
+                }
+                if (structUsedByField.getName().equals(structType.getName())) {
+                    //this field is using this type!!
+                    field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0);
+                    field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching());
+                }
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java
new file mode 100644
index 00000000000..503977a8f49
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java
@@ -0,0 +1,65 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.yahoo.searchdefinition.document.FieldSet;
+
+/**
+ * The field sets owned by a {@link Search}
+ * Both built in and user defined.
+ * @author vegardh
+ *
+ */
+public class FieldSets {
+
+    private final Map<String, FieldSet> userFieldSets = new LinkedHashMap<>();
+    private final Map<String, FieldSet> builtInFieldSets = new LinkedHashMap<>();
+
+    /**
+     * Adds an entry to user field sets, creating entries as needed
+     *
+     * @param setName name of a field set
+     * @param field field to add to field set
+     */
+    public void addUserFieldSetItem(String setName, String field) {
+        if (userFieldSets.get(setName) == null) {
+            // First entry in this set
+            userFieldSets.put(setName, new FieldSet(setName));
+        }
+        userFieldSets.get(setName).addFieldName(field);
+    }
+
+    /**
+     * Adds an entry to built in field sets, creating entries as needed
+     *
+     * @param setName name of a field set
+     * @param field field to add to field set
+     */
+    public void addBuiltInFieldSetItem(String setName, String field) {
+        if (builtInFieldSets.get(setName) == null) {
+            // First entry in this set
+            builtInFieldSets.put(setName, new FieldSet(setName));
+        }
+        builtInFieldSets.get(setName).addFieldName(field);
+    }
+
+    /**
+     * The built in field sets, unmodifiable
+     * @return built in field sets
+     */
+    public Map<String, FieldSet> builtInFieldSets() {
+        return Collections.unmodifiableMap(builtInFieldSets);
+    }
+    
+    /**
+     * The user defined field sets, unmodifiable
+     * @return user field sets
+     */
+    public Map<String, FieldSet> userFieldSets() {
+        return Collections.unmodifiableMap(userFieldSets);
+    }
+    
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
new file mode 100644
index 00000000000..9ed7d8677b9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
@@ -0,0 +1,203 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.Stemming;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * An index definition in a search definition.
+ * Two indices are equal if they have the same name and the same settings, except
+ * alias settings (which are excluded).
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class Index implements Cloneable, Serializable {
+
+    public static enum Type {
+        VESPA("vespa");
+        private String name;
+        private Type(String name) { this.name = name; }
+        public String getName() { return name; }
+
+    }
+
+    // Please see hashCode, equals and copy when adding attributes to this
+
+    /** The search definition-unique name of this index */
+    private String name;
+
+    /** The rank type of this index */
+    private RankType rankType=null;
+
+   /** Whether this index supports prefix search */
+    private boolean prefix;
+
+    /** The list of aliases (Strings) to this index name */
+    private Set<String> aliases=new java.util.LinkedHashSet<>(1);
+
+    /**
+     * The stemming setting of this field, or null to use the default.
+     * Default is determined by the owning search definition.
+     */
+    private Stemming stemming=null;
+
+    /** Whether the content of this index is normalized */
+    private boolean normalized=true;
+
+    /** The set of all searchable fields which should be searched with this index. May not null */
+    public Set<String> matchGroup=new LinkedHashSet<>();
+
+    private Type type = Type.VESPA;
+
+    /** The boolean index definition, if set */
+    private BooleanIndexDefinition boolIndex;
+
+    public Index(String name) {
+        this(name, false);
+    }
+
+    public Index(String name, boolean prefix) {
+        this.name=name;
+        this.prefix=prefix;
+    }
+
+    public void setName(String name) { this.name=name; }
+
+    public String getName() { return name; }
+
+    /** Sets the rank type of this field */
+    public void setRankType(RankType rankType) { this.rankType=rankType; }
+
+    /** Returns the rank type of this field, or null if nothing is set */
+    public RankType getRankType() { return rankType; }
+
+    /** Return the stemming setting of this index, may be null */
+    public Stemming getStemming() { return stemming; }
+
+    /**
+     * Returns the (unmodifiable) set of searchable fields which should be searched
+     * when this index is searched. This is useful to specify that some attributes should be
+     * searched as well when an index is searched.
+     * This set is either empty, or if set contains both the name of this index, and some other
+     * indexes.
+     */
+    public Set<String> getMatchGroup() { return Collections.unmodifiableSet(matchGroup); }
+
+    /** Adds a searchable field name to be searched when this index is searched */
+    public void addToMatchGroup(String name) {
+        if (name.equals(this.name)) return;
+        if (matchGroup.size()==0) matchGroup.add(this.name);
+        matchGroup.add(name);
+    }
+
+    /**
+     * Whether this field should be stemmed in this search definition,
+     * this is never null
+     */
+    public Stemming getStemming(Search search) {
+        if (stemming!=null)
+            return stemming;
+        else
+            return search.getStemming();
+    }
+
+    /**
+     * Sets how this field should be stemmed, or set to null to use the default.
+     */
+    public void setStemming(Stemming stemming) { this.stemming=stemming; }
+
+    /** Returns whether this index supports prefix search, default is false */
+    public boolean isPrefix() { return prefix; }
+
+    /** Sets whether this index supports prefix search */
+    public void setPrefix(boolean prefix) { this.prefix=prefix; }
+
+    /** Adds an alias to this index name */
+    public void addAlias(String alias) {
+        aliases.add(alias);
+    }
+
+    /** Returns a read-only iterator of the aliases (Strings) to this index name */
+    public Iterator<String> aliasIterator() {
+        return Collections.unmodifiableSet(aliases).iterator();
+    }
+
+    public int hashCode() {
+            return name.hashCode() + ( prefix ? 17 : 0 );
+    }
+
+    public boolean equals(Object object) {
+            if ( ! (object instanceof Index)) return false;
+
+            Index other=(Index)object;
+            return
+                this.name.equals(other.name) &&
+                this.prefix==other.prefix &&
+                this.stemming==other.stemming &&
+                this.normalized==other.normalized;
+    }
+
+    public String toString() {
+        String rankTypeName=rankType==null ? "(none)" : rankType.name();
+            return
+                "index '" + name +
+                "' [ranktype: " + rankTypeName +
+                ", prefix: " + prefix + "]";
+    }
+
+    /** Makes a deep copy of this index */
+    public Object clone() {
+        try {
+            Index copy=(Index)super.clone();
+            copy.aliases=new LinkedHashSet<>(this.aliases);
+            return copy;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Programming error",e);
+        }
+    }
+
+    public Index copy() {
+        return (Index)clone();
+    }
+
+    /**
+     * The index engine type
+     * @return the type
+     */
+    public Type getType() {
+        return type;
+    }
+
+    /**
+     * Sets the index engine type
+     * @param type a index engine type
+     */
+    public void setType(Type type) {
+        this.type = type;
+    }
+
+    /**
+     * The boolean index definition
+     * @return the boolean index definition
+     */
+    public BooleanIndexDefinition getBooleanIndexDefiniton() {
+        return boolIndex;
+    }
+
+    /**
+     * Sets the boolean index definition
+     * @param def boolean index definition
+     */
+    public void setBooleanIndexDefiniton(BooleanIndexDefinition def) {
+        boolIndex = def;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java
new file mode 100644
index 00000000000..a248ca30a8e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
+
+import java.util.Map;
+
+/**
+ * Inlines macros in ranking expressions
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class MacroInliner extends ExpressionTransformer {
+
+    private final Map<String, RankProfile.Macro> macros;
+
+    public MacroInliner(Map<String, RankProfile.Macro> macros) {
+        this.macros = macros;
+    }
+
+    @Override
+    public ExpressionNode transform(ExpressionNode node) {
+        if (node instanceof ReferenceNode)
+            return transformFeatureNode((ReferenceNode)node);
+        if (node instanceof CompositeNode)
+            return transformChildren((CompositeNode)node);
+        return node;
+    }
+
+    private ExpressionNode transformFeatureNode(ReferenceNode feature) {
+        RankProfile.Macro macro = macros.get(feature.getName());
+        if (macro == null) return feature;
+        return transform(macro.getRankingExpression().getRoot()); // inline recursively and return
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
new file mode 100644
index 00000000000..3e943377611
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -0,0 +1,968 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.search.query.ranking.Diversity;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.FeatureList;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ConstantDereferencer;
+import com.yahoo.searchlib.rankingexpression.transform.Simplifier;
+import com.yahoo.config.application.api.ApplicationPackage;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Represents a rank profile - a named set of ranking settings
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class RankProfile implements Serializable, Cloneable {
+
+    /** The search definition-unique name of this rank profile */
+    private final String name;
+
+    /** The search definition owning this profile, or null if none */
+    private Search search=null;
+
+    /** The name of the rank profile inherited by this */
+    private String inheritedName = null;
+
+    /** The match settings of this profile */
+    protected MatchPhaseSettings matchPhaseSettings = null;
+
+    /** The rank settings of this profile */
+    protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>();
+
+    /** The ranking expression to be used for first phase */
+    private RankingExpression firstPhaseRanking= null;
+
+    /** The ranking expression to be used for second phase */
+    private RankingExpression secondPhaseRanking = null;
+
+    /** Number of hits to be reranked in second phase, -1 means use default */
+    private int rerankCount = -1;
+
+    /** Mysterious attribute */
+    private int keepRankCount = -1;
+
+    private int numThreadsPerSearch = -1;
+    private int numSearchPartitions = -1;
+
+    private double termwiseLimit = 1.0;
+
+    /** The drop limit used to drop hits with rank score less than or equal to this value */
+    private double rankScoreDropLimit = -Double.MAX_VALUE;
+
+    private Set<ReferenceNode> summaryFeatures;
+
+    private Set<ReferenceNode> rankFeatures;
+
+    /** The properties of this - a multimap */
+    private Map<String, List<RankProperty>> rankProperties = new LinkedHashMap<>();
+
+    private Boolean ignoreDefaultRankFeatures=null;
+
+    private String secondPhaseRankingString=null;
+
+    private String firstPhaseRankingString=null;
+
+    private Map<String, Macro> macros= new LinkedHashMap<>();
+
+    private Set<String> filterFields = new HashSet<>();
+
+    private final RankProfileRegistry rankProfileRegistry;
+
+    /** Constants in ranking expressions */
+    private Map<String, Value> constants = new HashMap<>();
+
+    private final TypeSettings attributeTypes = new TypeSettings();
+
+    private final TypeSettings queryFeatureTypes = new TypeSettings();
+
+    /**
+     * Creates a new rank profile
+     *
+     * @param name   the name of the new profile
+     * @param search the search definition owning this profile
+     * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles.
+     */
+    public RankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) {
+        this.name = name;
+        this.search = search;
+        this.rankProfileRegistry = rankProfileRegistry;
+    }
+
+    public String getName() { return name; }
+
+    /**
+     * Returns the search definition owning this, or null if none
+     *
+     * @return The search definition.
+     */
+    public Search getSearch() {
+        return search;
+    }
+
+    /**
+     * Sets the name of the rank profile this inherits. Both rank profiles must be present in the same search
+     * definition
+     *
+     * @param inheritedName The name of the profile that this inherits from.
+     */
+    public void setInherited(String inheritedName) {
+        this.inheritedName = inheritedName;
+    }
+
+    /**
+     * Returns the name of the profile this one inherits, or null if none is inherited
+     *
+     * @return The inherited name.
+     */
+    public String getInheritedName() {
+        return inheritedName;
+    }
+
+    /**
+     * Returns the inherited rank profile, or null if there is none
+     *
+     * @return The inherited profile.
+     */
+    public RankProfile getInherited() {
+        if (getSearch()==null) return getInheritedFromRegistry(inheritedName);
+        RankProfile inheritedInThisSearch = rankProfileRegistry.getRankProfile(search, inheritedName);
+        if (inheritedInThisSearch!=null) return inheritedInThisSearch;
+        return getInheritedFromRegistry(inheritedName);
+    }
+
+    private RankProfile getInheritedFromRegistry(String inheritedName) {
+        for (RankProfile r : rankProfileRegistry.allRankProfiles()) {
+            if (r.getName().equals(inheritedName)) {
+                return r;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Returns whether this profile inherits (directly or indirectly) the given profile
+     *
+     * @param name The profile name to compare this to.
+     * @return Whether or not this inherits from the named profile.
+     */
+    public boolean inherits(String name) {
+        RankProfile parent = getInherited();
+        while (parent != null) {
+            if (parent.getName().equals(name))
+                return true;
+            parent = parent.getInherited();
+        }
+        return false;
+    }
+
+    /**
+     * change match settings
+     * @param settings The new match settings
+     **/
+    public void setMatchPhaseSettings(MatchPhaseSettings settings) {
+        settings.checkValid();
+        this.matchPhaseSettings = settings;
+    }
+
+    public MatchPhaseSettings getMatchPhaseSettings() {
+        MatchPhaseSettings settings = this.matchPhaseSettings;
+        if (settings != null ) return settings;
+        if (getInherited() != null) return getInherited().getMatchPhaseSettings();
+        return null;
+    }
+
+    public void addRankSetting(RankSetting rankSetting) {
+        rankSettings.add(rankSetting);
+    }
+
+    public void addRankSetting(String fieldName, RankSetting.Type type, Object value) {
+        addRankSetting(new RankSetting(fieldName, type, value));
+    }
+
+    /**
+     * Returns the a rank setting of a field, or null if there is no such rank setting in this profile
+     *
+     * @param field The field whose settings to return.
+     * @param type  The type that the field is required to be.
+     * @return The rank setting found, or null.
+     */
+    public RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) {
+        for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) {
+            RankSetting setting = i.next();
+            if (setting.getFieldName().equals(field) &&
+                setting.getType().equals(type)) {
+                return setting;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Returns a rank setting of field or index, or null if there is no such rank setting in this profile or one it
+     * inherits
+     *
+     * @param field The field whose settings to return.
+     * @param type  The type that the field is required to be.
+     * @return The rank setting found, or null.
+     */
+    public RankSetting getRankSetting(String field, RankSetting.Type type) {
+        RankSetting rankSetting = getDeclaredRankSetting(field, type);
+        if (rankSetting != null) return rankSetting;
+
+        if (getInherited() != null) return getInherited().getRankSetting(field, type);
+
+        return null;
+    }
+
+    /**
+     * Returns the rank settings in this rank profile
+     *
+     * @return An iterator for the declared rank setting.
+     */
+    public Iterator<RankSetting> declaredRankSettingIterator() {
+        return Collections.unmodifiableSet(rankSettings).iterator();
+    }
+
+    /**
+     * Returns all settings in this profile or any profile it inherits
+     *
+     * @return An iterator for all rank settings of this.
+     */
+    public Iterator<RankSetting> rankSettingIterator() {
+        return rankSettings().iterator();
+    }
+
+    /**
+     * Returns a snapshot of the rank settings of this and everything it inherits
+     * Changes to the returned set will not be reflected in this rank profile.
+     */
+    public Set<RankSetting> rankSettings() {
+        Set<RankSetting> allSettings = new LinkedHashSet<>(rankSettings);
+        RankProfile parent = getInherited();
+        if (parent != null)
+            allSettings.addAll(parent.rankSettings());
+
+        return allSettings;
+    }
+
+    public void addConstant(String name, Value value) {
+        constants.put(name, value.freeze());
+    }
+
+    public void addConstantTensor(String name, TensorValue value) {
+        addConstant(name, value);
+    }
+
+    /**
+     * Returns an unmodifiable view of the constants to use in this.
+     */
+    public Map<String, Value> getConstants() {
+        if (constants.isEmpty())
+            return getInherited() != null ? getInherited().getConstants() : Collections.<String,Value>emptyMap();
+        if (getInherited() == null || getInherited().getConstants().isEmpty())
+            return Collections.unmodifiableMap(constants);
+
+        Map<String, Value> combinedConstants = new HashMap<>(getInherited().getConstants());
+        combinedConstants.putAll(constants);
+        return combinedConstants;
+    }
+
+    public void addAttributeType(String attributeName, String attributeType) {
+        attributeTypes.addType(attributeName, attributeType);
+    }
+
+    public Map<String, String> getAttributeTypes() {
+        return attributeTypes.getTypes();
+    }
+
+    public void addQueryFeatureType(String queryFeature, String queryFeatureType) {
+        queryFeatureTypes.addType(queryFeature, queryFeatureType);
+    }
+
+    public Map<String, String> getQueryFeatureTypes() {
+        return queryFeatureTypes.getTypes();
+    }
+
+    /**
+     * Returns the ranking expression to use by this. This expression must not be edited.
+     * Returns null if no expression is set.
+     */
+    public RankingExpression getFirstPhaseRanking() {
+        if (firstPhaseRanking!=null) return firstPhaseRanking;
+        if (getInherited()!=null) return getInherited().getFirstPhaseRanking();
+        return null;
+    }
+
+    public void setFirstPhaseRanking(RankingExpression rankingExpression) {
+        this.firstPhaseRanking=rankingExpression;
+    }
+
+    /**
+     * Returns the ranking expression to use by this. This expression must not be edited.
+     * Returns null if no expression is set.
+     */
+    public RankingExpression getSecondPhaseRanking() {
+        if (secondPhaseRanking!=null) return secondPhaseRanking;
+        if (getInherited()!=null) return getInherited().getSecondPhaseRanking();
+        return null;
+    }
+
+    public void setSecondPhaseRanking(RankingExpression rankingExpression) {
+        this.secondPhaseRanking=rankingExpression;
+    }
+
+    /**
+     * Called by parser to store the expression string, for delayed evaluation
+     * @param exp ranking expression for second phase
+     */
+    public void setSecondPhaseRankingString(String exp) {
+        this.secondPhaseRankingString = exp;
+    }
+
+    /**
+     * Called by parser to store the expression string, for delayed evaluation
+     * @param exp ranking expression for first phase
+     */
+    public void setFirstPhaseRankingString(String exp) {
+        this.firstPhaseRankingString = exp;
+    }
+
+    /** Returns a read-only view of the summary features to use in this profile. This is never null */
+    public Set<ReferenceNode> getSummaryFeatures() {
+        if (summaryFeatures!=null) return Collections.unmodifiableSet(summaryFeatures);
+        if (getInherited()!=null) return getInherited().getSummaryFeatures();
+        return Collections.emptySet();
+    }
+
+    public void addSummaryFeature(ReferenceNode feature) {
+        if (summaryFeatures==null)
+            summaryFeatures=new LinkedHashSet<>();
+        summaryFeatures.add(feature);
+    }
+
+    /**
+     * Adds the content of the given feature list to the internal list of summary features.
+     *
+     * @param features The features to add.
+     */
+    public void addSummaryFeatures(FeatureList features) {
+        for (ReferenceNode feature : features) {
+            addSummaryFeature(feature);
+        }
+    }
+
+    /** Returns a read-only view of the rank features to use in this profile. This is never null */
+    public Set<ReferenceNode> getRankFeatures() {
+        if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures);
+        if (getInherited() != null) return getInherited().getRankFeatures();
+        return Collections.emptySet();
+    }
+
+    public void addRankFeature(ReferenceNode feature) {
+        if (rankFeatures==null)
+            rankFeatures=new LinkedHashSet<>();
+        rankFeatures.add(feature);
+    }
+
+    /**
+     * Adds the content of the given feature list to the internal list of rank features.
+     *
+     * @param features The features to add.
+     */
+    public void addRankFeatures(FeatureList features) {
+        for (ReferenceNode feature : features) {
+            addRankFeature(feature);
+        }
+    }
+
+    /** Returns a read only flattened list view of the rank properties to use in this profile. This is never null. */
+    public List<RankProperty> getRankProperties() {
+        List<RankProperty> properties = new ArrayList<>();
+        for (List<RankProperty> propertyList : getRankPropertyMap().values()) {
+            properties.addAll(propertyList);
+        }
+        return Collections.unmodifiableList(properties);
+    }
+
+    /** Returns a read only map view of the rank properties to use in this profile. This is never null. */
+    public Map<String, List<RankProperty>> getRankPropertyMap() {
+        if (rankProperties.size() == 0 && getInherited() == null) return Collections.emptyMap();
+        if (rankProperties.size() == 0) return getInherited().getRankPropertyMap();
+        if (getInherited() == null) return Collections.unmodifiableMap(rankProperties);
+
+        // Neither is null
+        Map<String, List<RankProperty>> combined = new LinkedHashMap<>(getInherited().getRankPropertyMap());
+        combined.putAll(rankProperties); // Don't combine values across inherited properties
+        return Collections.unmodifiableMap(combined);
+    }
+
+    public void addRankProperty(String name, String parameter) {
+        addRankProperty(new RankProperty(name, parameter));
+    }
+
+    public void addRankProperty(RankProperty rankProperty) {
+        // Just the usual multimap semantics here
+        List<RankProperty> properties = rankProperties.get(rankProperty.getName());
+        if (properties == null) {
+            properties = new ArrayList<>(1);
+            rankProperties.put(rankProperty.getName(), properties);
+        }
+        properties.add(rankProperty);
+    }
+
+    public String toString() {
+        return "rank profile " + getName();
+    }
+
+    public int getRerankCount() {
+        if (rerankCount>=0) return rerankCount;
+        if (getInherited()!=null) return getInherited().getRerankCount();
+        return -1;
+    }
+
+    public int getNumThreadsPerSearch() {
+        return numThreadsPerSearch;
+    }
+
+    public void setNumThreadsPerSearch(int numThreads) {
+        this.numThreadsPerSearch = numThreads;
+    }
+
+    public void setNumSearchPartitions(int numSearchPartitions) {
+        this.numSearchPartitions = numSearchPartitions;
+    }
+
+    public int getNumSearchPartitions() { return numSearchPartitions; }
+
+    public double getTermwiseLimit() { return termwiseLimit; }
+    public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; }
+
+    /** Sets the rerank count. Set to -1 to use inherited */
+    public void setRerankCount(int rerankCount) {
+        this.rerankCount = rerankCount;
+    }
+
+    /** Whether we should ignore the default rank features. Set to null to use inherited */
+    public void setIgnoreDefaultRankFeatures(Boolean ignoreDefaultRankFeatures) {
+        this.ignoreDefaultRankFeatures = ignoreDefaultRankFeatures;
+    }
+
+    public boolean getIgnoreDefaultRankFeatures() {
+        if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures;
+        return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures();
+    }
+
+    /**
+     * Returns the string form of the second phase ranking expression.
+     * @return string form of second phase ranking expression
+     */
+    public String getSecondPhaseRankingString() {
+        if (secondPhaseRankingString != null) return secondPhaseRankingString;
+        if (getInherited() != null) return getInherited().getSecondPhaseRankingString();
+        return null;
+    }
+
+    /**
+     * Returns the string form of the first phase ranking expression.
+     * @return string form of first phase ranking expression
+     */
+    public String getFirstPhaseRankingString() {
+        if (firstPhaseRankingString != null) return firstPhaseRankingString;
+        if (getInherited() != null) return getInherited().getFirstPhaseRankingString();
+        return null;
+    }
+
+    public void addMacro(String name, boolean inline) {
+        macros.put(name, new Macro(name, inline));
+    }
+
+    /** Returns an unmodifiable view of the macros in this */
+    public Map<String, Macro> getMacros() {
+        if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap();
+        if (macros.size() == 0) return getInherited().getMacros();
+        if (getInherited() == null) return Collections.unmodifiableMap(macros);
+
+        // Neither is null
+        Map<String, Macro> allMacros = new LinkedHashMap<>(getInherited().getMacros());
+        allMacros.putAll(macros);
+        return Collections.unmodifiableMap(allMacros);
+
+    }
+
+    public int getKeepRankCount() {
+        if (keepRankCount>=0) return keepRankCount;
+        if (getInherited()!=null) return getInherited().getKeepRankCount();
+        return -1;
+    }
+
+    public void setKeepRankCount(int rerankArraySize) {
+        this.keepRankCount = rerankArraySize;
+    }
+
+    public double getRankScoreDropLimit() {
+        if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit;
+        if (getInherited()!=null) return getInherited().getRankScoreDropLimit();
+        return rankScoreDropLimit;
+    }
+
+    public void setRankScoreDropLimit(double rankScoreDropLimit) {
+        this.rankScoreDropLimit = rankScoreDropLimit;
+    }
+
+    public Set<String> filterFields() {
+        return filterFields;
+    }
+
+    /**
+     * Returns all filter fields in this profile and any profile it inherits.
+     * @return the set of all filter fields
+     */
+    public Set<String> allFilterFields() {
+        RankProfile parent = getInherited();
+        Set<String> retval = new LinkedHashSet<>();
+        if (parent != null) {
+            retval.addAll(parent.allFilterFields());
+        }
+        retval.addAll(filterFields());
+        return retval;
+    }
+
+    /**
+     * Will take the parser-set textual ranking expressions and turn into objects
+     */
+    public void parseExpressions() {
+        try {
+            parseRankingExpressions();
+            parseMacros();
+        } catch (ParseException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    /**
+     * Passes the contents of macros on to parser. Then put all the implied rank properties
+     * from those macros into the profile's props map.
+     */
+    private void parseMacros() throws ParseException {
+        for (Map.Entry<String, Macro> e : getMacros().entrySet()) {
+            String macroName = e.getKey();
+            Macro macro = e.getValue();
+            RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression());
+            macro.setRankingExpression(expr);
+            macro.setTextualExpression(expr.getRoot().toString());
+        }
+    }
+
+    /**
+     * Passes ranking expressions on to parser
+     * @throws ParseException if either of the ranking expressions could not be parsed
+     */
+    private void parseRankingExpressions() throws ParseException {
+        if (getFirstPhaseRankingString() != null)
+            setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString()));
+        if (getSecondPhaseRankingString() != null)
+            setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString()));
+    }
+
+    private RankingExpression parseRankingExpression(String expName, String exp) throws ParseException {
+        if (exp.trim().length() == 0)
+            throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expName + ".");
+
+        try {
+            RankingExpression expression = new RankingExpression(openRankingExpressionReader(expName, exp.trim()));
+            expression.setName(expName);
+            return expression;
+        }
+        catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
+            ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() +
+                                                          "' in " + getName()+ ", " + expName + ".");
+            throw (ParseException)exception.initCause(e);
+        }
+    }
+
+    private Reader openRankingExpressionReader(String expName, String expression) {
+        if ( ! expression.startsWith("file:")) return new StringReader(expression);
+
+        String fileName = expression.substring("file:".length()).trim();
+        if ( ! fileName.endsWith(ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX))
+            fileName = fileName + ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX;
+
+        final File file = new File(fileName);
+        if ( ! (file.isAbsolute()) && file.getPath().contains("/")) // See ticket 4102122
+            throw new IllegalArgumentException("In " + getName() +", " + expName + ", ranking references file '" + file +
+                                               "' in subdirectory, which is not supported.");
+
+        return search.getRankingExpression(fileName);
+    }
+
+    /** Shallow clones this */
+    @Override
+    public RankProfile clone() {
+        try {
+            // Note: This treats RankingExpression in Macros as immutables even though they are not
+            RankProfile clone = (RankProfile)super.clone();
+            clone.rankSettings = new LinkedHashSet<>(this.rankSettings);
+            clone.matchPhaseSettings = this.matchPhaseSettings; // hmm?
+            clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null;
+            clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null;
+            clone.rankProperties = new LinkedHashMap<>(this.rankProperties);
+            clone.macros = new LinkedHashMap<>(this.macros);
+            clone.filterFields = new HashSet<>(this.filterFields);
+            clone.constants = new HashMap<>(this.constants);
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Won't happen", e);
+        }
+    }
+
+    /**
+     * Returns a copy of this where the content is optimized for execution.
+     * Compiled profiles should never be modified.
+     */
+    public RankProfile compile() {
+        try {
+            RankProfile compiled = this.clone();
+            compiled.compileThis();
+            return compiled;
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Rank profile '" + getName() + "' is invalid", e);
+        }
+    }
+
+    private void compileThis() {
+        parseExpressions();
+
+        checkNameCollisions(getMacros(), getConstants());
+
+        Map<String, Macro> compiledMacros = new LinkedHashMap<>();
+        for (Map.Entry<String, Macro> macroEntry : getMacros().entrySet()) {
+            Macro compiledMacro = macroEntry.getValue().clone();
+            compiledMacro.setRankingExpression(compile(macroEntry.getValue().getRankingExpression(),
+                    getConstants(), Collections.<String, Macro>emptyMap()));
+            compiledMacros.put(macroEntry.getKey(), compiledMacro);
+        }
+        macros = compiledMacros;
+        Map<String, Macro> inlineMacros = keepInline(compiledMacros);
+        firstPhaseRanking = compile(this.getFirstPhaseRanking(), getConstants(), inlineMacros);
+        secondPhaseRanking = compile(this.getSecondPhaseRanking(), getConstants(), inlineMacros);
+    }
+
+    private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) {
+        for (Map.Entry<String, Macro> macroEntry : macros.entrySet()) {
+            if (constants.get(macroEntry.getKey()) != null)
+                throw new IllegalArgumentException("Cannot have both a constant and macro named '" + macroEntry.getKey() + "'");
+        }
+    }
+
+    private Map<String, Macro> keepInline(Map<String, Macro> macros) {
+        Map<String, Macro> inlineMacros = new HashMap<>();
+        for (Map.Entry<String, Macro> entry : macros.entrySet())
+            if (entry.getValue().getInline())
+                inlineMacros.put(entry.getKey(), entry.getValue());
+        return inlineMacros;
+    }
+
+    private RankingExpression compile(RankingExpression expression,
+                                      Map<String, Value> constants,
+                                      Map<String, Macro> inlineMacros) {
+        if (expression == null) return null;
+        Map<String, String> rankPropertiesOutput = new HashMap<>();
+        expression = new ConstantDereferencer(constants).transform(expression);
+        expression = new ConstantTensorTransformer(constants, rankPropertiesOutput).transform(expression);
+        expression = new MacroInliner(inlineMacros).transform(expression);
+        expression = new Simplifier().transform(expression);
+        for (Map.Entry<String, String> rankProperty : rankPropertiesOutput.entrySet()) {
+            addRankProperty(rankProperty.getKey(), rankProperty.getValue());
+        }
+        return expression;
+    }
+
+    /**
+     * A rank setting. The identity of a rank setting is its field name and type (not value).
+     * A rank setting is immutable.
+     */
+    public static class RankSetting implements Serializable {
+
+        private String fieldName;
+
+        private Type type;
+
+        /** The rank value */
+        private Object value;
+
+        public enum Type {
+
+            RANKTYPE("rank-type"),
+            LITERALBOOST("literal-boost"),
+            WEIGHT("weight"),
+            PREFERBITVECTOR("preferbitvector",true);
+
+            private String name;
+
+            /** True if this setting really pertains to an index, not a field within an index */
+            private boolean isIndexLevel;
+
+            private Type(String name) {
+                this(name,false);
+            }
+
+            private Type(String name,boolean isIndexLevel) {
+                this.name = name;
+                this.isIndexLevel=isIndexLevel;
+            }
+
+            /** True if this setting really pertains to an index, not a field within an index */
+            public boolean isIndexLevel() { return isIndexLevel; }
+
+            /** @return The name of this type */
+            public String getName() {
+                return name;
+            }
+
+            public String toString() {
+                return "type: " + name;
+            }
+
+        }
+
+        public RankSetting(String fieldName, RankSetting.Type type, Object value) {
+            this.fieldName = fieldName;
+            this.type = type;
+            this.value = value;
+        }
+
+        public String getFieldName() { return fieldName; }
+
+        public Type getType() { return type; }
+
+        public Object getValue() { return value; }
+
+        /** @return The value as an int, or a negative value if it is not an integer */
+        public int getIntValue() {
+            if (value instanceof Integer) {
+                return ((Integer)value);
+            }
+            else {
+                return -1;
+            }
+        }
+
+        public int hashCode() {
+            return fieldName.hashCode() + 17 * type.hashCode();
+        }
+
+        public boolean equals(Object object) {
+            if (!(object instanceof RankSetting)) {
+                return false;
+            }
+            RankSetting other = (RankSetting)object;
+            return
+                    fieldName.equals(other.fieldName) &&
+                    type.equals(other.type);
+        }
+
+        public String toString() {
+            return type + " setting " + fieldName + ": " + value;
+        }
+
+    }
+
+    /** A rank property. Rank properties are Value Objects */
+    public static class RankProperty implements Serializable {
+
+        private String name;
+        private String value;
+
+        public RankProperty(String name, String value) {
+            this.name = name;
+            this.value = value;
+        }
+
+        public String getName() { return name; }
+
+        public String getValue() { return value; }
+
+        @Override
+        public int hashCode() {
+            return name.hashCode() + 17*value.hashCode();
+        }
+
+        @Override
+        public boolean equals(Object object) {
+            if (! (object instanceof RankProperty)) return false;
+            RankProperty other=(RankProperty)object;
+            return (other.name.equals(this.name) && other.value.equals(this.value));
+        }
+
+        @Override
+        public String toString() {
+            return name + " = " + value;
+        }
+
+    }
+
+    /**
+     * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro
+     *
+     * @author vegardh
+     */
+    public static class Macro implements Serializable, Cloneable {
+
+        private String name=null;
+        private String textualExpression=null;
+        private RankingExpression expression=null;
+        private List<String> formalParams = new ArrayList<>();
+
+        /** True if this should be inlined into calling expressions. Useful for very cheap macros. */
+        private final boolean inline;
+
+        public Macro(String name, boolean inline) {
+            this.name = name;
+            this.inline = inline;
+        }
+
+        public void addParam(String name) {
+            formalParams.add(name);
+        }
+
+        public List<String> getFormalParams() {
+            return formalParams;
+        }
+
+        public String getTextualExpression() {
+            return textualExpression;
+        }
+
+        public void setTextualExpression(String textualExpression) {
+            this.textualExpression = textualExpression;
+        }
+
+        public void setRankingExpression(RankingExpression expr) {
+            this.expression=expr;
+        }
+
+        public RankingExpression getRankingExpression() {
+            return expression;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public boolean getInline() {
+            return inline && formalParams.size() == 0; // only inline no-arg macros;
+        }
+
+        public ExpressionFunction toExpressionMacro() {
+            return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression());
+        }
+
+        @Override
+        public Macro clone() {
+            try {
+                return (Macro)super.clone();
+            }
+            catch (CloneNotSupportedException e) {
+                throw new RuntimeException("Won't happen", e);
+            }
+        }
+
+        @Override
+        public String toString() {
+            return "macro " + getName() + ": " + expression;
+        }
+
+    }
+
+    public static final class DiversitySettings {
+        private String attribute = null;
+        private int minGroups = 0;
+        private double cutoffFactor = 10;
+        private Diversity.CutoffStrategy cutoffStrategy = Diversity.CutoffStrategy.loose;
+
+        public void setAttribute(String value) { attribute = value; }
+        public void setMinGroups(int value) { minGroups = value; }
+        public void setCutoffFactor(double value) { cutoffFactor = value; }
+        public void setCutoffStrategy(Diversity.CutoffStrategy strategy) { cutoffStrategy = strategy; }
+        public void setCutoffStrategy(String strategy) { cutoffStrategy = Diversity.CutoffStrategy.valueOf(strategy); }
+        public String getAttribute() { return attribute; }
+        public int getMinGroups() { return minGroups; }
+        public double getCutoffFactor() { return cutoffFactor; }
+        public Diversity.CutoffStrategy getCutoffStrategy() { return cutoffStrategy; }
+        public void checkValid() {
+            if (attribute == null || attribute.isEmpty()) {
+                throw new IllegalArgumentException("'diversity' did not set non-empty diversity attribute name.");
+            }
+            if (minGroups <= 0) {
+                throw new IllegalArgumentException("'diversity' did not set min-groups > 0");
+            }
+            if (cutoffFactor < 1.0) {
+                throw new IllegalArgumentException("diversity.cutoff.factor must be larger or equal to 1.0.");
+            }
+        }
+    }
+
+    public static class MatchPhaseSettings {
+        private String attribute = null;
+        private boolean ascending = false;
+        private int maxHits = 0; // try to get this many hits before degrading the match phase
+        private double maxFilterCoverage = 1.0; // Max coverage of original corpus that will trigger the filter.
+        private DiversitySettings diversity = null;
+        private double evaluationPoint = 0.20;
+        private double prePostFilterTippingPoint = 1.0;
+
+        public void setDiversity(DiversitySettings value) {
+            value.checkValid();
+            diversity = value;
+        }
+        public void setAscending(boolean value) { ascending = value; }
+        public void setAttribute(String value) { attribute = value; }
+        public void setMaxHits(int value) { maxHits = value; }
+        public void setMaxFilterCoverage(double value) { maxFilterCoverage = value; }
+        public void setEvaluationPoint(double evaluationPoint) { this.evaluationPoint = evaluationPoint; }
+        public void setPrePostFilterTippingPoint(double prePostFilterTippingPoint) { this.prePostFilterTippingPoint = prePostFilterTippingPoint; }
+
+        public boolean                getAscending() { return ascending; }
+        public String                 getAttribute() { return attribute; }
+        public int                      getMaxHits() { return maxHits; }
+        public double         getMaxFilterCoverage() { return maxFilterCoverage; }
+        public DiversitySettings      getDiversity() { return diversity; }
+        public double           getEvaluationPoint() { return evaluationPoint; }
+        public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; }
+
+        public void checkValid() {
+            if (attribute == null) {
+                throw new IllegalArgumentException("match-phase did not set any attribute");
+            }
+            if (! (maxHits > 0)) {
+                throw new IllegalArgumentException("match-phase did not set max-hits > 0");
+            }
+        }
+            
+    }
+
+    public static class TypeSettings {
+
+        private final Map<String, String> types = new HashMap<>();
+
+        public void addType(String name, String type) {
+            types.put(name, type);
+        }
+
+        public Map<String, String> getTypes() {
+            return Collections.unmodifiableMap(types);
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java
new file mode 100644
index 00000000000..a9ee3c3cc5c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import java.util.*;
+
+/**
+ * Mapping from name to {@link RankProfile} as well as a reverse mapping of {@link RankProfile} to {@link Search}.
+ * Having both of these mappings consolidated here will make it easier to remove dependencies on these mappings at
+ * run time, since it is essentially only used when building rank profile config at deployment time.
+ *
+ * TODO: Reconsider the difference between local and global maps. Right now, the local maps might better be
+ *       served from a different class owned by SearchBuilder.
+ *
+ * @author lulf
+ * @since 5.20
+ */
+public class RankProfileRegistry {
+
+    private final Map<RankProfile, Search> rankProfileToSearch = new LinkedHashMap<>();
+    private final Map<Search, Map<String, RankProfile>> rankProfiles = new LinkedHashMap<>();
+    /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */
+    static final Set<String> overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked"));
+
+    public RankProfileRegistry() {
+    }
+
+    public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Search search) {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        rankProfileRegistry.addRankProfile(new DefaultRankProfile(search, rankProfileRegistry));
+        rankProfileRegistry.addRankProfile(new UnrankedRankProfile(search, rankProfileRegistry));
+        return rankProfileRegistry;
+    }
+
+    /**
+     * Adds a rank profile to this registry
+     *
+     * @param rankProfile the rank profile to add
+     */
+    public void addRankProfile(RankProfile rankProfile) {
+        if (!rankProfiles.containsKey(rankProfile.getSearch())) {
+            rankProfiles.put(rankProfile.getSearch(), new LinkedHashMap<>());
+        }
+        checkForDuplicateRankProfile(rankProfile);
+        rankProfiles.get(rankProfile.getSearch()).put(rankProfile.getName(), rankProfile);
+        rankProfileToSearch.put(rankProfile, rankProfile.getSearch());
+    }
+
+    private void checkForDuplicateRankProfile(RankProfile rankProfile) {
+        final String rankProfileName = rankProfile.getName();
+        RankProfile existingRangProfileWithSameName = rankProfiles.get(rankProfile.getSearch()).get(rankProfileName);
+        if (existingRangProfileWithSameName == null) return;
+
+        if (!overridableRankProfileNames.contains(rankProfileName)) {
+            throw new IllegalArgumentException("Cannot add rank profile '" + rankProfileName + "' in search definition '"
+                    + rankProfile.getSearch().getName() + "', since it already exists");
+        }
+    }
+
+    /**
+     * Returns a named rank profile, null if the search definition doesn't have one with the given name
+     *
+     * @param search The {@link Search} that owns the rank profile.
+     * @param name The name of the rank profile
+     * @return The RankProfile to return.
+     */
+    public RankProfile getRankProfile(Search search, String name) {
+        return rankProfiles.get(search).get(name);
+    }
+
+    /**
+     * Rank profiles that are collected across clusters.
+     * @return A set of global {@link RankProfile} instances.
+     */
+    public Set<RankProfile> allRankProfiles() {
+        return rankProfileToSearch.keySet();
+    }
+
+    /**
+     * Rank profiles that are collected for a given search definition
+     * @param search {@link Search} to get rank profiles for.
+     * @return A collection of local {@link RankProfile} instances.
+     */
+    public Collection<RankProfile> localRankProfiles(Search search) {
+        Map<String, RankProfile> mapping = rankProfiles.get(search);
+        if (mapping == null) {
+            return Collections.emptyList();
+        }
+        return mapping.values();
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
new file mode 100644
index 00000000000..7d8a87cee8c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java
@@ -0,0 +1,126 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.TemporarySDDocumentType;
+
+import java.util.*;
+import java.util.logging.Level;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SDDocumentTypeOrderer {
+    private Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>();
+    private Set<Integer> seenTypes = new LinkedHashSet<>();
+    List<SDDocumentType> processingOrder = new LinkedList<>();
+    private DeployLogger deployLogger;
+
+    public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) {
+        this.deployLogger = deployLogger;
+        for (SDDocumentType type : sdTypes) {
+            createdSDTypes.put(type.getDocumentName(), type);
+        }
+        DocumentTypeManager dtm = new DocumentTypeManager();
+        for (DataType type : dtm.getDataTypes()) {
+            seenTypes.add(type.getId());
+        }
+
+    }
+
+    List<SDDocumentType> getOrdered() { return processingOrder; }
+
+    public void process() {
+        for (SDDocumentType type : createdSDTypes.values()) {
+            process(type);
+        }
+    }
+    private void process(SDDocumentType type) {
+        List<DataTypeName> toReplace = new ArrayList<>();
+        for (SDDocumentType sdoc : type.getInheritedTypes()) {
+            if (sdoc instanceof TemporarySDDocumentType) {
+                toReplace.add(sdoc.getDocumentName());
+            }
+        }
+        for (DataTypeName name : toReplace) {
+            SDDocumentType inherited = createdSDTypes.get(name);
+            if (inherited == null) {
+                throw new IllegalStateException("Document type '" + name + "' not found.");
+            }
+            process(inherited);
+            type.inherit(inherited);
+        }
+        visit(type);
+    }
+
+    private void visit(SDDocumentType docOrStruct) {
+        int id;
+        if (docOrStruct.isStruct()) {
+            id = new StructDataType(docOrStruct.getName()).getId();
+        } else {
+            id = new DocumentType(docOrStruct.getName()).getId();
+        }
+
+        if (seenTypes.contains(id)) {
+            return;
+        } else {
+            seenTypes.add((new StructDataType(docOrStruct.getName()).getId()));
+        }
+
+
+        for (Field field : docOrStruct.fieldSet()) {
+            if (!seenTypes.contains(field.getDataType().getId())) {
+                //we haven't seen this before, do it
+                visit(field.getDataType());
+            }
+        }
+        processingOrder.add(docOrStruct);
+    }
+
+    private SDDocumentType find(String name) {
+        SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name));
+        if (sdDocType != null) {
+            return sdDocType;
+        }
+        for(SDDocumentType sdoc : createdSDTypes.values()) {
+             for (SDDocumentType stype : sdoc.getTypes()) {
+                 if (stype.getName().equals(name)) {
+                    return stype;
+                 }
+             }
+        }
+        return null;
+    }
+    private void visit(DataType type) {
+        if (type instanceof StructuredDataType) {
+            StructuredDataType structType = (StructuredDataType) type;
+            SDDocumentType sdDocType = find(structType.getName());
+            if (sdDocType == null) {
+                throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'.");
+            }
+            visit(sdDocType);
+            return;
+        }
+
+        if (type instanceof MapDataType) {
+            MapDataType mType = (MapDataType) type;
+            visit(mType.getValueType());
+            visit(mType.getKeyType());
+        } else if (type instanceof WeightedSetDataType) {
+            WeightedSetDataType wType = (WeightedSetDataType) type;
+            visit(wType.getNestedType());
+        } else if (type instanceof CollectionDataType) {
+            CollectionDataType cType = (CollectionDataType) type;
+            visit(cType.getNestedType());
+        } else if (type instanceof AnnotationReferenceDataType) {
+            //do nothing
+        } else if (type instanceof PrimitiveDataType) {
+            //do nothing
+        } else {
+            deployLogger.log(Level.WARNING, "Unknown type : " + type);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
new file mode 100644
index 00000000000..43c6bb4b441
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -0,0 +1,555 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.document.*;
+import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+import java.io.Reader;
+import java.io.Serializable;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * <p>A search definition describes (or uses) some document types, defines how these are turned into a relevancy tuned
+ * index through indexing and how data from documents should be served at search time.</p> <p>The identity of this
+ * class is its name.</p>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+// TODO: Make a class owned by this, for each of these responsibilities:
+// Managing indexes, managing attributes, managing summary classes.
+// Ensure that after the processing step, all implicit instances of the above types are explicitly represented
+public class Search implements Serializable {
+
+    private static final Logger log = Logger.getLogger(Search.class.getName());
+    private static final String SD_DOC_FIELD_NAME = "sddocname";
+    private static final List<String> RESERVED_NAMES = Arrays.asList(
+            "index", "index_url", "summary", "attribute", "select_input", "host", "documentid",
+            "position", "split_foreach", "tokenize", "if", "else", "switch", "case", SD_DOC_FIELD_NAME, "relevancy");
+
+    /**
+     * @return True if the given field name is a reserved name.
+     */
+    public static boolean isReservedName(String name) {
+        return RESERVED_NAMES.contains(name);
+    }
+
+    // Field sets
+    private FieldSets fieldSets = new FieldSets();
+    
+    // Whether or not this object has been processed.
+    private boolean processed;
+
+    // The unique name of this search definition.
+    private String name;
+
+    // True if this doesn't define a search, just some documents.
+    private boolean documentsOnly = false;
+
+    // The stemming setting of this search definition. Default is SHORTEST.
+    private Stemming stemming = Stemming.SHORTEST;
+
+    // Documents contained in this definition.
+    private SDDocumentType docType;
+
+    // The extra fields of this search definition.
+    private Map<String, SDField> fields = new LinkedHashMap<>();
+
+    // The explicitly defined indices of this search definition.
+    private Map<String, Index> indices = new LinkedHashMap<>();
+
+    // The explicitly defined summaries of this search definition.
+    private Map<String, DocumentSummary> summaries = new LinkedHashMap<>();
+    // _Must_ preserve order
+
+    private ApplicationPackage sourceApplication;
+
+    /**
+     * Creates a search definition which just holds a set of documents which should not (here, directly) be searchable
+     */
+    protected Search() {
+        documentsOnly = true;
+    }
+
+    /**
+     * Creates a proper search definition
+     * @param name of the the searchdefinition
+     * @param sourceApplication the application containing this
+     */
+    public Search(String name, ApplicationPackage sourceApplication) {
+        this.sourceApplication = sourceApplication;
+        this.name = name;
+    }
+
+    protected void setName(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Returns true if this doesn't define a search, just some documents
+     *
+     * @return if the searchdefinition only has documents
+     */
+    public boolean isDocumentsOnly() {
+        return documentsOnly;
+    }
+
+    /**
+     * Sets the stemming default of fields. Default is ALL
+     *
+     * @param stemming set default stemming for this searchdefinition
+     * @throws NullPointerException if this is attempted set to null
+     */
+    public void setStemming(Stemming stemming) {
+        if (stemming == null) {
+            throw new NullPointerException("The stemming setting of a search definition " +
+                                           "can not be null");
+        }
+        this.stemming = stemming;
+    }
+
+    /**
+     * Returns whether fields should be stemmed by default or not. Default is ALL. This is never null.
+     *
+     * @return the default stemming for this searchdefinition
+     */
+    public Stemming getStemming() {
+        return stemming;
+    }
+
+    /**
+     * Adds a document type which is defined in this search definition
+     *
+     * @param document the document type to add
+     */
+    public void addDocument(SDDocumentType document) {
+        if (docType != null) {
+            throw new IllegalArgumentException("Searchdefinition cannot have more than one document");
+        }
+        docType = document;
+    }
+
+    /**
+     * Gets a document from this search definition
+     *
+     * @param name the name of the document to return
+     * @return the contained or used document type, or null if there is no such document
+     */
+    public SDDocumentType getDocument(String name) {
+        if (docType != null && name.equals(docType.getName())) {
+            return docType;
+        }
+        return null;
+    }
+
+    /**
+     * @return true if the document has been added.
+     */
+    public boolean hasDocument() {
+        return docType != null;
+    }
+
+    /**
+     * @return The document in this search.
+     */
+    public SDDocumentType getDocument() {
+        return docType;
+    }
+
+    /**
+     * Returns a list of all the fields of this search definition, that is all fields in all documents, in the documents
+     * they inherit, and all extra fields. The caller receives ownership to the list - subsequent changes to it will not
+     * impact this Search
+     *
+     * @return the list of fields in this searchdefinition
+     */
+    public List<SDField> allFieldsList() {
+        List<SDField> allFields = new ArrayList<>();
+        allFields.addAll(extraFieldList());
+        for (Field field : docType.fieldSet()) {
+            allFields.add((SDField)field);
+        }
+        return allFields;
+    }
+
+    /**
+     * Returns the content of a ranking expression file
+     */
+    public Reader getRankingExpression(String fileName) {
+        return sourceApplication.getRankingExpression(fileName);
+    }
+
+    /**
+     * Returns a field defined in this search definition or one if its documents. Fields in this search definition takes
+     * precedence over document fields having the same name
+     *
+     * @param name of the field
+     * @return the SDField representing the field
+     */
+    public SDField getField(String name) {
+        SDField field = getExtraField(name);
+        if (field != null) {
+            return field;
+        }
+        return (SDField)docType.getField(name);
+    }
+
+    /**
+     * Returns a field defined in one of the documents of this search definition. This does <b>not</b> include the extra
+     * fields defined outside of a document (those accessible through the getExtraField() method).
+     *
+     * @param name The name of the field to return.
+     * @return The named field, or null if not found.
+     */
+    public SDField getDocumentField(String name) {
+        return (SDField)docType.getField(name);
+    }
+
+    /**
+     * Adds an extra field of this search definition not contained in a document
+     *
+     * @param field to add to the searchdefinitions list of external fields.
+     */
+    public void addExtraField(SDField field) {
+        if (fields.containsKey(field.getName())) {
+            log.warning("Duplicate field " + field.getName() + " in search definition " + getName());
+        } else {
+            field.setIsExtraField(true);
+            fields.put(field.getName(), field);
+        }
+    }
+
+    public Collection<SDField> extraFieldList() {
+        return fields.values();
+    }
+    public Collection<SDField> allExtraFields() {
+        Map<String, SDField> extraFields = new TreeMap<>();
+        for (Field field : docType.fieldSet()) {
+            SDField sdField = (SDField) field;
+            if (sdField.isExtraField()) {
+                extraFields.put(sdField.getName(), sdField);
+            }
+        }
+        for (SDField field : extraFieldList()) {
+            extraFields.put(field.getName(), field);
+        }
+        return extraFields.values();
+    }
+
+    /**
+     * Returns a field by name, or null if it is not present
+     *
+     * @param fieldName the name of the external field to get
+     * @return the SDField of this name
+     */
+    public SDField getExtraField(String fieldName) {
+        return fields.get(fieldName);
+    }
+
+    /**
+     * Adds an explicitly defined index to this search definition
+     *
+     * @param index the index to add
+     */
+    public void addIndex(Index index) {
+        indices.put(index.getName(), index);
+    }
+
+    /**
+     * <p>Returns an index, or null if no index with this name has had some <b>explicit settings</b> applied. Even if
+     * this returns null, the index may be implicitly defined by an indexing statement.</p>
+     * <p>This will return the
+     * index whether it is defined on this search or on one of its fields</p>
+     *
+     * @param name the name of the index to get
+     * @return the index requested
+     */
+    public Index getIndex(String name) {
+        List<Index> sameIndices = new ArrayList<>(1);
+        Index searchIndex = indices.get(name);
+        if (searchIndex != null) {
+            sameIndices.add(searchIndex);
+        }
+
+        for (SDField field : allFieldsList()) {
+            Index index = field.getIndex(name);
+            if (index != null) {
+                sameIndices.add(index);
+            }
+        }
+        if (sameIndices.size() == 0) {
+            return null;
+        }
+        if (sameIndices.size() == 1) {
+            return sameIndices.get(0);
+        }
+        return consolidateIndices(sameIndices);
+    }
+
+    public boolean existsIndex(String name) {
+        if (indices.get(name) != null) {
+            return true;
+        }
+        for (SDField field : allFieldsList()) {
+            if (field.existsIndex(name)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Consolidates a set of index settings for the same index into one
+     *
+     * @param indices The list of indexes to consolidate.
+     * @return The consolidated index
+     */
+    private Index consolidateIndices(List<Index> indices) {
+        Index first = indices.get(0);
+        Index consolidated = new Index(first.getName());
+        consolidated.setRankType(first.getRankType());
+        consolidated.setType(first.getType());
+        for (Index current : indices) {
+            if (current.isPrefix()) {
+                consolidated.setPrefix(true);
+            }
+
+            if (consolidated.getRankType() == null) {
+                consolidated.setRankType(current.getRankType());
+            } else {
+                if (current.getRankType() != null &&
+                    !consolidated.getRankType().equals(current.getRankType()))
+                {
+                    log.warning("Conflicting rank type settings for " +
+                                first.getName() + " in " + this + ", using " +
+                                consolidated.getRankType());
+                }
+            }
+
+            for (Iterator<String> j = current.aliasIterator(); j.hasNext();) {
+                consolidated.addAlias(j.next());
+            }
+        }
+        return consolidated;
+    }
+
+    /**
+     * All explicitly defined indices, both on this search definition itself (returned first) and all its fields
+     *
+     * @return The list of explicit defined indexes.
+     */
+    public List<Index> getExplicitIndices() {
+        List<Index> allIndices = new ArrayList<>(indices.values());
+        for (SDField field : allFieldsList()) {
+            for (Index index : field.getIndices().values()) {
+                allIndices.add(index);
+            }
+        }
+        return Collections.unmodifiableList(allIndices);
+    }
+
+    /**
+     * Adds an explicitly defined summary to this search definition
+     *
+     * @param summary The summary to add.
+     */
+    public void addSummary(DocumentSummary summary) {
+        summaries.put(summary.getName(), summary);
+    }
+
+    /**
+     * <p>Returns a summary class defined by this search definition, or null if no summary with this name is defined.
+     * The default summary, named "default" is always present.</p>
+     *
+     * @param name the name of the summary to get.
+     * @return Summary found.
+     */
+    public DocumentSummary getSummary(String name) {
+        return summaries.get(name);
+    }
+
+    /**
+     * Returns the first explicit instance found of a summary field with this name, or null if not present (implicitly
+     * or explicitly) in any summary class.
+     *
+     * @param name The name of the summaryfield to get.
+     * @return SummaryField to return.
+     */
+    public SummaryField getSummaryField(String name) {
+        for (DocumentSummary summary : summaries.values()) {
+            SummaryField summaryField = summary.getSummaryField(name);
+            if (summaryField != null) {
+                return summaryField;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Returns the first explicit instance found of a summary field with this name, or null if not present explicitly in
+     * any summary class
+     *
+     * @param name Thge name of the explicit summary field to get.
+     * @return The SummaryField found.
+     */
+    public SummaryField getExplicitSummaryField(String name) {
+        for (DocumentSummary summary : summaries.values()) {
+            SummaryField summaryField = summary.getSummaryField(name);
+            if (summaryField != null && !summaryField.isImplicit()) {
+                return summaryField;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Summaries defined by fields of this search definition. The default summary, named "default", is always the first
+     * one in the returned iterator.
+     *
+     * @return The map of document summaries.
+     */
+    public Map<String, DocumentSummary> getSummaries() {
+        return summaries;
+    }
+
+    /**
+     * <p>Returns all summary fields, of all document summaries, which has the given field as source. If there are
+     * multiple summary fields with the same name, the last one will be used (they should all have the same content, if
+     * this is a valid search definition).</p> <p>The map gets owned by the receiver.</p>
+     *
+     * @param field The source field.
+     * @return The map of summary fields found.
+     */
+    public Map<String, SummaryField> getSummaryFields(SDField field) {
+        Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>();
+        for (DocumentSummary documentSummary : summaries.values()) {
+            for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+                if (summaryField.hasSource(field.getName())) {
+                    summaryFields.put(summaryField.getName(), summaryField);
+                }
+            }
+        }
+        return summaryFields;
+    }
+
+    /**
+     * <p>Returns one summary field for each summary field name. If there are multiple summary fields with the same
+     * name, the last one will be used. Multiple fields of the same name should all have the same content in a valid
+     * search definition, except from the destination set. So this method can be used for all summary handling except
+     * processing the destination set.</p> <p>The map gets owned by the receiver.</p>
+     *
+     * @return Map of unique summary fields
+     */
+    public Map<String, SummaryField> getUniqueNamedSummaryFields() {
+        Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>();
+        for (DocumentSummary documentSummary : summaries.values()) {
+            for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+                summaryFields.put(summaryField.getName(), summaryField);
+            }
+        }
+        return summaryFields;
+    }
+
+    public int hashCode() {
+        return name.hashCode();
+    }
+
+    /**
+     * Returns the first occurence of an attribute having this name, or null if none
+     *
+     * @param name Name of attribute
+     * @return The Attribute with given name.
+     */
+    public Attribute getAttribute(String name) {
+        for (SDField field : allFieldsList()) {
+            Attribute attribute = field.getAttributes().get(name);
+            if (attribute != null) {
+                return attribute;
+            }
+        }
+        return null;
+    }
+
+    public boolean equals(Object o) {
+        if (!(o instanceof Search)) {
+            return false;
+        }
+
+        Search other = (Search)o;
+        return getName().equals(other.getName());
+    }
+
+    public String toString() {
+        return "search definition '" + getName() + "'";
+    }
+
+    public boolean isAccessingDiskSummary(SummaryField field) {
+        if (!field.getTransform().isInMemory()) {
+            return true;
+        }
+        if (field.getSources().size() == 0) {
+            return isAccessingDiskSummary(getName());
+        }
+        for (SummaryField.Source source : field.getSources()) {
+            if (isAccessingDiskSummary(source.getName())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean isAccessingDiskSummary(String source) {
+        SDField field = getField(source);
+        if (field == null) {
+            return false;
+        }
+        if (field.doesSummarying() && !field.doesAttributing()) {
+            return true;
+        }
+        return false;
+    }
+
+    public void process() {
+        if (processed) {
+            throw new IllegalStateException("Search '" + getName() + "' already processed.");
+        }
+        processed = true;
+    }
+
+    public boolean isProcessed() {
+        return processed;
+    }
+
+    /**
+     * The field set settings for this search
+     *
+     * @return field set settings for this
+     */
+    public FieldSets fieldSets() {
+        return fieldSets;
+    }
+
+    /**
+     * For adding structs defined in document scope
+     *
+     * @param dt The struct to add.
+     * @return self, for chaining
+     */
+    public Search addType(SDDocumentType dt) {
+        docType.addType(dt); // TODO This is a very very dirty thing. It must go
+        return this;
+    }
+
+    public Search addAnnotation(SDAnnotationType dt) {
+        docType.addAnnotation(dt);
+        return this;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
new file mode 100644
index 00000000000..9c9bc559dc3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -0,0 +1,429 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.document.DocumentTypeManager;
+import com.yahoo.io.IOUtils;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.searchdefinition.derived.SearchOrderer;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.searchdefinition.parser.SDParser;
+import com.yahoo.searchdefinition.parser.SimpleCharStream;
+import com.yahoo.searchdefinition.parser.TokenMgrError;
+import com.yahoo.searchdefinition.processing.Processing;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+
+/**
+ * Helper class for importing {@link Search} objects in an unambiguous way. The pattern for using this is to 1) Import
+ * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank
+ * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built
+ * search objects using the {@link #getSearch(String)} method.
+ *
+ * @author TODO: Who created this?
+ */
+// TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage
+public class SearchBuilder {
+
+    private final DocumentTypeManager docTypeMgr = new DocumentTypeManager();
+    private List<Search> searchList = new LinkedList<>();
+    private ApplicationPackage app = null;
+    private boolean isBuilt = false;
+    private DocumentModel model = new DocumentModel();
+    private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+
+    public SearchBuilder() {
+        this.app = MockApplicationPackage.createEmpty();
+    }
+
+    public SearchBuilder(ApplicationPackage app) {
+        this.app = app;
+    }
+
+    public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) {
+        this.app = app;
+        this.rankProfileRegistry = rankProfileRegistry;
+    }
+
+    public SearchBuilder(RankProfileRegistry rankProfileRegistry) {
+        this(MockApplicationPackage.createEmpty(), rankProfileRegistry);
+    }
+
+    /**
+     * Import search definition.
+     *
+     * @param fileName The name of the file to import.
+     * @param deployLogger Logger for deploy messages.
+     * @return The name of the imported object.
+     * @throws IOException    Thrown if the file can not be read for some reason.
+     * @throws ParseException Thrown if the file does not contain a valid search definition.       ```
+     */
+    public String importFile(String fileName, DeployLogger deployLogger) throws IOException, ParseException {
+        File file = new File(fileName);
+        return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent(), deployLogger);
+    }
+
+    /**
+     * Import search definition.
+     *
+     * @param fileName The name of the file to import.
+     * @return The name of the imported object.
+     * @throws IOException    Thrown if the file can not be read for some reason.
+     * @throws ParseException Thrown if the file does not contain a valid search definition.
+     */
+    public String importFile(String fileName) throws IOException, ParseException {
+        return importFile(fileName, new BaseDeployLogger());
+    }
+    public String importFile(Path file) throws IOException, ParseException {
+        return importFile(file.toString(), new BaseDeployLogger());
+    }
+
+    /**
+     * Reads and parses the search definition string provided by the given reader. Once all search definitions have been
+     * imported, call {@link #build()}.
+     *
+     * @param reader       The reader whose content to import.
+     * @param searchDefDir The path to use when resolving file references.
+     * @return The name of the imported object.
+     * @throws ParseException Thrown if the file does not contain a valid search definition.
+     */
+    public String importReader(NamedReader reader, String searchDefDir, DeployLogger deployLogger) throws IOException, ParseException {
+        return importString(IOUtils.readAll(reader), searchDefDir, deployLogger);
+    }
+
+    /**
+     * See #{@link #importReader}
+     *
+     * Convenience, should only be used for testing as logs will be swallowed.
+     */
+    public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException {
+        return importString(IOUtils.readAll(reader), searchDefDir, new BaseDeployLogger());
+    }
+
+    /**
+     * Import search definition.
+     *
+     * @param str the string to parse.
+     * @return the name of the imported object.
+     * @throws ParseException thrown if the file does not contain a valid search definition.
+     */
+    public String importString(String str) throws ParseException {
+        return importString(str, null, new BaseDeployLogger());
+    }
+
+    private String importString(String str, String searchDefDir, DeployLogger deployLogger) throws ParseException {
+        Search search;
+        SimpleCharStream stream = new SimpleCharStream(str);
+        try {
+            search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir);
+        } catch (TokenMgrError e) {
+            throw new ParseException("Unknown symbol: " + e.getMessage());
+        } catch (ParseException pe) {
+            throw new ParseException(stream.formatException(pe.getMessage()));
+        }
+        return importRawSearch(search);
+    }
+
+    /**
+     * Registers the given search object to the internal list of objects to be processed during {@link #build()}. A
+     * {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most
+     * programmatically constructed search objects used in unit tests.
+     *
+     * @param rawSearch The object to import.
+     * @return The name of the imported object.
+     * @throws IllegalArgumentException Thrown if the given search object has already been processed.
+     */
+    public String importRawSearch(Search rawSearch) {
+        if (rawSearch.getName() == null) {
+            throw new IllegalArgumentException("Search has no name.");
+        }
+        String rawName = rawSearch.getName();
+        if (rawSearch.isProcessed()) {
+            throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+                    "' has already been processed.");
+        }
+        for (Search search : searchList) {
+            if (rawName.equals(search.getName())) {
+                throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+                        "' has already been added.");
+            }
+        }
+        searchList.add(rawSearch);
+        return rawName;
+    }
+
+    /**
+     * Registers the given search object to the internal list of objects to be processed during {@link #build()}. A
+     * {@link Search} object is considered to be "processed" if it has not already been processed. This is the case for most
+     * programmatically constructed search objects used in unit tests.
+     *
+     * @param processed The object to import.
+     * @return The name of the imported object.
+     * @throws IllegalArgumentException Thrown if the given search object has already been processed.
+     */
+    public String importProcessedSearch(Search processed) {
+        if (processed.getName() == null) {
+            throw new IllegalArgumentException("Search has no name.");
+        }
+        String rawName = processed.getName();
+        if (!processed.isProcessed()) {
+            throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+                    "' has not been processed.");
+        }
+        for (Search search : searchList) {
+            if (rawName.equals(search.getName())) {
+                throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+                        "' has already been added.");
+            }
+        }
+        searchList.add(processed);
+        return rawName;
+    }
+
+    /**
+     * Only for testing.
+     *
+     * Processes and finalizes the imported search definitions so that they become available through the {@link
+     * #getSearch(String)} method.
+     *
+     * @throws IllegalStateException Thrown if this method has already been called.
+     */
+    public void build() {
+        build(new BaseDeployLogger(), new QueryProfiles());
+    }
+
+    /**
+     * Processes and finalizes the imported search definitions so that they become available through the {@link
+     * #getSearch(String)} method.
+     *
+     * @throws IllegalStateException Thrown if this method has already been called.
+     * @param deployLogger The logger to use during build
+     * @param queryProfiles The query profiles contained in the application this search is part of.
+     */
+    public void build(DeployLogger deployLogger, QueryProfiles queryProfiles) {
+        if (isBuilt) {
+            throw new IllegalStateException("Searches already built.");
+        }
+        List<Search> built = new ArrayList<>();
+        List<SDDocumentType> sdocs = new ArrayList<>();
+        sdocs.add(SDDocumentType.VESPA_DOCUMENT);
+        for (Search search : searchList) {
+            if (search.hasDocument()) {
+                sdocs.add(search.getDocument());
+            }
+        }
+        SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
+        orderer.process();
+        for (SDDocumentType sdoc : orderer.getOrdered()) {
+            new FieldOperationApplierForStructs().process(sdoc);
+            new FieldOperationApplier().process(sdoc);
+        }
+
+        DocumentModelBuilder builder = new DocumentModelBuilder(model);
+        for (Search search : new SearchOrderer().order(searchList)) {
+            new FieldOperationApplierForSearch().process(search);
+            // These two needed for a couple of old unit tests, ideally these are just read from app
+            process(search, deployLogger, queryProfiles);
+            built.add(search);
+        }
+        builder.addToModel(searchList);
+        if ( ! builder.valid() ) {
+            throw new IllegalArgumentException("Impossible to build a correct model.");
+        }
+        searchList = built;
+        isBuilt = true;
+    }
+    
+    /**
+     * Processes and returns the given {@link Search} object. This method has been factored out of the {@link
+     * #build()} method so that subclasses can choose not to build anything.
+     *
+     * @param search The object to build.
+     */
+    protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) {
+        Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    /**
+     * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object
+     * built. This method will never return null.
+     *
+     * @return The build object.
+     * @throws IllegalStateException Thrown if there is not exactly one search.
+     */
+    public Search getSearch() {
+        if ( ! isBuilt)  throw new IllegalStateException("Searches not built.");
+        if (searchList.size() != 1)
+            throw new IllegalStateException("This call only works if we have 1 search definition. Search definitions: " + searchList);
+
+        return searchList.get(0);
+    }
+
+    public DocumentModel getModel() {
+        return model;
+    }
+
+    /**
+     * Returns the built {@link Search} object that has the given name. If the name is unknown, this method will simply
+     * return null.
+     *
+     * @param name the name of the search definition to return,
+     *             or null to return the only one or throw an exception if there are multiple to choose from
+     * @return the built object, or null if none with this name
+     * @throws IllegalStateException if {@link #build()} has not been called.
+     */
+    public Search getSearch(String name) {
+        if ( ! isBuilt)  throw new IllegalStateException("Searches not built.");
+        if (name == null) return getSearch();
+
+        for (Search search : searchList)
+            if (search.getName().equals(name)) return search;
+        return null;
+    }
+
+    /**
+     * Convenience method to return a list of all built {@link Search} objects.
+     *
+     * @return The list of built searches.
+     */
+    public List<Search> getSearchList() {
+        return new ArrayList<>(searchList);
+    }
+
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a string.
+     *
+     * @param sd   The string to build from.
+     * @return The built {@link SearchBuilder} object.
+     * @throws ParseException Thrown if there was a problem parsing the string.
+     */
+    public static SearchBuilder createFromString(String sd) throws ParseException {
+        SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty());
+        builder.importString(sd);
+        builder.build();
+        return builder;
+    }
+    
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a file. Only for testing.
+     *
+     * @param fileName the file to build from
+     * @return the built {@link SearchBuilder} object
+     * @throws IOException    if there was a problem reading the file.
+     * @throws ParseException if there was a problem parsing the file content.
+     */
+    public static SearchBuilder createFromFile(String fileName) throws IOException, ParseException {
+        return createFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry());
+    }
+
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a file.
+     *
+     * @param fileName the file to build from.
+     * @param deployLogger logger for deploy messages.
+     * @param rankProfileRegistry registry for rank profiles.
+     * @return the built {@link SearchBuilder} object.
+     * @throws IOException    if there was a problem reading the file.
+     * @throws ParseException if there was a problem parsing the file content.
+     */
+    public static SearchBuilder createFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry)
+            throws IOException, ParseException {
+        SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), rankProfileRegistry);
+        builder.importFile(fileName);
+        builder.build(deployLogger, new QueryProfiles());
+        return builder;
+    }
+
+    public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException {
+        return createFromDirectory(dir, new RankProfileRegistry());
+    }
+    public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException {
+        SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry);
+        for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) {
+            builder.importFile(i.next());
+        }
+        builder.build(new BaseDeployLogger(), new QueryProfiles());
+        return builder;
+    }
+
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a file. Only for testing.
+     *
+     * @param fileName The file to build from.
+     * @return The built {@link Search} object.
+     * @throws IOException    Thrown if there was a problem reading the file.
+     * @throws ParseException Thrown if there was a problem parsing the file content.
+     */
+    public static Search buildFromFile(String fileName) throws IOException, ParseException {
+        return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry());
+    }
+
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a file.
+     *
+     * @param fileName The file to build from.
+     * @param rankProfileRegistry Registry for rank profiles.
+     * @return The built {@link Search} object.
+     * @throws IOException    Thrown if there was a problem reading the file.
+     * @throws ParseException Thrown if there was a problem parsing the file content.
+     */
+    public static Search buildFromFile(String fileName, RankProfileRegistry rankProfileRegistry)
+            throws IOException, ParseException {
+        return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry);
+    }
+
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a file.
+     *
+     * @param fileName The file to build from.
+     * @param deployLogger Logger for deploy messages.
+     * @param rankProfileRegistry Registry for rank profiles.
+     * @return The built {@link Search} object.
+     * @throws IOException    Thrown if there was a problem reading the file.
+     * @throws ParseException Thrown if there was a problem parsing the file content.
+     */
+    public static Search buildFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry)
+            throws IOException, ParseException {
+        return createFromFile(fileName, deployLogger, rankProfileRegistry).getSearch();
+    }
+    
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a raw object.
+     *
+     * @param rawSearch The raw object to build from.
+     * @return The built {@link SearchBuilder} object.
+     * @see #importRawSearch(Search)
+     */
+    public static SearchBuilder createFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) {
+        SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
+        builder.importRawSearch(rawSearch);
+        builder.build();
+        return builder;
+    }
+
+    /**
+     * Convenience factory method to import and build a {@link Search} object from a raw object.
+     *
+     * @param rawSearch The raw object to build from.
+     * @return The built {@link Search} object.
+     * @see #importRawSearch(Search)
+     */
+    public static Search buildFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) {
+        return createFromRawSearch(rawSearch, rankProfileRegistry).getSearch();
+    }
+
+    public RankProfileRegistry getRankProfileRegistry() {
+        return rankProfileRegistry;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java
new file mode 100644
index 00000000000..3df0c1f9953
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.io.IOException;
+
+/**
+ * A SearchBuilder that does not run the processing chain for searches
+ *
+ */
+public class UnprocessingSearchBuilder extends SearchBuilder {
+
+    public UnprocessingSearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) {
+        super(app, rankProfileRegistry);
+    }
+
+    public UnprocessingSearchBuilder() {
+        super();
+    }
+
+    public UnprocessingSearchBuilder(RankProfileRegistry rankProfileRegistry) {
+        super(rankProfileRegistry);
+    }
+
+    @Override
+    public void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) {
+        // empty
+    }
+
+    public static Search buildUnprocessedFromFile(String fileName) throws IOException, ParseException {
+        SearchBuilder builder = new UnprocessingSearchBuilder();
+        builder.importFile(fileName);
+        builder.build();
+        return builder.getSearch();
+    }
+}
\ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java
new file mode 100644
index 00000000000..beab254bd89
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+
+/**
+ * A search that was derived from an sd file containing no search element(s), only
+ * document specifications.
+ *
+ * @author vegardh
+ *
+ */
+ // Award for best class name goes to ...
+public class UnproperSearch extends Search {
+    // This class exists because the parser accepts SD files without search { ... , and
+    // there are unit tests using it too, BUT there are many nullpointer bugs if you try to
+    // deploy such a file. Using this class to try to catch those.
+    // TODO: Throw away this when we properly support doc-only SD files.
+
+    public UnproperSearch() {
+        // empty
+    }
+
+    @Override
+    public void addDocument(SDDocumentType docType) {
+        if (getName() == null) {
+            setName(docType.getName());
+        }
+        super.addDocument(docType);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java
new file mode 100644
index 00000000000..f465112fbaf
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.parser.ParseException;
+
+/**
+ * A low-cost ranking profile to use for watcher queries etc.
+ *
+ * @author <a href="mailto:vegardh@yahoo-inc.com">Vegard Havdal</a>
+ */
+public class UnrankedRankProfile extends RankProfile {
+
+    public UnrankedRankProfile(Search search, RankProfileRegistry rankProfileRegistry) {
+        super("unranked", search, rankProfileRegistry);
+        try {
+            RankingExpression exp = new RankingExpression("value(0)");
+            this.setFirstPhaseRanking(exp);
+        } catch (ParseException e) {
+            throw new IllegalArgumentException("Could not parse the ranking expression 'value(0)' when setting up " +
+                    "the 'unranked' rank profile");
+        }
+        this.setIgnoreDefaultRankFeatures(true);
+        this.setKeepRankCount(0);
+        this.setRerankCount(0);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
new file mode 100644
index 00000000000..1130e5630a3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -0,0 +1,157 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.Ranking;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Sorting;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression;
+
+import java.util.*;
+
+/**
+ * The set of all attribute fields defined by a search definition
+ *
+ * @author <a href="mailto:bratseth@overture.com">Jon S Bratseth</a>
+ */
+public class AttributeFields extends Derived implements AttributesConfig.Producer {
+    private Map<String, Attribute> attributes = new java.util.LinkedHashMap<>();
+
+    /**
+     * Flag indicating if a position-attribute has been found
+     */
+    private boolean hasPosition = false;
+
+    public AttributeFields(Search search) {
+        derive(search);
+    }
+
+    /**
+     * Derives everything from a field
+     */
+    protected void derive(SDField field, Search search) {
+        if (field.usesStructOrMap() &&
+            !field.getDataType().equals(PositionDataType.INSTANCE) &&
+            !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
+        {
+            return; // Ignore struct fields for indexed search (only implemented for streaming search)
+        }
+        deriveAttributes(field);
+    }
+
+    /**
+     * Return an attribute by name, or null if it doesn't exist
+     */
+    public Attribute getAttribute(String attributeName) {
+        return attributes.get(attributeName);
+    }
+
+    public boolean containsAttribute(String attributeName) {
+        return getAttribute(attributeName) != null;
+    }
+
+    /**
+     * Derives one attribute. TODO: Support non-default named attributes
+     */
+    private void deriveAttributes(SDField field) {
+        for (Attribute fieldAttribute : field.getAttributes().values()) {
+            Attribute attribute = getAttribute(fieldAttribute.getName());
+            if (attribute == null) {
+                attributes.put(fieldAttribute.getName(), fieldAttribute);
+                attribute = getAttribute(fieldAttribute.getName());
+            }
+            Ranking ranking = field.getRanking();
+            if (ranking != null && ranking.isFilter()) {
+                attribute.setEnableBitVectors(true);
+                attribute.setEnableOnlyBitVector(true);
+            }
+        }
+
+        if (field.containsExpression(ToPositionExpression.class)) {
+            // TODO: Move this check to processing and remove this
+            if (hasPosition) {
+                throw new IllegalArgumentException("Can not specify more than one " +
+                                                   "set of position attributes per " + "field: " + field.getName());
+            }
+            hasPosition = true;
+        }
+    }
+
+    /**
+     * Returns a read only attribute iterator
+     */
+    public Iterator attributeIterator() {
+        return attributes().iterator();
+    }
+
+    public Collection<Attribute> attributes() {
+        return Collections.unmodifiableCollection(attributes.values());
+    }
+
+    public String toString() {
+        return "attributes " + getName();
+    }
+
+    protected String getDerivedName() {
+        return "attributes";
+    }
+
+    private Map<String, AttributesConfig.Attribute.Builder> toMap(List<AttributesConfig.Attribute.Builder> ls) {
+        Map<String, AttributesConfig.Attribute.Builder> ret = new LinkedHashMap<>();
+        for (AttributesConfig.Attribute.Builder builder : ls) {
+            ret.put((String) ConfigInstanceUtil.getField(builder, "name"), builder);
+        }
+        return ret;
+    }
+
+    @Override
+    public void getConfig(AttributesConfig.Builder builder) {
+        for (Attribute attribute : attributes.values()) {
+            AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder()
+                .name(attribute.getName())
+                .datatype(AttributesConfig.Attribute.Datatype.Enum.valueOf(attribute.getType().getExportAttributeTypeName()))
+                .collectiontype(AttributesConfig.Attribute.Collectiontype.Enum.valueOf(attribute.getCollectionType().getName()));
+            if (attribute.isRemoveIfZero()) {
+                aaB.removeifzero(true);
+            }
+            if (attribute.isCreateIfNonExistent()) {
+                aaB.createifnonexistent(true);
+            }
+            aaB.enablebitvectors(attribute.isEnabledBitVectors());
+            aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector());
+            if (attribute.isFastSearch()) {
+                aaB.fastsearch(true);
+            }
+            if (attribute.isFastAccess()) {
+                aaB.fastaccess(true);
+            }
+            if (attribute.isHuge()) {
+                aaB.huge(true);
+            }
+            if (attribute.getSorting().isDescending()) {
+                aaB.sortascending(false);
+            }
+            if (attribute.getSorting().getFunction() != Sorting.Function.UCA) {
+                aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString()));
+            }
+            if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) {
+                aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString()));
+            }
+            if (!attribute.getSorting().getLocale().isEmpty()) {
+                aaB.sortlocale(attribute.getSorting().getLocale());
+            }
+            aaB.arity(attribute.arity());
+            aaB.lowerbound(attribute.lowerBound());
+            aaB.upperbound(attribute.upperBound());
+            aaB.densepostinglistthreshold(attribute.densePostingListThreshold());
+            if (attribute.tensorType().isPresent()) {
+                aaB.tensortype(attribute.tensorType().get().toString());
+            }
+            builder.attribute(aaB);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
new file mode 100644
index 00000000000..643eeb23c87
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java
@@ -0,0 +1,136 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.ConfigInstance.Builder;
+import com.yahoo.io.IOUtils;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.text.StringUtilities;
+import java.io.IOException;
+import java.io.Writer;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
+
+/**
+ * Superclass of all derived configurations
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public abstract class Derived implements Exportable {
+
+    private String name;
+
+    public String getName() { return name; }
+
+    protected final void setName(String name) { this.name=name; }
+
+    /**
+     * Derives the content of this configuration. This
+     * default calls derive(Document) for each document
+     * and derive(SDField) for each search definition level field
+     * AND sets the name of this to the name of the input search definition
+     */
+    protected void derive(Search search) {
+        setName(search.getName());
+        derive(search.getDocument(), search);
+        for (Index index : search.getExplicitIndices()) {
+            derive(index, search);
+        }
+        for (SDField field : search.allExtraFields() ) {
+            derive(field,search);
+        }
+    }
+
+
+    /**
+     * Derives the content of this configuration. This
+     * default calls derive(SDField) for each document field
+     */
+    protected void derive(SDDocumentType document,Search search) {
+        for (Field field : document.fieldSet()) {
+            SDField sdField = (SDField) field;
+            if (!sdField.isExtraField()) {
+                derive(sdField, search);
+            }
+        }
+    }
+
+    /**
+     * Derives the content of this configuration. This
+     * default does nothing.
+     */
+    protected void derive(SDField field,Search search) {}
+
+    /**
+     * Derives the content of this configuration. This
+     * default does nothing.
+     */
+    protected void derive(Index index, Search search) {
+    }
+
+    protected abstract String getDerivedName();
+
+    /** Returns the value of getName if true, the given number as a string otherwise */
+    protected String getIndex(int number,boolean labels) {
+        return labels ? getName() : String.valueOf(number);
+    }
+
+    /**
+     * Exports this derived configuration to its .cfg file
+     * in toDirectory
+     *
+     * @param toDirectory the directory to export to, or null
+     *
+     */
+    public final void export(String toDirectory)
+            throws IOException {
+        Writer writer=null;
+        try {
+            String fileName=getDerivedName() + ".cfg";
+            if (toDirectory!=null) writer=IOUtils.createWriter(toDirectory + "/" + fileName,false);
+            try {
+                exportBuilderConfig(writer);
+            } catch (ClassNotFoundException | InstantiationException
+                    | IllegalAccessException | NoSuchMethodException
+                    | SecurityException | IllegalArgumentException
+                    | InvocationTargetException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        finally {
+            if (writer!=null) IOUtils.closeWriter(writer);
+        }
+    }
+
+    /**
+     * Checks what this is a producer of, instantiate that and export to writer
+     */
+    // TODO move to ReflectionUtil, and move that to unexported pkg
+    private void exportBuilderConfig(Writer writer) throws ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, SecurityException, IllegalArgumentException, InvocationTargetException, IOException {
+        for (Class<?> intf : getClass().getInterfaces()) {
+            if (ConfigInstance.Producer.class.isAssignableFrom(intf)) {
+                Class<?> configClass = intf.getEnclosingClass();
+                String builderClassName = configClass.getCanonicalName()+"$Builder";
+                Class<?> builderClass = Class.forName(builderClassName);
+                ConfigInstance.Builder builder = (Builder) builderClass.newInstance();
+                Method getConfig = getClass().getMethod("getConfig", builderClass);
+                getConfig.invoke(this, builder);
+                ConfigInstance inst = (ConfigInstance) configClass.getConstructor(builderClass).newInstance(builder);
+                List<String> payloadL = ConfigInstance.serialize(inst);
+                String payload = StringUtilities.implodeMultiline(payloadL);
+                writer.write(payload);
+            }
+        }
+    }
+
+    @Override
+    public String getFileName() {
+        return getDerivedName() + ".cfg";
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
new file mode 100644
index 00000000000..973e4e6100c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -0,0 +1,184 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.io.IOUtils;
+import com.yahoo.protect.Validator;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.validation.Validation;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.List;
+
+/**
+ * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when
+ * necessary.
+ *
+ * @author bratseth
+ */
+public class DerivedConfiguration {
+
+    private Search search;
+    private Summaries summaries;
+    private SummaryMap summaryMap;
+    private Juniperrc juniperrc;
+    private AttributeFields attributeFields;
+    private RankProfileList rankProfileList;
+    private IndexingScript indexingScript;
+    private IndexInfo indexInfo;
+    private VsmFields streamingFields;
+    private VsmSummary streamingSummary;
+    private IndexSchema indexSchema;
+
+    /**
+     * Creates a complete derived configuration from a search definition.
+     *
+     * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is
+     *               live. Which means that this object will be inconsistent when the given search definition is later
+     *               modified.
+     * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
+     */
+    public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry) {
+        this(search, null, new BaseDeployLogger(), rankProfileRegistry);
+    }
+
+    /**
+     * Creates a complete derived configuration snapshot from a search definition.
+     *
+     * @param search             The search to derive a configuration from. Derived objects will be snapshots, but this
+     *                           argument is live. Which means that this object will be inconsistent when the given
+     *                           search definition is later modified.
+     * @param abstractSearchList Search definition this one inherits from, only superclass configuration should be
+     *                           generated. Null or empty list if there is none.
+     * @param deployLogger       a {@link DeployLogger} for logging when
+     *                           doing operations on this
+     * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
+     */
+    public DerivedConfiguration(Search search, List<Search> abstractSearchList, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) {
+        Validator.ensureNotNull("Search definition", search);
+        if (!search.isProcessed()) {
+            throw new IllegalArgumentException("Search '" + search.getName() + "' not processed.");
+        }
+        this.search = search;
+        if (!search.isDocumentsOnly()) {
+            streamingFields = new VsmFields(search);
+            streamingSummary = new VsmSummary(search);
+        }
+        if (abstractSearchList != null) {
+            for (Search abstractSearch : abstractSearchList) {
+                if (!abstractSearch.isProcessed()) {
+                    throw new IllegalArgumentException("Search '" + search.getName() + "' not processed.");
+                }
+            }
+        }
+        if (!search.isDocumentsOnly()) {
+            summaries = new Summaries(search, deployLogger);
+            summaryMap = new SummaryMap(search, summaries);
+            juniperrc = new Juniperrc(search);
+            attributeFields = new AttributeFields(search);
+            rankProfileList = new RankProfileList(search, attributeFields, rankProfileRegistry);
+            indexingScript = new IndexingScript(search);
+            indexInfo = new IndexInfo(search);
+            indexSchema = new IndexSchema(search);
+        }
+        Validation.validate(this, search);
+    }
+
+    /**
+     * Exports a complete set of configuration-server format config files.
+     *
+     * @param toDirectory  the directory to export to, current dir if null
+     * @throws IOException if exporting fails, some files may still be created
+     */
+    public void export(String toDirectory) throws IOException {
+        if (!search.isDocumentsOnly()) {
+            summaries.export(toDirectory);
+            summaryMap.export(toDirectory);
+            juniperrc.export(toDirectory);
+            attributeFields.export(toDirectory);
+            streamingFields.export(toDirectory);
+            streamingSummary.export(toDirectory);
+            indexSchema.export(toDirectory);
+            rankProfileList.export(toDirectory);
+            indexingScript.export(toDirectory);
+            indexInfo.export(toDirectory);
+        }
+    }
+
+    public static void exportDocuments(DocumentmanagerConfig.Builder documentManagerCfg, String toDirectory) throws IOException {
+        exportCfg(new DocumentmanagerConfig(documentManagerCfg), toDirectory + "/" + "documentmanager.cfg");
+    }
+
+    public static void exportDocuments(DocumenttypesConfig.Builder documentTypesCfg, String toDirectory) throws IOException {
+        exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg");
+    }
+
+    private static void exportCfg(ConfigInstance instance, String fileName) throws IOException {
+        Writer writer = null;
+        try {
+            writer = IOUtils.createWriter(fileName, false);
+            if (writer != null) {
+                writer.write(instance.toString());
+                writer.write("\n");
+            }
+        } finally {
+            if (writer != null) {
+                IOUtils.closeWriter(writer);
+            }
+        }
+    }
+
+    public Summaries getSummaries() {
+        return summaries;
+    }
+
+    public AttributeFields getAttributeFields() {
+        return attributeFields;
+    }
+
+    public IndexingScript getIndexingScript() {
+        return indexingScript;
+    }
+
+    public IndexInfo getIndexInfo() {
+        return indexInfo;
+    }
+
+    public void setIndexingScript(IndexingScript script) {
+        this.indexingScript = script;
+    }
+
+    public Search getSearch() {
+        return search;
+    }
+    
+    public RankProfileList getRankProfileList() {
+        return rankProfileList;
+    }
+    
+    public VsmSummary getVsmSummary() {
+        return streamingSummary;
+    }
+    
+    public VsmFields getVsmFields() {
+        return streamingFields;
+    }
+
+    public IndexSchema getIndexSchema() {
+        return indexSchema;
+    }
+
+    public Juniperrc getJuniperrc() {
+        return juniperrc;
+    }
+    
+    public SummaryMap getSummaryMap() {
+        return summaryMap;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java
new file mode 100644
index 00000000000..76bf4492788
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.UnprocessingSearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Auxiliary facade for deriving configs from search definitions
+ *
+ * @author bratseth
+ */
+public class Deriver {
+
+    /**
+     * Derives only document manager.
+     *
+     *
+     * @param sdFileNames The name of the search definition files to derive from.
+     * @param toDir       The directory to write configuration to.
+     * @return The list of Search objects, possibly "unproper ones", from sd files containing only document
+     */
+    public static SearchBuilder deriveDocuments(List<String> sdFileNames, String toDir) {
+        SearchBuilder builder = getUnprocessingSearchBuilder(sdFileNames);
+        DocumentmanagerConfig.Builder documentManagerCfg = new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder());
+        try {
+            DerivedConfiguration.exportDocuments(documentManagerCfg, toDir);
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+        return builder;
+    }
+
+    public static SearchBuilder getSearchBuilder(List<String> sds) {
+        SearchBuilder builder = new SearchBuilder();
+        try {
+            for (String s : sds) {
+                builder.importFile(s);
+            }
+        } catch (ParseException | IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+        builder.build();
+        return builder;
+    }
+
+    public static SearchBuilder getUnprocessingSearchBuilder(List<String> sds) {
+        SearchBuilder builder = new UnprocessingSearchBuilder();
+        try {
+            for (String s : sds) {
+                builder.importFile(s);
+            }
+        } catch (ParseException | IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+        builder.build();
+        return builder;
+    }
+
+    public static DocumentmanagerConfig.Builder getDocumentManagerConfig(String sd) {
+        return getDocumentManagerConfig(Collections.singletonList(sd));
+    }
+
+    public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds) {
+        return new DocumentManager().produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder());
+    }
+
+    public static DocumenttypesConfig.Builder getDocumentTypesConfig(String sd) {
+        return getDocumentTypesConfig(Collections.singletonList(sd));
+    }
+
+    public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> sds) {
+        return new DocumentTypes().produce(getSearchBuilder(sds).getModel(), new DocumenttypesConfig.Builder());
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java
new file mode 100644
index 00000000000..bcc295b91e0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+/**
+ * Classes exportable to configurations
+ *
+ * @author  <a href="mailto:bratseth@overture.com">bratseth</a>
+ */
+public interface Exportable {
+
+    /**
+     * Exports the configuration of this object
+     *
+     *
+     * @param  toDirectory the directory to export to, does not write to disk if null
+     * @throws java.io.IOException if exporting fails, some files may still be created
+     */
+    public void export(String toDirectory) throws java.io.IOException;
+
+    /**
+     * The (short) name of the exported file
+     * @return a String with the (short) name of the exported file
+     */
+    public String getFileName();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java
new file mode 100644
index 00000000000..9dfc1bc2af7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * The rank settings of a field used for native rank features.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class FieldRankSettings {
+
+    private static final Logger logger = Logger.getLogger(FieldRankSettings.class.getName());
+
+    private String fieldName;
+
+    private final Map<String, NativeTable> tables = new LinkedHashMap<>();
+
+    public FieldRankSettings(String fieldName) {
+        this.fieldName = fieldName;
+    }
+
+    public void addTable(NativeTable table) {
+        NativeTable existing = tables.get(table.getType().getName());
+        if (existing != null) {
+            logger.info("Using already specified rank table " + existing + " for field " + fieldName + ", not " + table);
+            return;
+        }
+        tables.put(table.getType().getName(), table);
+    }
+
+    public static boolean isIndexFieldTable(NativeTable table) {
+        return isFieldMatchTable(table) || isProximityTable(table);
+    }
+
+    public static boolean isAttributeFieldTable(NativeTable table) {
+        return isAttributeMatchTable(table);
+    }
+
+    private static boolean isFieldMatchTable(NativeTable table) {
+        return (table.getType().equals(NativeTable.Type.FIRST_OCCURRENCE) ||
+                table.getType().equals(NativeTable.Type.OCCURRENCE_COUNT));
+    }
+
+    private static boolean isAttributeMatchTable(NativeTable table) {
+        return (table.getType().equals(NativeTable.Type.WEIGHT));
+    }
+
+    private static boolean isProximityTable(NativeTable table) {
+        return (table.getType().equals(NativeTable.Type.PROXIMITY) ||
+                table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY));
+    }
+
+    public Map<String,String> deriveRankProperties(int part) {
+        Map<String,String> ret = new LinkedHashMap<>();
+        int i = part;
+        for (Iterator<NativeTable> itr = tables.values().iterator(); itr.hasNext(); ++i) {
+            NativeTable table = itr.next();
+            if (isFieldMatchTable(table)) {
+                ret.put("nativeFieldMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
+            }
+            if (isAttributeMatchTable(table)) {
+                ret.put("nativeAttributeMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
+            }
+            if (isProximityTable(table)) {
+                ret.put("nativeProximity." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName());
+            }
+        }
+        return ret;
+    }
+
+    public String toString() {
+        return "rank settings of field " + fieldName;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java
new file mode 100644
index 00000000000..d61e57a621e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+
+/**
+ * The result transformation of a named field
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class FieldResultTransform {
+
+    private String fieldName;
+
+    private SummaryTransform transform;
+
+    private String argument;
+
+    public FieldResultTransform(String fieldName,SummaryTransform transform,String argument) {
+        this.fieldName=fieldName;
+        this.transform=transform;
+        this.argument = argument;
+    }
+
+    public String getFieldName() { return fieldName; }
+
+    public SummaryTransform getTransform() { return transform; }
+
+    public void setTransform(SummaryTransform transform) { this.transform=transform; }
+
+    /** Returns the argument of this (used as input to the backend docsum rewriter) */
+    public String getArgument() { return argument; }
+
+    public int hashCode() {
+        return fieldName.hashCode() + 11*transform.hashCode() + 17* argument.hashCode();
+    }
+
+    public boolean equals(Object o) {
+        if (! (o instanceof FieldResultTransform)) return false;
+        FieldResultTransform other=(FieldResultTransform)o;
+
+        return
+            this.fieldName.equals(other.fieldName) &&
+            this.transform.equals(other.transform) &&
+            this.argument.equals(other.argument);
+    }
+
+    public String toString() {
+        String sourceString="";
+        if ( ! argument.equals(fieldName))
+            sourceString=" (argument: " + argument + ")";
+        return "field " + fieldName + ": " + transform + sourceString;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java
new file mode 100644
index 00000000000..cab4176d696
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.document.datatypes.*;
+
+/**
+ * A type of an index structure
+ *
+ * @author bratseth
+ */
+public class Index {
+
+    /** The index type enumeration */
+    public static class Type {
+
+        public static final Type TEXT=new Type("text");
+        public static final Type INT64=new Type("long");
+        public static final Type BOOLEANTREE=new Type("booleantree");
+
+        private String name;
+
+        private Type(String name) {
+            this.name=name;
+        }
+
+        public int hashCode() {
+            return name.hashCode();
+        }
+
+        public String getName() { return name; }
+
+        public boolean equals(Object other) {
+            if ( ! (other instanceof Type)) return false;
+            return this.name.equals(((Type)other).name);
+        }
+
+        public String toString() {
+            return "type: " + name;
+        }
+
+    }
+
+    /** Sets the right index type from a field type */
+    public static Type convertType(DataType fieldType) {
+        FieldValue fval = fieldType.createFieldValue();
+        if (fieldType instanceof NumericDataType) {
+            return Type.INT64;
+        } else if (fval instanceof StringFieldValue) {
+            return Type.TEXT;
+        } else if (fval instanceof Raw) {
+            return Type.BOOLEANTREE;
+        } else if (fval instanceof PredicateFieldValue) {
+            return Type.BOOLEANTREE;
+        } else if (fieldType instanceof CollectionDataType) {
+            return convertType(((CollectionDataType) fieldType).getNestedType());
+        } else {
+            throw new IllegalArgumentException("Don't know which index type to " +
+                    "convert " + fieldType + " to");
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
new file mode 100644
index 00000000000..e98ee662b3a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
@@ -0,0 +1,559 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.*;
+import com.yahoo.searchdefinition.processing.ExactMatch;
+import com.yahoo.searchdefinition.processing.NGramMatch;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.search.config.IndexInfoConfig;
+
+import java.util.*;
+
+/**
+ * Per-index commands which should be applied to queries prior to searching
+ *
+ * @author bratseth
+ */
+public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
+
+    private static final String CMD_ATTRIBUTE = "attribute";
+    private static final String CMD_DEFAULT_POSITION = "default-position";
+    private static final String CMD_DYNTEASER = "dynteaser";
+    private static final String CMD_FULLURL = "fullurl";
+    private static final String CMD_HIGHLIGHT = "highlight";
+    private static final String CMD_INDEX = "index";
+    private static final String CMD_LOWERCASE = "lowercase";
+    private static final String CMD_MATCH_GROUP = "match-group ";
+    private static final String CMD_NORMALIZE = "normalize";
+    private static final String CMD_STEM = "stem";
+    private static final String CMD_URLHOST = "urlhost";
+    private static final String CMD_WORD = "word";
+    private static final String CMD_PLAIN_TOKENS = "plain-tokens";
+    private static final String CMD_MULTIVALUE = "multivalue";
+    private static final String CMD_FAST_SEARCH = "fast-search";
+    private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds";
+    private static final String CMD_NUMERICAL = "numerical";
+    private Set<IndexCommand> commands = new java.util.LinkedHashSet<>();
+    private Map<String, String> aliases = new java.util.LinkedHashMap<>();
+    private Map<String, FieldSet> fieldSets;
+    private Search search;
+
+    public IndexInfo(Search search) {
+        this.fieldSets = search.fieldSets().userFieldSets();
+        addIndexCommand("sddocname", CMD_INDEX);
+        addIndexCommand("sddocname", CMD_WORD);
+        derive(search);
+    }
+
+    protected void derive(Search search) {
+        super.derive(search); // Derive per field
+        this.search = search;
+        // Populate fieldsets with actual field objects, bit late to do that here but
+        for (FieldSet fs : fieldSets.values()) {
+            for (String fieldName : fs.getFieldNames()) {
+                fs.fields().add(search.getField(fieldName));
+            }
+        }
+        // Must follow, because index settings overrides field settings
+        for (Index index : search.getExplicitIndices()) {
+            derive(index, search);
+        }
+
+        // Commands for summary fields
+        // TODO: Move to fieldinfo and implement differently. This is not right
+        for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) {
+            if (summaryField.getTransform().isTeaser()) {
+                addIndexCommand(summaryField.getName(), CMD_DYNTEASER);
+            }
+            if (summaryField.getTransform().isBolded()) {
+                addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT);
+            }
+        }
+    }
+
+    protected void derive(Index index, Search search) {
+        if (index.getMatchGroup().size() > 0) {
+            addIndexCommand(index.getName(), CMD_MATCH_GROUP + toSpaceSeparated(index.getMatchGroup()));
+        }
+    }
+
+    private String toSpaceSeparated(Collection c) {
+        StringBuffer b = new StringBuffer();
+        for (Iterator i = c.iterator(); i.hasNext();) {
+            b.append(i.next());
+            if (i.hasNext()) {
+                b.append(" ");
+            }
+        }
+        return b.toString();
+    }
+
+    protected void derive(SDField field, Search search) {
+        if (field.getDataType().equals(DataType.PREDICATE)) {
+            Index index = field.getIndex(field.getName());
+            if (index != null) {
+                BooleanIndexDefinition options = index.getBooleanIndexDefiniton();
+                if (options.hasLowerBound() || options.hasUpperBound()) {
+                    addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" +
+                            (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." +
+                            (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]");
+                }
+            }
+        }
+
+        // Field level aliases
+        for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
+            String alias = e.getKey();
+            String name = e.getValue();
+            addIndexAlias(alias, name);
+        }
+        if (field.usesStructOrMap()) {
+            for (SDField structField : field.getStructFields()) {
+                derive(structField, search); // Recursion
+            }
+        }
+
+        if (field.getDataType().equals(PositionDataType.INSTANCE) ||
+            field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
+        {
+            addIndexCommand(field.getName(), CMD_DEFAULT_POSITION);
+        }
+
+        addIndexCommand(field, CMD_INDEX); // List the indices
+
+        if (field.doesIndexing() || field.doesLowerCasing()) {
+            addIndexCommand(field, CMD_LOWERCASE);
+        }
+
+        if (field.getDataType().isMultivalue()) {
+            addIndexCommand(field, CMD_MULTIVALUE);
+        }
+
+        if (field.doesAttributing() && !field.doesIndexing()) {
+            addIndexCommand(field.getName(), CMD_ATTRIBUTE);
+            Attribute attribute = field.getAttributes().get(field.getName());
+            if (attribute != null && attribute.isFastSearch())
+                addIndexCommand(field.getName(), CMD_FAST_SEARCH);
+        } else if (field.doesIndexing()) {
+            if (stemSomehow(field, search)) {
+                addIndexCommand(field, stemCmd(field, search), new StemmingOverrider(this, search));
+            }
+            if (normalizeAccents(field)) {
+                addIndexCommand(field, CMD_NORMALIZE);
+            }
+        }
+
+        if (isUriField(field)) {
+            addUriIndexCommands(field);
+        }
+
+        if (field.getDataType() instanceof NumericDataType) {
+            addIndexCommand(field, CMD_NUMERICAL);
+        }
+
+        // Explicit commands
+        for (String command : field.getQueryCommands()) {
+            addIndexCommand(field, command);
+        }
+
+    }
+
+    static String stemCmd(SDField field, Search search) {
+        return CMD_STEM + ":" + field.getStemming(search).toStemMode();
+    }
+
+    private boolean stemSomehow(SDField field, Search search) {
+        if (field.getStemming(search).equals(Stemming.NONE)) return false;
+        return isTypeOrNested(field, DataType.STRING);
+    }
+
+    private boolean normalizeAccents(SDField field) {
+        return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING);
+    }
+
+    private boolean isTypeOrNested(SDField field, DataType type) {
+        return field.getDataType().equals(type) || field.getDataType().equals(DataType.getArray(type)) ||
+               field.getDataType().equals(DataType.getWeightedSet(type));
+    }
+
+    private boolean isUriField(Field field) {
+        DataType fieldType = field.getDataType();
+        if (DataType.URI.equals(fieldType)) {
+            return true;
+        }
+        if (fieldType instanceof CollectionDataType &&
+            DataType.URI.equals(((CollectionDataType)fieldType).getNestedType()))
+        {
+            return true;
+        }
+        return false;
+    }
+
+    private void addUriIndexCommands(SDField field) {
+        String fieldName = field.getName();
+        addIndexCommand(fieldName, CMD_FULLURL);
+        addIndexCommand(fieldName, CMD_LOWERCASE);
+        addIndexCommand(fieldName + "." + fieldName, CMD_FULLURL);
+        addIndexCommand(fieldName + "." + fieldName, CMD_LOWERCASE);
+        addIndexCommand(fieldName + ".path", CMD_FULLURL);
+        addIndexCommand(fieldName + ".path", CMD_LOWERCASE);
+        addIndexCommand(fieldName + ".query", CMD_FULLURL);
+        addIndexCommand(fieldName + ".query", CMD_LOWERCASE);
+        addIndexCommand(fieldName + ".hostname", CMD_URLHOST);
+        addIndexCommand(fieldName + ".hostname", CMD_LOWERCASE);
+
+        // XXX hack
+        Index index = field.getIndex("hostname");
+        if (index != null) {
+            addIndexCommand(index, CMD_URLHOST);
+        }
+    }
+
+    /**
+     * Sets a command for all indices of a field
+     */
+    private void addIndexCommand(Index index, String command) {
+        addIndexCommand(index.getName(), command);
+    }
+
+    /**
+     * Sets a command for all indices of a field
+     */
+    private void addIndexCommand(SDField field, String command) {
+        addIndexCommand(field, command, null);
+    }
+
+    /**
+     * Sets a command for all indices of a field
+     */
+    private void addIndexCommand(SDField field, String command, IndexOverrider overrider) {
+        if (overrider == null || !overrider.override(field.getName(), command, field)) {
+            addIndexCommand(field.getName(), command);
+        }
+    }
+
+    private void addIndexCommand(String indexName, String command) {
+        commands.add(new IndexCommand(indexName, command));
+    }
+
+    private void addIndexAlias(String alias, String indexName) {
+        aliases.put(alias, indexName);
+    }
+
+    /**
+     * Returns whether a particular command is prsent in this index info
+     */
+    public boolean hasCommand(String indexName, String command) {
+        return commands.contains(new IndexCommand(indexName, command));
+    }
+
+    private boolean notInCommands(String index) {
+        for (IndexCommand command : commands) {
+            if (command.getIndex().equals(index)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public void getConfig(IndexInfoConfig.Builder builder) {
+        IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder();
+        iiB.name(getName());
+        for (IndexCommand command : commands) {
+            iiB.command(
+                    new IndexInfoConfig.Indexinfo.Command.Builder()
+                        .indexname(command.getIndex())
+                        .command(command.getCommand()));
+        }
+        // Make user defined field sets searchable
+        for (FieldSet fieldSet : fieldSets.values()) {
+        	 if (notInCommands(fieldSet.getName())) {
+        		 addFieldSetCommands(iiB, fieldSet);
+        	 }
+        }
+
+        for (Map.Entry<String, String> e : aliases.entrySet()) {
+            iiB.alias(
+                    new IndexInfoConfig.Indexinfo.Alias.Builder()
+                        .alias(e.getKey())
+                        .indexname(e.getValue()));
+        }
+        builder.indexinfo(iiB);
+    }
+
+    // TODO: This implementation is completely brain dead.
+    // Move it to the FieldSetValidity processor (and rename it) as that already has to look at this.
+    // Also add more explicit testing of this, e.g to indexinfo_fieldsets in ExportingTestCase. - Jon
+    private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) {
+        // Explicit query commands on the field set, overrides everything.
+        if (!fieldSet.queryCommands().isEmpty()) {
+            for (String qc : fieldSet.queryCommands()) {
+                iiB.command(
+                        new IndexInfoConfig.Indexinfo.Command.Builder()
+                        .indexname(fieldSet.getName())
+                        .command(qc));
+            }
+            return;
+        }        
+        boolean anyIndexing = false;
+        boolean anyAttributing = false;
+        boolean anyLowerCasing = false;
+        boolean anyStemming = false;
+        boolean anyNormalizing = false;
+        String stemmingCommand = null;
+        Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching
+        // First a pass over the fields to read some params to decide field settings implicitly:
+        for (SDField field : fieldSet.fields()) {
+            if (field.doesIndexing()) {
+                anyIndexing = true;
+            }
+            if (field.doesAttributing()) {
+                anyAttributing = true;
+            }
+            if (field.doesIndexing() || field.doesLowerCasing()) {
+                anyLowerCasing = true;
+            }
+            if (stemming(field)) {
+                anyStemming = true;
+                stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode();
+            }
+            if (field.getNormalizing().doRemoveAccents()) {
+                anyNormalizing = true;
+            }
+            if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType)
+                fieldSetMatching = field.getMatching();
+        }
+        if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) {
+            // We have both attributes and indexes and no explicit match setting ->
+            // use default matching as that at least works if the data in the attribute consists
+            // of single tokens only.
+            fieldSetMatching = new Matching();
+        }
+        if (anyLowerCasing) {
+            iiB.command(
+                    new IndexInfoConfig.Indexinfo.Command.Builder()
+                        .indexname(fieldSet.getName())
+                        .command(CMD_LOWERCASE));
+        }
+        if (hasMultiValueField(fieldSet)) {
+            iiB.command(
+                    new IndexInfoConfig.Indexinfo.Command.Builder()
+                            .indexname(fieldSet.getName())
+                            .command(CMD_MULTIVALUE));
+        }
+        if (anyIndexing) {
+            iiB.command(
+                    new IndexInfoConfig.Indexinfo.Command.Builder()
+                        .indexname(fieldSet.getName())
+                        .command(CMD_INDEX));
+            if ( ! isExactMatch(fieldSetMatching)) {
+                if (anyStemming) {
+                    iiB.command(
+                        new IndexInfoConfig.Indexinfo.Command.Builder()
+                            .indexname(fieldSet.getName())
+                            .command(stemmingCommand));
+                }
+                if (anyNormalizing)
+                    iiB.command(
+                            new IndexInfoConfig.Indexinfo.Command.Builder()
+                                .indexname(fieldSet.getName())
+                                .command(CMD_NORMALIZE));
+            }
+        } else {
+            // Assume only attribute fields
+            iiB
+            .command(
+                new IndexInfoConfig.Indexinfo.Command.Builder()
+                    .indexname(fieldSet.getName())
+                    .command(CMD_ATTRIBUTE))
+            .command(
+                new IndexInfoConfig.Indexinfo.Command.Builder()
+                    .indexname(fieldSet.getName())
+                    .command(CMD_INDEX));
+        }
+        if (fieldSetMatching != null) {
+            // Explicit matching set on fieldset
+            if (fieldSetMatching.getType().equals(Matching.Type.EXACT)) {
+                String term = fieldSetMatching.getExactMatchTerminator();
+                if (term==null) term=ExactMatch.DEFAULT_EXACT_TERMINATOR;
+                iiB.command(
+                        new IndexInfoConfig.Indexinfo.Command.Builder()
+                            .indexname(fieldSet.getName())
+                            .command("exact "+term));
+            } else if (fieldSetMatching.getType().equals(Matching.Type.WORD)) {
+                iiB.command(
+                        new IndexInfoConfig.Indexinfo.Command.Builder()
+                            .indexname(fieldSet.getName())
+                            .command(CMD_WORD));
+            } else if (fieldSetMatching.getType().equals(Matching.Type.GRAM)) {
+                iiB.command(
+                        new IndexInfoConfig.Indexinfo.Command.Builder()
+                            .indexname(fieldSet.getName())
+                            .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE)));
+            } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) {
+                
+            }
+            
+        }
+       
+    }
+
+    private boolean hasMultiValueField(FieldSet fieldSet) {
+        for (SDField field : fieldSet.fields()) {
+            if (field.getDataType().isMultivalue())
+                return true;
+        }
+        return false;
+    }
+
+    private Stemming getEffectiveStemming(SDField field) {
+        Stemming active = field.getStemming(search);
+        if (field.getIndex(field.getName()) != null) {
+            if (field.getIndex(field.getName()).getStemming()!=null) {
+                active = field.getIndex(field.getName()).getStemming();
+            }
+        }
+        if (active != null) {
+            return active;
+        }
+        // assume default
+        return Stemming.SHORTEST;
+    }
+
+    private boolean stemming(SDField field) {
+        if (field.getStemming() != null) {
+            return !field.getStemming().equals(Stemming.NONE);
+        }
+        if (search.getStemming()==Stemming.NONE) return false;
+        if (field.getIndex(field.getName())==null) return true;
+        if (field.getIndex(field.getName()).getStemming()==null) return true;
+        return !(field.getIndex(field.getName()).getStemming().equals(Stemming.NONE));
+    }
+
+    private boolean isExactMatch(Matching m) {
+        if (m==null) return false;
+        if (m.getType().equals(Matching.Type.EXACT)) return true;
+        if (m.getType().equals(Matching.Type.WORD)) return true;
+        return false;
+    }
+    
+    /**
+     * Returns a read only iterator over the index commands of this
+     */
+    public Iterator<IndexCommand> indexCommandIterator() {
+        return Collections.unmodifiableSet(commands).iterator();
+    }
+
+    protected String getDerivedName() {
+        return "index-info";
+    }
+
+    /**
+     * An index command. Null commands are also represented, to detect consistency issues. This is an (immutable) value
+     * object.
+     */
+    public static class IndexCommand {
+
+        private String index;
+
+        private String command;
+
+        public IndexCommand(String index, String command) {
+            this.index = index;
+            this.command = command;
+        }
+
+        public String getIndex() {
+            return index;
+        }
+
+        public String getCommand() {
+            return command;
+        }
+
+        /**
+         * Returns true if this is the null command (do nothing)
+         */
+        public boolean isNull() {
+            return command.equals("");
+        }
+
+        public int hashCode() {
+            return index.hashCode() + 17 * command.hashCode();
+        }
+
+        public boolean equals(Object object) {
+            if (!(object instanceof IndexCommand)) {
+                return false;
+            }
+
+            IndexCommand other = (IndexCommand)object;
+            return
+                    other.index.equals(this.index) &&
+                    other.command.equals(this.command);
+        }
+
+        public String toString() {
+            return "index command " + command + " on index " + index;
+        }
+
+    }
+
+    /**
+     * A command which may override the command setting of a field for a particular index
+     */
+    private static abstract class IndexOverrider {
+
+        protected IndexInfo owner;
+
+        public IndexOverrider(IndexInfo owner) {
+            this.owner = owner;
+        }
+
+        /**
+         * Override the setting of this index for this field, returns true if overriden, false if this index should be
+         * set according to the field
+         */
+        public abstract boolean override(String indexName, String command, SDField field);
+
+    }
+
+    private static class StemmingOverrider extends IndexOverrider {
+
+        private Search search;
+
+        public StemmingOverrider(IndexInfo owner, Search search) {
+            super(owner);
+            this.search = search;
+        }
+
+        public boolean override(String indexName, String command, SDField field) {
+            if (search == null) {
+                return false;
+            }
+
+            Index index = search.getIndex(indexName);
+            if (index == null) {
+                return false;
+            }
+
+            Stemming indexStemming = index.getStemming();
+            if (indexStemming == null) {
+                return false;
+            }
+
+            if (Stemming.NONE.equals(indexStemming)) {
+                // Add nothing
+            } else {
+                owner.addIndexCommand(indexName, CMD_STEM + ":" + indexStemming.toStemMode());
+            }
+            return true;
+        }
+
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
new file mode 100644
index 00000000000..05b8a1bf5e7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
@@ -0,0 +1,231 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.config.search.IndexschemaConfig;
+
+import java.util.*;
+
+/**
+ * Deriver of indexschema config containing information of all index fields with name and data type.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
+
+    private final List<IndexField> fields = new ArrayList<>();
+    private final Map<String, FieldCollection> collections = new LinkedHashMap<>();
+    private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>();
+
+    public IndexSchema(Search search) {
+        fieldSets.putAll(search.fieldSets().userFieldSets());
+        derive(search);
+    }
+
+    public boolean containsField(String fieldName) {
+        return fields.stream().anyMatch(field -> field.getName().equals(fieldName));
+    }
+
+    @Override
+    protected void derive(Search search) {
+        super.derive(search);
+    }
+
+    private void deriveIndexFields(SDField field, Search search) {
+        if (!field.doesIndexing() &&
+            !field.isIndexStructureField())
+        {
+            return;
+        }
+        List<Field> lst = flattenField(field);
+        if (lst.isEmpty()) {
+            return;
+        }
+        String fieldName = field.getName();
+        for (Field flatField : lst) {
+            deriveIndexFields(flatField, search);
+        }
+        if (lst.size() > 1) {
+            FieldSet fieldSet = new FieldSet(fieldName);
+            for (Field flatField : lst) {
+                fieldSet.addFieldName(flatField.getName());
+            }
+            fieldSets.put(fieldName, fieldSet);
+        }
+    }
+
+    private void deriveIndexFields(Field field, Search search) {
+        IndexField toAdd = new IndexField(field.getName(), Index.convertType(field.getDataType()), field.getDataType());
+        com.yahoo.searchdefinition.Index definedIndex = search.getIndex(field.getName());
+        if (definedIndex != null) {
+            toAdd.setIndexSettings(definedIndex);
+        }
+        fields.add(toAdd);
+        addFieldToCollection(field.getName(), field.getName()); // implicit
+    }
+
+    private FieldCollection getCollection(String collectionName) {
+        FieldCollection retval = collections.get(collectionName);
+        if (retval == null) {
+            collections.put(collectionName, new FieldCollection(collectionName));
+            return collections.get(collectionName);
+        }
+        return retval;
+    }
+
+    private void addFieldToCollection(String fieldName, String collectionName) {
+        FieldCollection collection = getCollection(collectionName);
+        collection.fields.add(fieldName);
+    }
+
+    @Override
+    protected void derive(SDField field, Search search) {
+        if (field.usesStructOrMap()) {
+            return; // unsupported
+        }
+        deriveIndexFields(field, search);
+    }
+
+    @Override
+    protected String getDerivedName() {
+        return "indexschema";
+    }
+
+    @Override
+    public void getConfig(IndexschemaConfig.Builder icB) {
+        for (int i = 0; i < fields.size(); ++i) {
+            IndexField f = fields.get(i);
+            IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder()
+                .name(f.getName())
+                .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType()))
+                .prefix(f.hasPrefix())
+                .phrases(f.hasPhrases())
+                .positions(f.hasPositions());
+            if (f.getSdType() !=null && !f.getSdType().equals(com.yahoo.searchdefinition.Index.Type.VESPA)) {
+                ifB.indextype(IndexschemaConfig.Indexfield.Indextype.Enum.valueOf(f.getSdType().toString()));
+            }
+            if (!f.getCollectionType().equals("SINGLE")) {
+                ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType()));
+            }
+            icB.indexfield(ifB);
+        }
+        for (FieldSet fieldSet : fieldSets.values()) {
+            IndexschemaConfig.Fieldset.Builder fsB = new IndexschemaConfig.Fieldset.Builder()
+                .name(fieldSet.getName());
+            for (String f : fieldSet.getFieldNames()) {
+                fsB.field(new IndexschemaConfig.Fieldset.Field.Builder()
+                        .name(f));
+            }
+            icB.fieldset(fsB);
+        }
+    }
+
+    static List<Field> flattenField(Field field) {
+        DataType fieldType = field.getDataType();
+        if (fieldType.getPrimitiveType() != null){
+            return Collections.singletonList(field);
+        }
+        if (fieldType instanceof ArrayDataType) {
+            boolean header = field.isHeader();
+            List<Field> ret = new LinkedList<>();
+            Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header);
+            for (Field flatField : flattenField(innerField)) {
+                ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header));
+            }
+            return ret;
+        }
+        if (fieldType instanceof StructuredDataType) {
+            List<Field> ret = new LinkedList<>();
+            String fieldName = field.getName();
+            for (Field childField : ((StructuredDataType)fieldType).getFields()) {
+                for (Field flatField : flattenField(childField)) {
+                    ret.add(new Field(fieldName + "." + flatField.getName(), flatField));
+                }
+            }
+            return ret;
+        }
+        throw new UnsupportedOperationException(fieldType.getName());
+    }
+
+    public List<IndexField> getFields() {
+        return fields;
+    }
+
+    /**
+     * Representation of an index field with name and data type.
+     */
+    public static class IndexField {
+        private String name;
+        private Index.Type type;
+        private com.yahoo.searchdefinition.Index.Type sdType; // The index type in "user intent land"
+        private DataType sdFieldType;
+        private boolean prefix = false;
+        private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility?
+        private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility?
+        private BooleanIndexDefinition boolIndex = null;
+
+        public IndexField(String name, Index.Type type, DataType sdFieldType) {
+            this.name = name;
+            this.type = type;
+            this.sdFieldType = sdFieldType;
+        }
+        public void setIndexSettings(com.yahoo.searchdefinition.Index index) {
+            if (type.equals(Index.Type.TEXT)) {
+                prefix = index.isPrefix();
+            }
+            sdType = index.getType();
+            boolIndex = index.getBooleanIndexDefiniton();
+        }
+        public String getName() { return name; }
+        public Index.Type getRawType() { return type; }
+        public String getType() {
+            return type.equals(Index.Type.INT64)
+                    ? "INT64"
+                    : type.equals(Index.Type.BOOLEANTREE)
+                        ? "BOOLEANTREE"
+                        : "STRING";
+        }
+	    public String getCollectionType() {
+	        return (sdFieldType == null)
+                    ? "SINGLE"
+                    : (sdFieldType instanceof WeightedSetDataType)
+		                ? "WEIGHTEDSET"
+                        : (sdFieldType instanceof ArrayDataType)
+                            ? "ARRAY"
+                            : "SINGLE";
+	    }
+        public boolean hasPrefix() { return prefix; }
+        public boolean hasPhrases() { return phrases; }
+        public boolean hasPositions() { return positions; }
+
+        public BooleanIndexDefinition getBooleanIndexDefinition() {
+            return boolIndex;
+        }
+
+        /**
+         * The user set index type
+         * @return the type
+         */
+        public com.yahoo.searchdefinition.Index.Type getSdType() {
+            return sdType;
+        }
+    }
+
+    /**
+     * Representation of a collection of fields (aka index, physical view).
+     */
+    @SuppressWarnings({ "UnusedDeclaration" })
+    private static class FieldCollection {
+
+        private final String name;
+        private final List<String> fields = new ArrayList<>();
+
+        FieldCollection(String name) {
+            this.name = name;
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
new file mode 100644
index 00000000000..94e4dec567f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
@@ -0,0 +1,146 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.*;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder;
+
+import java.util.*;
+
+/**
+ * An indexing language script derived from a search definition. An indexing script contains a set of indexing
+ * statements, organized in a composite structure of indexing code snippets.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public final class IndexingScript extends Derived implements IlscriptsConfig.Producer {
+
+    private final List<String> docFields = new LinkedList<>();
+    private final List<Expression> expressions = new LinkedList<>();
+
+    public IndexingScript(Search search) {
+        derive(search);
+    }
+
+    protected void derive(SDField field, Search search) {
+        if (!field.isExtraField()) {
+            docFields.add(field.getName());
+        }
+        if (field.usesStructOrMap() &&
+            !field.getDataType().equals(PositionDataType.INSTANCE) &&
+            !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)))
+        {
+            return; // unsupported
+        }
+        ScriptExpression script = field.getIndexingScript();
+        if (!script.isEmpty()) {
+            expressions.add(new StatementExpression(new ClearStateExpression(),
+                                                    new GuardExpression(script)));
+        }
+    }
+
+    public Iterable<Expression> expressions() {
+        return Collections.unmodifiableCollection(expressions);
+    }
+
+    @Override
+    public String getDerivedName() {
+        return "ilscripts";
+    }
+
+    @Override
+    public void getConfig(IlscriptsConfig.Builder configBuilder) {
+        IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder();
+        ilscriptBuilder.doctype(getName());
+        for (String fieldName : docFields) {
+            ilscriptBuilder.docfield(fieldName);
+        }
+        addContentInOrder(ilscriptBuilder);
+        configBuilder.ilscript(ilscriptBuilder);
+    }
+
+    private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) {
+        ArrayList<Expression> later = new ArrayList<>();
+        Set<String> touchedFields = new HashSet<String>();
+        for (Expression exp : expressions) {
+            FieldScanVisitor fieldFetcher = new FieldScanVisitor();
+            if (modifiesSelf(exp)) {
+                later.add(exp);
+            } else {
+              ilscriptBuilder.content(exp.toString());
+            }
+            fieldFetcher.visit(exp);
+            touchedFields.addAll(fieldFetcher.touchedFields());
+        }
+        for (Expression exp : later) {
+            ilscriptBuilder.content(exp.toString());
+        }
+        generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields);
+    }
+
+    private void generateSyntheticStatementsForUntouchedFields(Builder ilscriptBuilder, Set<String> touchedFields) {
+        Set<String> fieldsWithSyntheticStatements = new HashSet<String>(docFields);
+        fieldsWithSyntheticStatements.removeAll(touchedFields);
+        List<String> orderedFields = new ArrayList<String>(fieldsWithSyntheticStatements);
+        Collections.sort(orderedFields);
+        for (String fieldName : orderedFields) {
+            StatementExpression copyField = new StatementExpression(new InputExpression(fieldName),
+                    new PassthroughExpression(fieldName));
+            ilscriptBuilder.content(copyField.toString());
+        }
+    }
+
+    private boolean modifiesSelf(Expression exp) {
+        MyExpVisitor visitor = new MyExpVisitor();
+        visitor.visit(exp);
+        return visitor.modifiesSelf();
+    }
+
+    private class MyExpVisitor extends ExpressionVisitor {
+        private String inputField = null;
+        private String outputField = null;
+
+        public boolean modifiesSelf() { return outputField != null && outputField.equals(inputField); }
+
+        @Override
+        protected void doVisit(Expression expression) {
+            if (modifiesSelf()) {
+                return;
+            }
+            if (expression instanceof InputExpression) {
+                inputField = ((InputExpression) expression).getFieldName();
+            }
+            if (expression instanceof OutputExpression) {
+                outputField = ((OutputExpression) expression).getFieldName();
+            }
+        }
+    }
+
+    private static class FieldScanVisitor extends ExpressionVisitor {
+        List<String> touchedFields = new ArrayList<String>();
+        List<String> candidates = new ArrayList<String>();
+
+        @Override
+        protected void doVisit(Expression exp) {
+            if (exp instanceof OutputExpression) {
+                touchedFields.add(((OutputExpression) exp).getFieldName());
+            }
+            if (exp instanceof InputExpression) {
+                candidates.add(((InputExpression) exp).getFieldName());
+            }
+            if (exp instanceof ZCurveExpression) {
+                touchedFields.addAll(candidates);
+            }
+        }
+
+        Collection<String> touchedFields() {
+            Collection<String> output = touchedFields;
+            touchedFields = null; // deny re-use to try and avoid obvious bugs
+            return output;
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java
new file mode 100644
index 00000000000..9ef0ddbc723
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.config.search.summary.JuniperrcConfig;
+
+import java.util.Set;
+
+/**
+ * Generated juniperrc-config for controlling juniper.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Juniperrc extends Derived implements JuniperrcConfig.Producer {
+
+    // List of all fields that should be bolded.
+    private Set<String> boldingFields = new java.util.LinkedHashSet<>();
+
+    /**
+     * Constructs a new juniper rc instance for a given search object. This will derive the configuration automatically,
+     * so there is no need to call {@link #derive(com.yahoo.searchdefinition.Search)}.
+     *
+     * @param search The search model to use for deriving.
+     */
+    public Juniperrc(Search search) {
+        derive(search);
+    }
+
+    // Inherit doc from Derived.
+    @Override
+    protected void derive(Search search) {
+        super.derive(search);
+        for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) {
+            if (summaryField.getTransform() == SummaryTransform.BOLDED) {
+                boldingFields.add(summaryField.getName());
+            }
+        }
+    }
+
+    // Inherit doc from Derived.
+    protected String getDerivedName() {
+        return "juniperrc";
+    }
+
+    @Override
+    public void getConfig(JuniperrcConfig.Builder builder) {
+        if (boldingFields.size() != 0) {
+            builder.prefix(true);
+            for (String name : boldingFields) {
+                builder.override(new JuniperrcConfig.Override.Builder()
+                    .fieldname(name)
+                    .length(65536)
+                    .max_matches(1)
+                    .min_length(8192)
+                    .surround_max(65536));
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java
new file mode 100644
index 00000000000..c1176807519
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.RankType;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * The definition of a rank type used for native rank features.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class NativeRankTypeDefinition {
+
+    /** The type this defines */
+    private RankType type;
+
+    /** The rank tables of this rank type */
+    private List<NativeTable> rankTables = new java.util.ArrayList<>();
+
+    public NativeRankTypeDefinition(RankType type) {
+        this.type = type;
+    }
+
+    public RankType getType() {
+        return type;
+    }
+
+    public void addTable(NativeTable table) {
+        rankTables.add(table);
+    }
+
+    /** Returns an unmodifiable list of the tables in this type definition */
+    public Iterator<NativeTable> rankSettingIterator() {
+        return Collections.unmodifiableList(rankTables).iterator();
+    }
+
+    public String toString() {
+        return "native definition of rank type '" + type + "'";
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java
new file mode 100644
index 00000000000..18856627b70
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.RankType;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * A set of rank type definitions used for native rank features.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class NativeRankTypeDefinitionSet {
+
+    /** The name of this rank definition set */
+    private String name;
+
+    /** The unmodifiable rank type implementations in this set */
+    private final Map<RankType, NativeRankTypeDefinition> typeDefinitions;
+
+    /** Returns the default rank type (about) */
+    public static RankType getDefaultRankType() { return RankType.ABOUT; }
+
+    public NativeRankTypeDefinitionSet(String name) {
+        this.name = name;
+
+        Map<RankType, NativeRankTypeDefinition> typeDefinitions = new java.util.LinkedHashMap<>();
+        typeDefinitions.put(RankType.IDENTITY, createIdentityRankType(RankType.IDENTITY));
+        typeDefinitions.put(RankType.ABOUT, createAboutRankType(RankType.ABOUT));
+        typeDefinitions.put(RankType.TAGS, createTagsRankType(RankType.TAGS));
+        typeDefinitions.put(RankType.EMPTY, createEmptyRankType(RankType.EMPTY));
+        this.typeDefinitions = Collections.unmodifiableMap(typeDefinitions);
+    }
+
+    private NativeRankTypeDefinition createEmptyRankType(RankType type) {
+        NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+        rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)"));
+        rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)"));
+        rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)"));
+        rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)"));
+        rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)"));
+        return rank;
+    }
+
+    private NativeRankTypeDefinition createAboutRankType(RankType type) {
+        NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+        rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"));
+        rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"));
+        rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"));
+        rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"));
+        rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"));
+        return rank;
+    }
+
+    private NativeRankTypeDefinition createIdentityRankType(RankType type) {
+        NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+        rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)"));
+        rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"));
+        rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)"));
+        rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)"));
+        rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)"));
+        return rank;
+    }
+
+    private NativeRankTypeDefinition createTagsRankType(RankType type) {
+        NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type);
+        rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)"));
+        rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)"));
+        rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)"));
+        rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)"));
+        rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)"));
+        return rank;
+    }
+
+    /**
+     * Returns a rank type definition if given an existing rank type name,
+     * or null if given a rank type which has no native implementation (meaning somebody forgot to add it),
+    */
+    public NativeRankTypeDefinition getRankTypeDefinition(RankType type) {
+        if (type == RankType.DEFAULT)
+            type = getDefaultRankType();
+        return typeDefinitions.get(type);
+    }
+
+    /** Returns an unmodifiable map of the type definitions in this */
+    public Map<RankType, NativeRankTypeDefinition> types() { return typeDefinitions; }
+
+    public String toString() {
+        return "native rank type definitions " + name;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java
new file mode 100644
index 00000000000..512d4a37647
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+/**
+ * A named rank table of a certain type.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class NativeTable {
+
+    private String name;
+
+    private Type type;
+
+    /** A table type enumeration */
+    public static class Type {
+
+        public static Type FIRST_OCCURRENCE = new Type("firstOccurrenceTable");
+        public static Type OCCURRENCE_COUNT = new Type("occurrenceCountTable");
+        public static Type WEIGHT = new Type("weightTable");
+        public static Type PROXIMITY = new Type("proximityTable");
+        public static Type REVERSE_PROXIMITY = new Type("reverseProximityTable");
+
+        private String name;
+
+        private Type(String name) {
+            this.name = name;
+        }
+
+        public String getName() { return name; }
+
+        public boolean equals(Object object) {
+            if (!(object instanceof Type)) {
+                return false;
+            }
+            Type other = (Type)object;
+            return this.name.equals(other.name);
+        }
+
+        public int hashCode() {
+            return name.hashCode();
+        }
+
+        public String toString() {
+            return getName();
+        }
+    }
+
+    public NativeTable(Type type, String name) {
+        this.type = type;
+        this.name = name;
+    }
+
+    public Type getType() { return type; }
+
+    public String getName() { return name; }
+
+    public int hashCode() {
+        return type.hashCode() + 17*name.hashCode();
+    }
+
+    public boolean equals(Object object) {
+        if (! (object instanceof NativeTable)) return false;
+        NativeTable other = (NativeTable)object;
+        return other.getName().equals(this.getName()) && other.getType().equals(this.getType());
+    }
+
+    public String toString() {
+        return getType() + ": " + getName();
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
new file mode 100644
index 00000000000..183bfd6ddd4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.Search;
+import java.util.Map;
+
+/**
+ * The derived rank profiles of a search definition
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class RankProfileList extends Derived implements RankProfilesConfig.Producer {
+
+    private RawRankProfile defaultProfile;
+
+    private Map<String, RawRankProfile> rankProfiles=new java.util.LinkedHashMap<>();
+
+    /**
+     * Creates a rank profile
+     *
+     * @param search the search definition this is a rank profile from
+     * @param attributeFields the attribute fields to create a ranking for
+     */
+    public RankProfileList(Search search, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry) {
+        setName(search.getName());
+        deriveRankProfiles(rankProfileRegistry, search, attributeFields);
+    }
+
+    private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, Search search, AttributeFields attributeFields) {
+        defaultProfile = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "default"), attributeFields);
+        rankProfiles.put(defaultProfile.getName(), defaultProfile);
+
+        for (RankProfile rank : rankProfileRegistry.localRankProfiles(search)) {
+            if ("default".equals(rank.getName()))
+                continue;
+            RawRankProfile rawRank=new RawRankProfile(rank, attributeFields);
+            rankProfiles.put(rawRank.getName(), rawRank);
+        }
+    }
+
+    public Map<String, RawRankProfile> getRankProfiles() {
+        return rankProfiles;
+    }
+
+    /** @return A named raw rank profile, or null if it is not present */
+    public RawRankProfile getRankProfile(String name) {
+        return rankProfiles.get(name);
+    }
+
+    @Override
+    public String getDerivedName() { return "rank-profiles"; }
+
+    @Override
+    public void getConfig(RankProfilesConfig.Builder builder) {
+        for (RawRankProfile rank : rankProfiles.values() ) {
+            rank.getConfig(builder);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
new file mode 100644
index 00000000000..cf3b10ccadd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -0,0 +1,374 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.parser.ParseException;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import java.util.*;
+
+/**
+ * A rank profile derived from a search definition, containing exactly the features available natively in the server
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class RawRankProfile implements RankProfilesConfig.Producer {
+
+    private String name;
+
+    public String getName() {
+        return name;
+    }
+
+    public String toString() {
+        return " rank profile " + name;
+    }
+
+    @Override
+    public void getConfig(RankProfilesConfig.Builder builder) {
+        RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName());
+        getRankProperties(b);
+        builder.rankprofile(b);
+    }
+
+    private void getRankProperties(RankProfilesConfig.Rankprofile.Builder b) {
+        RankProfilesConfig.Rankprofile.Fef.Builder fefB = new RankProfilesConfig.Rankprofile.Fef.Builder();
+        for (Map.Entry<String, Object> e : configProperties.entrySet()) {
+            String key = e.getKey().replaceFirst(".part\\d+$", "");
+            String val = e.getValue().toString();
+            fefB.property(new RankProfilesConfig.Rankprofile.Fef.Property.Builder().name(key).value(val));
+        }
+        b.fef(fefB);
+    }
+
+    // TODO: These are to expose coupling between the strings used here and elsewhere
+    public final static String summaryFeatureFefPropertyPrefix = "vespa.summary.feature";
+    public final static String rankFeatureFefPropertyPrefix = "vespa.dump.feature";
+
+    /**
+     * Returns an immutable view of the config properties this returns
+     */
+    public Map<String, Object> configProperties() {
+        return Collections.unmodifiableMap(configProperties);
+    }
+
+    private final Map<String, Object> configProperties;
+
+    /**
+     * Creates a raw rank profile from the given rank profile
+     */
+    public RawRankProfile(RankProfile rankProfile, AttributeFields attributeFields) {
+        this.name = rankProfile.getName();
+        configProperties = new Deriver(rankProfile, attributeFields).derive();
+    }
+
+    private static class Deriver {
+
+        /**
+         * The field rank settings of this profile
+         */
+        private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>();
+
+        private RankingExpression firstPhaseRanking = null;
+
+        private RankingExpression secondPhaseRanking = null;
+
+        private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>();
+
+        private Set<ReferenceNode> rankFeatures = new LinkedHashSet<>();
+
+        private List<RankProfile.RankProperty> rankProperties = new ArrayList<>();
+
+        /**
+         * Rank properties for weight settings to make these available to feature executors
+         */
+        private List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>();
+
+        private boolean ignoreDefaultRankFeatures = false;
+
+        private RankProfile.MatchPhaseSettings matchPhaseSettings = null;
+
+        private int rerankCount = -1;
+        private int keepRankCount = -1;
+        private int numThreadsPerSearch = -1;
+        private int numSearchPartitions = -1;
+        private double termwiseLimit = 1.0;
+        private double rankScoreDropLimit = -Double.MAX_VALUE;
+
+        /**
+         * The rank type definitions used to derive settings for the native rank features
+         */
+        private NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default");
+
+        private RankProfile rankProfile;
+
+        private Set<String> filterFields = new java.util.LinkedHashSet<>();
+
+        /**
+         * Creates a raw rank profile from the given rank profile
+         */
+        public Deriver(RankProfile rankProfile, AttributeFields attributeFields) {
+            this.rankProfile = rankProfile.compile();
+            deriveRankingFeatures(this.rankProfile);
+            deriveRankTypeSetting(this.rankProfile, attributeFields);
+            deriveFilterFields(this.rankProfile);
+            deriveWeightProperties(this.rankProfile);
+        }
+
+        private void deriveFilterFields(RankProfile rp) {
+            filterFields.addAll(rp.allFilterFields());
+        }
+
+        public void deriveRankingFeatures(RankProfile rankProfile) {
+            firstPhaseRanking = rankProfile.getFirstPhaseRanking();
+            secondPhaseRanking = rankProfile.getSecondPhaseRanking();
+            summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures());
+            rankFeatures = rankProfile.getRankFeatures();
+            rerankCount = rankProfile.getRerankCount();
+            matchPhaseSettings = rankProfile.getMatchPhaseSettings();
+            numThreadsPerSearch = rankProfile.getNumThreadsPerSearch();
+            numSearchPartitions = rankProfile.getNumSearchPartitions();
+            termwiseLimit = rankProfile.getTermwiseLimit();
+            keepRankCount = rankProfile.getKeepRankCount();
+            rankScoreDropLimit = rankProfile.getRankScoreDropLimit();
+            ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures();
+            rankProperties = new ArrayList<>(rankProfile.getRankProperties());
+            derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros());
+        }
+
+        private void derivePropertiesAndSummaryFeaturesFromMacros(Map<String, RankProfile.Macro> macros) {
+            if (macros.isEmpty()) return;
+            Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>();
+            for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) {
+                expressionMacros.put(macro.getKey(), macro.getValue().toExpressionMacro());
+            }
+
+            Map<String, String> macroProperties = new LinkedHashMap<>();
+            macroProperties.putAll(deriveMacroProperties(expressionMacros));
+            if (firstPhaseRanking != null) {
+                macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values())));
+            }
+            if (secondPhaseRanking != null) {
+                macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values())));
+            }
+            for (Map.Entry<String, String> e : macroProperties.entrySet()) {
+                rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue()));
+            }
+            SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties);
+            replaceMacroSummaryFeatures(context);
+        }
+
+        private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) {
+            SerializationContext context = new SerializationContext(eMacros);
+            for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) {
+                String script = e.getValue().getBody().getRoot().toString(context, null, null);
+                context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), script);
+            }
+            return context.serializedFunctions();
+        }
+
+        private void replaceMacroSummaryFeatures(SerializationContext context) {
+            if (summaryFeatures == null) return;
+            Map<String, ReferenceNode> macroSummaryFeatures = new LinkedHashMap<>();
+            for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) {
+                ReferenceNode referenceNode = i.next();
+                // Is the feature a macro?
+                if (context.getFunction(referenceNode.getName()) != null) {
+                    context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()),
+                            referenceNode.toString(context, null, null));
+                    ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput());
+                    macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode);
+                    i.remove(); // Will add the expanded one in next block
+                }
+            }
+            // Then, replace the summary features that were macros
+            for (Map.Entry<String, ReferenceNode> e : macroSummaryFeatures.entrySet()) {
+                summaryFeatures.add(e.getValue());
+            }
+        }
+
+        private void deriveWeightProperties(RankProfile rankProfile) {
+
+            for (RankProfile.RankSetting setting : rankProfile.rankSettings()) {
+                if (!setting.getType().equals(RankProfile.RankSetting.Type.WEIGHT)) {
+                    continue;
+                }
+                boostAndWeightRankProperties.add(new RankProfile.RankProperty("vespa.fieldweight." + setting.getFieldName(),
+                        String.valueOf(setting.getIntValue())));
+            }
+        }
+
+        /**
+         * Adds the type boosts from a rank profile
+         */
+        private void deriveRankTypeSetting(RankProfile rankProfile, AttributeFields attributeFields) {
+            for (Iterator<RankProfile.RankSetting> i = rankProfile.rankSettingIterator(); i.hasNext(); ) {
+                RankProfile.RankSetting setting = i.next();
+                if (!setting.getType().equals(RankProfile.RankSetting.Type.RANKTYPE)) continue;
+
+                deriveNativeRankTypeSetting(setting.getFieldName(), (RankType) setting.getValue(), attributeFields,
+                        hasDefaultRankTypeSetting(rankProfile, setting.getFieldName()));
+            }
+        }
+
+        public void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) {
+            if (isDefaultSetting) return;
+
+            NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType);
+            if (definition == null) throw new IllegalArgumentException("In field '" + fieldName + "': " +
+                                                                       rankType + " is known but has no implementation. " +
+                                                                       "Supported rank types: " +
+                                                                       nativeRankTypeDefinitions.types().keySet());
+
+            FieldRankSettings settings = deriveFieldRankSettings(fieldName);
+            for (Iterator<NativeTable> i = definition.rankSettingIterator(); i.hasNext(); ) {
+                NativeTable table = i.next();
+                // only add index field tables if we are processing an index field and
+                // only add attribute field tables if we are processing an attribute field
+                if ((FieldRankSettings.isIndexFieldTable(table) && attributeFields.getAttribute(fieldName) == null) ||
+                    (FieldRankSettings.isAttributeFieldTable(table) && attributeFields.getAttribute(fieldName) != null)) {
+                    settings.addTable(table);
+                }
+            }
+        }
+
+        private boolean hasDefaultRankTypeSetting(RankProfile rankProfile, String fieldName) {
+            RankProfile.RankSetting setting =
+                    rankProfile.getRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE);
+            return setting != null && setting.getValue().equals(RankType.DEFAULT);
+        }
+
+        public FieldRankSettings deriveFieldRankSettings(String fieldName) {
+            FieldRankSettings settings = fieldRankSettings.get(fieldName);
+            if (settings == null) {
+                settings = new FieldRankSettings(fieldName);
+                fieldRankSettings.put(fieldName, settings);
+            }
+            return settings;
+        }
+
+        /**
+         * Derives the properties this produces. Equal keys are suffixed with .part0 etc, remove when exporting to file
+         *
+         * @return map of the derived properties
+         */
+        public Map<String, Object> derive() {
+            Map<String, Object> props = new LinkedHashMap<>();
+            int i = 0;
+            for (RankProfile.RankProperty property : rankProperties) {
+                if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) {
+                    // Could have been set by macro expansion. Set expressions, then skip this property.
+                    try {
+                        firstPhaseRanking = new RankingExpression(property.getValue());
+                    } catch (ParseException e) {
+                        throw new IllegalArgumentException("Could not parse second phase expression", e);
+                    }
+                    continue;
+                }
+                if ("rankingExpression(secondphase).rankingScript".equals(property.getName())) {
+                    try {
+                        secondPhaseRanking = new RankingExpression(property.getValue());
+                    } catch (ParseException e) {
+                        throw new IllegalArgumentException("Could not parse second phase expression", e);
+                    }
+                    continue;
+                }
+                props.put(property.getName() + ".part" + i, property.getValue());
+                i++;
+            }
+            props.putAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase"));
+            props.putAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase"));
+            for (FieldRankSettings settings : fieldRankSettings.values()) {
+                props.putAll(settings.deriveRankProperties(i));
+            }
+            i = 0;
+            for (RankProfile.RankProperty property : boostAndWeightRankProperties) {
+                props.put(property.getName() + ".part" + i, property.getValue());
+                i++;
+            }
+            i = 0;
+            for (ReferenceNode feature : summaryFeatures) {
+                props.put(summaryFeatureFefPropertyPrefix + ".part" + i, feature.toString());
+                i++;
+            }
+            i = 0;
+            for (ReferenceNode feature : rankFeatures) {
+                props.put(rankFeatureFefPropertyPrefix + ".part" + i, feature.toString());
+                i++;
+            }
+            if (numThreadsPerSearch > 0) {
+                props.put("vespa.matching.numthreadspersearch", numThreadsPerSearch + "");
+            }
+            if (numSearchPartitions >= 0) {
+                props.put("vespa.matching.numsearchpartitions", numSearchPartitions + "");
+            }
+            if (termwiseLimit < 1.0) {
+                props.put("vespa.matching.termwise_limit", termwiseLimit + "");
+            }
+            if (matchPhaseSettings != null) {
+                props.put("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute());
+                props.put("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "");
+                props.put("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "");
+                props.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "");
+                props.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "");
+                props.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "");
+                RankProfile.DiversitySettings diversitySettings = rankProfile.getMatchPhaseSettings().getDiversity();
+                if (diversitySettings != null) {
+                    props.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute());
+                    props.put("vespa.matchphase.diversity.mingroups", diversitySettings.getMinGroups());
+                    props.put("vespa.matchphase.diversity.cutoff.factor", diversitySettings.getCutoffFactor());
+                    props.put("vespa.matchphase.diversity.cutoff.strategy", diversitySettings.getCutoffStrategy());
+                }
+            }
+            if (rerankCount > -1) {
+                props.put("vespa.hitcollector.heapsize", rerankCount + "");
+            }
+            if (keepRankCount > -1) {
+                props.put("vespa.hitcollector.arraysize", keepRankCount + "");
+            }
+            if (rankScoreDropLimit > -Double.MAX_VALUE) {
+                props.put("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "");
+            }
+            if (ignoreDefaultRankFeatures) {
+                props.put("vespa.dump.ignoredefaultfeatures", true);
+            }
+            Iterator filterFieldsIterator = filterFields.iterator();
+            while (filterFieldsIterator.hasNext()) {
+                String fieldName = (String) filterFieldsIterator.next();
+                props.put("vespa.isfilterfield." + fieldName + ".part42", true);
+            }
+            for (Map.Entry<String, String> attributeType : rankProfile.getAttributeTypes().entrySet()) {
+                props.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue());
+            }
+            for (Map.Entry<String, String> queryFeatureType : rankProfile.getQueryFeatureTypes().entrySet()) {
+                props.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue());
+            }
+            if (props.size() >= 1000000) throw new RuntimeException("Too many rank properties");
+            return props;
+        }
+
+        private Map<String, String> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) {
+            Map<String, String> ret = new LinkedHashMap<>();
+            if (expression == null) {
+                return ret;
+            }
+            String name = expression.getName();
+            if ("".equals(name)) {
+                name = phase;
+            }
+            if (expression.getRoot() instanceof ReferenceNode) {
+                ret.put("vespa.rank." + phase, expression.getRoot().toString());
+            } else {
+                ret.put("vespa.rank." + phase, "rankingExpression(" + name + ")");
+                ret.put("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString());
+            }
+            return ret;
+        }
+
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java
new file mode 100644
index 00000000000..69133e31fb3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java
@@ -0,0 +1,105 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.DataTypeName;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.Search;
+
+import java.util.*;
+
+/**
+ * <p>A class which can reorder a list of search definitions such that any supertype
+ * always preceed any subtype. Subject to this condition the given order
+ * is preserved (the minimal reordering is done).</p>
+ *
+ * <p>This class is <b>not</b> multithread safe. Only one ordering must be done
+ * at the time in any instance.</p>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SearchOrderer {
+
+    /** A map from DataTypeName to the Search defining them */
+    private Map<DataTypeName, Search> documentNameToSearch=new java.util.HashMap<>();
+
+    /**
+     * Reorders the given list of search definitions such that any supertype
+     * always preceed any subtype. Subject to this condition the given order
+     * is preserved (the minimal reordering is done).
+     *
+     * @return a new list containing the same search instances in the right order
+     */
+    public List<Search> order(List<Search> unordered) {
+        Collections.sort(unordered, new Comparator<Search>() {
+            @Override
+            public int compare(Search lhs, Search rhs) {
+                return lhs.getName().compareTo(rhs.getName());
+            }
+        });
+
+        // No, this is not a fast algorithm...
+        indexOnDocumentName(unordered);
+        List<Search> ordered=new java.util.ArrayList<>(unordered.size());
+        List<Search> moveOutwards=new java.util.ArrayList<>();
+        for (Search search: unordered) {
+            if (containsInherited(ordered,search)) {
+                addOrdered(ordered,search,moveOutwards);
+            }
+            else {
+                moveOutwards.add(search);
+            }
+        }
+
+        // Any leftovers means we have search definitions with undefined inheritants.
+        // This is warned about elsewhere.
+        ordered.addAll(moveOutwards);
+
+        documentNameToSearch.clear();
+        return ordered;
+    }
+
+    private void addOrdered(List<Search> ordered,Search search,List<Search> moveOutwards) {
+        ordered.add(search);
+        Search eligibleMove;
+        do {
+            eligibleMove=removeFirstEligibleMoveOutwards(moveOutwards,ordered);
+            if (eligibleMove!=null)
+                ordered.add(eligibleMove);
+        } while (eligibleMove!=null);
+    }
+
+    /** Removes and returns the first search from the move list which can now be added, or null if none */
+    private Search removeFirstEligibleMoveOutwards(List<Search> moveOutwards,List<Search> ordered) {
+        for (Search move : moveOutwards) {
+            if (containsInherited(ordered,move)) {
+                moveOutwards.remove(move);
+                return move;
+            }
+        }
+        return null;
+    }
+
+    private boolean containsInherited(List<Search> list,Search search) {
+        if (search.getDocument() == null) {
+            return true;
+        }
+        for (SDDocumentType sdoc : search.getDocument().getInheritedTypes() ) {
+            DataTypeName inheritedName=sdoc.getDocumentName();
+            if ("document".equals(inheritedName.getName())) continue;
+            Search inheritedSearch=documentNameToSearch.get(inheritedName);
+            if (!list.contains(inheritedSearch))
+                return false;
+        }
+        return true;
+    }
+
+    private void indexOnDocumentName(List<Search> searches) {
+        documentNameToSearch.clear();
+        for (Search search : searches) {
+            if (search.getDocument() != null) {
+                documentNameToSearch.put(search.getDocument().getDocumentName(),search);
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
new file mode 100644
index 00000000000..357e0d40f49
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import java.util.List;
+
+/**
+ * A list of derived summaries
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class Summaries extends Derived implements SummaryConfig.Producer {
+
+    private List<SummaryClass> summaries=new java.util.ArrayList<>(1);
+
+    public Summaries(Search search, DeployLogger deployLogger) {
+        // Make sure the default is first
+        summaries.add(new SummaryClass(search,search.getSummary("default"), deployLogger));
+        for (DocumentSummary summary : search.getSummaries().values()) {
+            if (!summary.getName().equals("default"))
+                summaries.add(new SummaryClass(search,summary, deployLogger));
+        }
+    }
+
+    protected String getDerivedName() { return "summary"; }
+
+    @Override
+    public void getConfig(SummaryConfig.Builder builder) {
+        builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode());
+        for (SummaryClass summaryClass : summaries) {
+            builder.classes(summaryClass.getSummaryClassConfig());
+        }
+    }    
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
new file mode 100644
index 00000000000..d21523caea2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.prelude.fastsearch.DocsumDefinitionSet;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.config.search.SummaryConfig;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+import java.util.logging.Level;
+
+/**
+ * A summary derived from a search definition.
+ * Each summary definition have at least one summary, the default
+ * which has the same name as the search definition.
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SummaryClass extends Derived {
+
+    /** True if this summary class needs to access summary information on disk */
+    private boolean accessingDiskSummary=false;
+
+    /** The summary fields of this indexed by name */
+    private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>();
+
+    private DeployLogger deployLogger = new BaseDeployLogger();
+
+    private final Random random = new Random(7);
+
+    /**
+     * Creates a summary class from a search definition summary
+     *
+     * @param deployLogger a {@link DeployLogger}
+     */
+    public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) {
+        this.deployLogger = deployLogger;
+        deriveName(summary);
+        deriveFields(search,summary);
+        deriveImplicitFields(summary);
+    }
+
+    private void deriveName(DocumentSummary summary) {
+        setName(summary.getName());
+    }
+
+    /** MUST be called after all other fields are added */
+    private void deriveImplicitFields(DocumentSummary summary) {
+        if (summary.getName().equals("default")) {
+            addField("documentid", DataType.STRING);
+        }
+    }
+
+    private void deriveFields(Search search, DocumentSummary summary) {
+        for (SummaryField summaryField : summary.getSummaryFields()) {
+            if (!accessingDiskSummary && search.isAccessingDiskSummary(summaryField)) {
+                accessingDiskSummary = true;
+            }
+            addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform());
+        }
+    }
+
+    private void addField(String name, DataType type) {
+        addField(name, type, null);
+    }
+
+    private void addField(String name, DataType type, SummaryTransform transform) {
+        if (fields.containsKey(name)) {
+            SummaryClassField sf = fields.get(name);
+            if (!SummaryClassField.convertDataType(type, transform).equals(sf.getType())) {
+                deployLogger.log(Level.WARNING, "Conflicting definition of field " + name + ". " +
+                               "Declared as type " + sf.getType() + " and " +
+                               type);
+            }
+        } else {
+            fields.put(name, new SummaryClassField(name, type, transform));
+        }
+    }
+
+
+    /** Returns an iterator of the fields of this summary. Removes on this iterator removes the field from this summary */
+    public Iterator<SummaryClassField> fieldIterator() {
+        return fields.values().iterator();
+    }
+
+    public void addField(SummaryClassField field) {
+        fields.put(field.getName(),field);
+    }
+
+    /** Returns the writable map of fields of this summary */ // TODO: Make read only, move writers to iterator/addField
+    public Map<String,SummaryClassField> getFields() { return fields; }
+
+    public SummaryClassField getField(String name) {
+        return fields.get(name);
+    }
+
+    public int getFieldCount() { return fields.size(); }
+
+    public int hashCode() {
+        int number = 1;
+        int hash = getName().hashCode();
+        for (Iterator i = fieldIterator(); i.hasNext(); ) {
+            SummaryClassField field = (SummaryClassField)i.next();
+            hash += number * (field.getName().hashCode() +
+                              17*field.getType().getName().hashCode());
+            number++;
+        }
+        if (hash < 0)
+            hash *= -1;
+        return hash;
+    }
+
+    public SummaryConfig.Classes.Builder getSummaryClassConfig() {
+        SummaryConfig.Classes.Builder classBuilder = new SummaryConfig.Classes.Builder();
+        int id = hashCode();
+        if (id == DocsumDefinitionSet.SLIME_MAGIC_ID) {
+            deployLogger.log(Level.WARNING, "Summary class '" + getName() + "' hashes to the SLIME_MAGIC_ID '" + id +
+                                            "'. This is unlikely but I autofix it for you by adding a random number.");
+            id += random.nextInt();
+        }
+        classBuilder.
+            id(id).
+            name(getName());
+        for (SummaryClassField field : fields.values() ) {
+            classBuilder.fields(new SummaryConfig.Classes.Fields.Builder().
+                    name(field.getName()).
+                    type(field.getType().getName()));
+        }
+        return classBuilder;
+    }
+    
+    protected String getDerivedName() { return "summary"; }
+
+    public String toString() {
+        return "summary class " + getName();
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
new file mode 100644
index 00000000000..bb1dd87f314
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+
+/**
+ * A summary field derived from a search definition
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SummaryClassField {
+
+    private final String name;
+
+    private final Type type;
+
+    /** The summary field type enumeration */
+    public enum Type {
+
+        BYTE("byte"),
+        SHORT("short"),
+        INTEGER("integer"),
+        INT64("int64"),
+        FLOAT("float"),
+        DOUBLE("double"),
+        STRING("string"),
+        DATA("data"),
+        LONGSTRING("longstring"),
+        LONGDATA("longdata"),
+        XMLSTRING("xmlstring"),
+        FEATUREDATA("featuredata"),
+        JSONSTRING("jsonstring");
+
+        private String name;
+
+        private Type(String name) {
+            this.name = name;
+        }
+
+        /** Returns the name of this type */
+        public String getName() {
+            return name;
+        }
+
+        public String toString() {
+            return "type: " + name;
+        }
+    }
+
+    public SummaryClassField(String name, DataType type, SummaryTransform transform) {
+        this.name = name;
+        this.type = convertDataType(type, transform);
+    }
+
+    public String getName() { return name; }
+
+    public Type getType() { return type; }
+
+    /** Converts to the right summary field type from a field datatype and a transform*/
+    public static Type convertDataType(DataType fieldType, SummaryTransform transform) {
+        FieldValue fval = fieldType.createFieldValue();
+        if (fval instanceof StringFieldValue) {
+            if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) {
+                return Type.FEATUREDATA;
+            } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) {
+                return Type.FEATUREDATA;
+            } else {
+                return Type.LONGSTRING;
+            }
+        } else if (fval instanceof IntegerFieldValue) {
+            return Type.INTEGER;
+        } else if (fval instanceof LongFieldValue) {
+            return Type.INT64;
+        } else if (fval instanceof FloatFieldValue) {
+            return Type.FLOAT;
+        } else if (fval instanceof DoubleFieldValue) {
+            return Type.DOUBLE;
+        } else if (fval instanceof ByteFieldValue) {
+            return Type.BYTE;
+        } else if (fval instanceof Raw) {
+            return Type.DATA;
+        } else if (fval instanceof Struct) {
+            return Type.JSONSTRING;
+        } else if (fval instanceof PredicateFieldValue) {
+            return Type.STRING;
+        } else if (fval instanceof TensorFieldValue) {
+            return Type.JSONSTRING;
+        } else if (fieldType instanceof CollectionDataType) {
+            if (transform != null && transform.equals(SummaryTransform.POSITIONS)) {
+                return Type.XMLSTRING;
+            } else {
+                return Type.JSONSTRING;
+            }
+        } else if (fieldType instanceof MapDataType) {
+            return Type.JSONSTRING;
+        } else {
+            throw new IllegalArgumentException("Don't know which summary type to " +
+                    "convert " + fieldType + " to");
+        }
+    }
+
+    public String toString() {
+        return "summary class field " + name;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java
new file mode 100644
index 00000000000..d59c671e6a5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.config.search.SummarymapConfig;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * A summary map (describing search-time summary field transformations)
+ * derived from a search definition
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class SummaryMap extends Derived implements SummarymapConfig.Producer {
+
+    private Map<String,FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>();
+
+    /** Crateate a summary map from a search definition */
+    public SummaryMap(Search search, Summaries summaries) {
+        derive(search, summaries);
+    }
+
+    protected void derive(Search search, Summaries summaries) {
+        // TODO: This should really derive from the 'summaries' argument. Bug?
+        for (DocumentSummary documentSummary : search.getSummaries().values()) {
+            derive(documentSummary);
+        }
+        super.derive(search);
+    }
+
+    protected void derive(SDField field, Search search) {
+    }
+
+    private void derive(DocumentSummary documentSummary) {
+        for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+            if (summaryField.getTransform()== SummaryTransform.NONE) continue;
+
+            if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE ||
+                summaryField.getTransform()==SummaryTransform.DISTANCE ||
+                summaryField.getTransform()==SummaryTransform.GEOPOS ||
+                summaryField.getTransform()==SummaryTransform.POSITIONS) {
+                resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(),
+                                                                                     summaryField.getTransform(),
+                                                                                     summaryField.getSingleSource()));
+            } else {
+                // Note: Currently source mapping is handled in the indexing statement,
+                // by creating a summary field for each of the values
+                // This works, but is suboptimal. We could consolidate to a minimal set and
+                // use the right value from the minimal set as the third parameter here,
+                // and add "override" commands to multiple static values
+                resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(),
+                                                                                     summaryField.getTransform(),
+                                                                                     summaryField.getName()));
+            }
+        }
+    }
+
+    /** Returns a read-only iterator of the FieldResultTransforms of this summary map */
+    public Iterator resultTransformIterator() {
+        return Collections.unmodifiableCollection(resultTransforms.values()).iterator();
+    }
+
+    protected String getDerivedName() { return "summarymap"; }
+
+    /** Returns the command name of a transform */
+    private String getCommand(SummaryTransform transform) {
+        if (transform.equals(SummaryTransform.DISTANCE))
+            return "absdist";
+        else if (transform.isDynamic())
+            return "dynamicteaser";
+        else
+            return transform.getName();
+    }
+
+    /**
+     * Does this summary command name stand for a dynamic transform?
+     * We need this because some model information is shared through configs instead of model - see usage
+     */
+    public static boolean isDynamicCommand(String commandName) {
+        return (commandName.equals("dynamicteaser") || commandName.equals("smartsummary"));
+    }
+
+    @Override
+    public void getConfig(SummarymapConfig.Builder builder) {
+        builder.defaultoutputclass(-1);
+        for (FieldResultTransform frt : resultTransforms.values()) {
+            SummarymapConfig.Override.Builder oB = new SummarymapConfig.Override.Builder()
+                .field(frt.getFieldName())
+                .command(getCommand(frt.getTransform()));
+            if (frt.getTransform().isDynamic() ||
+                    frt.getTransform().equals(SummaryTransform.ATTRIBUTE) ||
+                    frt.getTransform().equals(SummaryTransform.DISTANCE) ||
+                    frt.getTransform().equals(SummaryTransform.GEOPOS) ||
+                    frt.getTransform().equals(SummaryTransform.POSITIONS) ||
+                    frt.getTransform().equals(SummaryTransform.TEXTEXTRACTOR))
+                {
+                    oB.arguments(frt.getArgument());
+                } else {
+                    oB.arguments("");
+                }
+            builder.override(oB);
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java
new file mode 100644
index 00000000000..65698675884
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java
@@ -0,0 +1,275 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.searchdefinition.FieldSets;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Vertical streaming matcher field specification
+ */
+public class VsmFields extends Derived implements VsmfieldsConfig.Producer {
+
+    private final Map<String, StreamingField> fields=new LinkedHashMap<>();
+    private final Map<String, StreamingDocumentType> doctypes=new LinkedHashMap<>();
+
+    public VsmFields(Search search) {
+        addSearchdefinition(search);
+    }
+
+    private void addSearchdefinition(Search search) {
+        derive(search);
+    }
+
+    protected void derive(SDDocumentType document,Search search) {
+        super.derive(document, search);
+        StreamingDocumentType docType=getDocumentType(document.getName());
+        if (docType == null) {
+            docType = new StreamingDocumentType(document.getName(), search.fieldSets());
+            doctypes.put(document.getName(), docType);
+        }
+        for (Object o : document.fieldSet()) {
+            derive(docType, (SDField) o);
+        }
+    }
+
+    protected void derive(StreamingDocumentType document, SDField field) {
+        if (field.usesStructOrMap()) {
+            for (SDField structField : field.getStructFields()) {
+                derive(document, structField); // Recursion
+            }
+        } else {
+
+            if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) )
+                return;
+
+            StreamingField streamingField=new StreamingField(field);
+            addField(streamingField.getName(),streamingField);
+            deriveIndices(document, field, streamingField);
+        }
+    }
+
+    private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField) {
+        if (field.doesIndexing()) {
+            addFieldToIndices(document, field.getName(), streamingField);
+        } else if (field.doesAttributing()) {
+            for (String indexName : field.getAttributes().keySet()) {
+                addFieldToIndices(document, indexName, streamingField);
+            }
+        }
+    }
+
+    private void addFieldToIndices(StreamingDocumentType document, String indexName, StreamingField streamingField) {
+        if (indexName.contains(".")) {
+            addFieldToIndices(document, indexName.substring(0,indexName.lastIndexOf(".")), streamingField); // Recursion
+        }
+        document.addIndexField(indexName, streamingField.getName());
+    }
+
+    private void addField(String name, StreamingField field) {
+        fields.put(name, field);
+    }
+
+    /** Returns a streaming index, or null if there is none with this name */
+    public StreamingDocumentType getDocumentType(String name) {
+        return doctypes.get(name);
+    }
+
+    public String getDerivedName() {
+        return "vsmfields";
+    }
+
+    @Override
+    public void getConfig(VsmfieldsConfig.Builder vsB) {
+        for (StreamingField streamingField : fields.values()) {
+            vsB.fieldspec(streamingField.getFieldSpecConfig());
+        }
+        for (StreamingDocumentType streamingDocType : doctypes.values()) {
+            vsB.documenttype(streamingDocType.getDocTypeConfig());
+        }
+    }
+
+    private static class StreamingField {
+
+        private final String name;
+
+        /** Whether this field does prefix matching by default */
+        private final Matching matching;
+
+        /** The type of this field */
+        private final Type type;
+
+        private final boolean isAttribute;
+
+        /** The streaming field type enumeration */
+        public static class Type {
+
+            public static Type INT8=new Type("int8","INT8");
+            public static Type INT16=new Type("int16","INT16");
+            public static Type INT32=new Type("int32","INT32");
+            public static Type INT64=new Type("int64","INT64");
+            public static Type FLOAT=new Type("float","FLOAT");
+            public static Type DOUBLE=new Type("double","DOUBLE");
+            public static Type STRING=new Type("string","AUTOUTF8");
+            public static Type UNSEARCHABLESTRING=new Type("string","NONE");
+
+            private String name;
+
+            private String searchMethod;
+
+            private Type(String name,String searchMethod) {
+                this.name=name;
+                this.searchMethod=searchMethod;
+            }
+
+            public int hashCode() {
+                return name.hashCode();
+            }
+
+            /** Returns the name of this type */
+            public String getName() { return name; }
+
+            public String getSearchMethod() { return searchMethod; }
+
+            public boolean equals(Object other) {
+                if ( ! (other instanceof Type)) return false;
+                return this.name.equals(((Type)other).name);
+            }
+
+            public String toString() {
+                return "type: " + name;
+            }
+
+        }
+
+        public StreamingField(SDField field) {
+            this(field.getName(),field.getDataType(),field.getMatching(), field.doesAttributing());
+        }
+
+        private StreamingField(String name,DataType sourceType, Matching  matching, boolean isAttribute) {
+            this.name = name;
+            this.type = convertType(sourceType);
+            this.matching = matching;
+            this.isAttribute = isAttribute;
+        }
+
+        /** Converts to the right index type from a field datatype */
+        private static Type convertType(DataType fieldType) {
+            FieldValue fval = fieldType.createFieldValue();
+            if (fieldType.equals(DataType.FLOAT)) {
+                return Type.FLOAT;
+            } else if (fieldType.equals(DataType.LONG)) {
+                return Type.INT64;
+            } else if (fieldType.equals(DataType.DOUBLE)) {
+                return Type.DOUBLE;
+            } else if (fieldType.equals(DataType.BYTE)) {
+                return Type.INT8;
+            } else if (fieldType instanceof NumericDataType) {
+                return Type.INT32;
+            } else if (fval instanceof StringFieldValue) {
+                return Type.STRING;
+            } else if (fval instanceof Raw) {
+                return Type.STRING;
+            } else if (fval instanceof PredicateFieldValue) {
+                return Type.UNSEARCHABLESTRING;
+            } else if (fval instanceof TensorFieldValue) {
+                return Type.UNSEARCHABLESTRING;
+            } else if (fieldType instanceof CollectionDataType) {
+                return convertType(((CollectionDataType) fieldType).getNestedType());
+            } else {
+                throw new IllegalArgumentException("Don't know which streaming" +
+                        " field type to " + "convert " + fieldType + " to");
+            }
+        }
+
+        public String getName() { return name; }
+
+        public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() {
+            VsmfieldsConfig.Fieldspec.Builder fB = new VsmfieldsConfig.Fieldspec.Builder();
+            String matchingName = matching.getType().getName();
+            if (matching.getType().equals(Matching.Type.TEXT))
+                matchingName = "";
+            if (matching.getType() != Matching.Type.EXACT) {
+                if (matching.isPrefix()) {
+                    matchingName = "prefix";
+                } else if (matching.isSubstring()) {
+                    matchingName = "substring";
+                } else if (matching.isSuffix()) {
+                    matchingName = "suffix";
+                }
+            }
+            if (type != Type.STRING) {
+                matchingName = "";
+            }
+            fB.name(getName())
+              .searchmethod(VsmfieldsConfig.Fieldspec.Searchmethod.Enum.valueOf(type.getSearchMethod()))
+              .arg1(matchingName)
+              .fieldtype(isAttribute
+                             ? VsmfieldsConfig.Fieldspec.Fieldtype.ATTRIBUTE
+                             : VsmfieldsConfig.Fieldspec.Fieldtype.INDEX);
+            if (matching.maxLength() != null) {
+                fB.maxlength(matching.maxLength());
+            }
+            return fB;
+        }
+
+        public boolean equals(Object o) {
+            if (o.getClass().equals(getClass())) {
+                StreamingField sf = (StreamingField)o;
+                return name.equals(sf.name) &&
+                        matching.equals(sf.matching) &&
+                        type.equals(sf.type);
+            }
+            return false;
+        }
+
+    }
+
+    private static class StreamingDocumentType {
+        private final String name;
+        private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>();
+        private final Map<String, FieldSet> userFieldSets;
+
+        public StreamingDocumentType(String name, FieldSets fieldSets) {
+            this.name=name;
+            userFieldSets = fieldSets.userFieldSets();
+        }
+
+        public VsmfieldsConfig.Documenttype.Builder getDocTypeConfig() {
+            VsmfieldsConfig.Documenttype.Builder dtB = new VsmfieldsConfig.Documenttype.Builder();
+            dtB.name(name);
+            Map<String, FieldSet> all = new LinkedHashMap<>();
+            all.putAll(fieldSets);
+            all.putAll(userFieldSets);
+            for (Map.Entry<String, FieldSet> e : all.entrySet()) {
+                VsmfieldsConfig.Documenttype.Index.Builder indB = new VsmfieldsConfig.Documenttype.Index.Builder();
+                indB.name(e.getValue().getName());
+                for (String field : e.getValue().getFieldNames()) {
+                    indB.field(new VsmfieldsConfig.Documenttype.Index.Field.Builder().name(field));
+                }
+                dtB.index(indB);
+            }
+            return dtB;
+        }
+
+        public String getName() { return name; }
+
+        public void addIndexField(String indexName, String fieldName) {
+            FieldSet fs = fieldSets.get(indexName);
+            if (fs == null) {
+                fs = new FieldSet(indexName);
+                fieldSets.put(indexName, fs);
+            }
+            fs.addFieldName(fieldName);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java
new file mode 100644
index 00000000000..aaf376f5cd9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java
@@ -0,0 +1,107 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig;
+
+import java.util.*;
+
+/**
+ * Vertical streaming matcher summary specification
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class VsmSummary extends Derived implements VsmsummaryConfig.Producer {
+    private Map<SummaryField, List<String>> summaryMap = new java.util.LinkedHashMap<>(1);
+
+    public VsmSummary(Search search) {
+        derive(search);
+    }
+
+    @Override
+    protected void derive(Search search) {
+        // Use the default class, as it is the superset
+        derive(search, search.getSummary("default"));
+    }
+
+    private void derive(Search search, DocumentSummary documentSummary) {
+        if (documentSummary==null) return;
+        for (SummaryField summaryField : documentSummary.getSummaryFields()) {
+            List<String> from = toStringList(summaryField.sourceIterator());
+
+            if (doMapField(search, summaryField)) {
+                SDField sdField = search.getField(summaryField.getName());
+                if (sdField != null && PositionDataType.INSTANCE.equals(sdField.getDataType())) {
+                    summaryMap.put(summaryField, Collections.singletonList(summaryField.getName()));
+                } else {
+                    summaryMap.put(summaryField, from);
+                }
+            }
+        }
+    }
+
+    /**
+     * Don't include field in map if sources are the same as the struct sub fields for the SDField.
+     * But do map if not all do summarying.
+     * Don't map if not struct either.
+     * @param summaryField a {@link SummaryField}
+     */
+    private boolean doMapField(Search search, SummaryField summaryField) {
+        SDField sdField = search.getField(summaryField.getName());
+        SDDocumentType document = search.getDocument();
+        if (sdField==null || ((document != null) && (document.getField(summaryField.getName()) == sdField))) {
+            return true;
+        }
+        if (summaryField.getVsmCommand().equals(SummaryField.VsmCommand.FLATTENJUNIPER)) {
+            return true;
+        }
+        if (!sdField.usesStructOrMap()) {
+            return !(sdField.getName().equals(summaryField.getName()));
+        }
+        if (summaryField.getSourceCount()==sdField.getStructFields().size()) {
+            for (SummaryField.Source source : summaryField.getSources()) {
+                if (!sdField.getStructFields().contains(new SDField(search.getDocument(), source.getName(), sdField.getDataType()))) { // equals() uses just name
+                    return true;
+                }
+                if (sdField.getStructField(source.getName())!=null && !sdField.getStructField(source.getName()).doesSummarying()) {
+                    return true;
+                }
+            }
+            // The sources in the summary field are the same as the sub-fields in the SD field.
+            // All sub fields do summarying.
+            // Don't map.
+            return false;
+        }
+        return true;
+    }
+
+    private List<String> toStringList(Iterator i) {
+        List<String> ret = new ArrayList<>();
+        while (i.hasNext()) {
+            ret.add(i.next().toString());
+        }
+        return ret;
+    }
+
+    public String getDerivedName() {
+        return "vsmsummary";
+    }
+
+    @Override
+    public void getConfig(VsmsummaryConfig.Builder vB) {
+        for (Map.Entry<SummaryField, List<String>> entry : summaryMap.entrySet()) {
+            VsmsummaryConfig.Fieldmap.Builder fmB = new VsmsummaryConfig.Fieldmap.Builder().summary(entry.getKey().getName());
+            for (String field : entry.getValue()) {
+                fmB.document(new VsmsummaryConfig.Fieldmap.Document.Builder().field(field));
+            }
+            fmB.command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString()));
+            vB.fieldmap(fmB);
+        }
+    }
+    
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java
new file mode 100644
index 00000000000..4e63729ea34
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java
new file mode 100644
index 00000000000..360b4d0d637
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived.validation;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.searchdefinition.derived.IndexingScript;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+
+/**
+ * @author <a href="mailto:mlidal@yahoo-inc.com">Mathias M Lidal</a>
+ */
+public class IndexStructureValidator extends Validator {
+
+    public IndexStructureValidator(DerivedConfiguration config, Search search) {
+        super(config, search);
+    }
+
+    public void validate() {
+        IndexingScript script = config.getIndexingScript();
+        for (Expression exp : script.expressions()) {
+            new OutputVisitor(search.getDocument(), exp).visit(exp);
+        }
+    }
+
+    private static class OutputVisitor extends ExpressionVisitor {
+
+        final SDDocumentType docType;
+        final Expression exp;
+
+        public OutputVisitor(SDDocumentType docType, Expression exp) {
+            this.docType = docType;
+            this.exp = exp;
+        }
+
+        @Override
+        protected void doVisit(Expression exp) {
+            if (!(exp instanceof OutputExpression)) {
+                return;
+            }
+            String fieldName = ((OutputExpression)exp).getFieldName();
+            if (docType.getField(fieldName) != null) {
+                return;
+            }
+            throw new IllegalArgumentException("Indexing expression '" + this.exp + "' refers to field '" +
+                                               fieldName + "' which does not exist in the index structure.");
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java
new file mode 100644
index 00000000000..9b01881ddd8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived.validation;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+
+public class Validation {
+
+    public static void validate(DerivedConfiguration config, Search search) {
+        new IndexStructureValidator(config, search).validate();
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java
new file mode 100644
index 00000000000..8c7c5afcb15
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived.validation;
+
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+
+import java.util.logging.Logger;
+
+/**
+ * @author mathiasm
+ */
+public abstract class Validator {
+    protected DerivedConfiguration config;
+    protected Search search;
+
+    protected Validator(DerivedConfiguration config, Search search) {
+        this.config = config;
+        this.search = search;
+    }
+
+    public abstract void validate();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
new file mode 100644
index 00000000000..35f6a41f0d8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -0,0 +1,320 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.*;
+import com.yahoo.document.datatypes.*;
+import com.yahoo.tensor.TensorType;
+
+import java.io.Serializable;
+import java.util.LinkedHashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * A search-time document attribute (per-document in-memory value).
+ * This belongs to the field defining the attribute.
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public final class Attribute implements Cloneable, Serializable {
+
+    // Remember to change hashCode and equals when you add new fields
+
+    private String name;
+
+    private Type type;
+    private CollectionType collectionType;
+
+    /** True if only the enum information should be read from this attribute
+     *  (i.e. the actual values are not relevant, only which documents have the
+     *   same values) Used for collapsing and unique.
+     */
+    private boolean removeIfZero = false;
+    private boolean createIfNonExistent = false;
+    private boolean enableBitVectors = false;
+    private boolean enableOnlyBitVector = false;
+
+    private boolean fastSearch = false;
+    private boolean fastAccess = false;
+    private boolean huge = false;
+    private int arity = BooleanIndexDefinition.DEFAULT_ARITY;
+    private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND;
+    private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND;
+    private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD;
+    private Optional<TensorType> tensorType = Optional.empty();
+
+    private boolean isPosition = false;
+    private final Sorting sorting = new Sorting();
+
+    /** The aliases for this attribute */
+    private final Set<String> aliases = new LinkedHashSet<>();
+
+    /**
+     * True if this attribute should be returned during first pass of search.
+     * Null means make the default decision for this kind of attribute
+     */
+    private Boolean prefetch = null;
+
+    /** The attribute type enumeration */
+    public enum Type {
+        BYTE("byte", "INT8"),
+        SHORT("short", "INT16"),
+        INTEGER("integer", "INT32"),
+        LONG("long", "INT64"),
+        FLOAT("float", "FLOAT"),
+        DOUBLE("double", "DOUBLE"),
+        STRING("string", "STRING"),
+        PREDICATE("predicate", "PREDICATE"),
+        TENSOR("tensor", "TENSOR");
+
+        private final String myName;  // different from what name() returns.
+        private final String exportAttributeTypeName;
+
+        private Type(String name, String exportAttributeTypeName) {
+            this.myName=name;
+            this.exportAttributeTypeName = exportAttributeTypeName;
+        }
+
+        public String getName() { return myName; }
+        public String getExportAttributeTypeName() { return exportAttributeTypeName; }
+
+        public String toString() {
+            return "type: " + myName;
+        }
+    }
+
+    /** The attribute collection type enumeration */
+    public enum CollectionType  {
+
+        SINGLE("SINGLE"),
+        ARRAY("ARRAY"),
+        WEIGHTEDSET ("WEIGHTEDSET");
+
+        private final String name;
+
+        private CollectionType(String name) {
+            this.name=name;
+        }
+
+        public String getName() { return name; }
+
+        public String toString() {
+            return "collectiontype: " + name;
+        }
+    }
+
+    /** Creates an attribute with default settings */
+    public Attribute(String name,DataType fieldType) {
+        this(name,convertDataType(fieldType), convertCollectionType(fieldType));
+        setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false);
+        setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false);
+    }
+
+    public Attribute(String name,Type type, CollectionType collectionType) {
+        this.name=name;
+        setType(type);
+        setCollectionType(collectionType);
+    }
+
+    /**
+     * <p>Returns whether this attribute should be included in the "attributeprefetch" summary
+     * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc.
+     *
+     * <p>Single value attributes are prefetched by default if summary is true.
+     * Multi value attributes are not.</p>
+     */
+    public boolean isPrefetch() {
+        if (prefetch!=null) return prefetch.booleanValue();
+
+        if (CollectionType.SINGLE.equals(collectionType)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /** Returns the prefetch value of this, null if the default is used. */
+    public Boolean getPrefetchValue() { return prefetch; }
+
+    public boolean isRemoveIfZero()       { return removeIfZero; }
+    public boolean isCreateIfNonExistent(){ return createIfNonExistent; }
+    public boolean isEnabledBitVectors()  { return enableBitVectors; }
+    public boolean isEnabledOnlyBitVector() { return enableOnlyBitVector; }
+    public boolean isFastSearch()         { return fastSearch; }
+    public boolean isFastAccess()         { return fastAccess; }
+    public boolean isHuge()               { return huge; }
+    public boolean isPosition()           { return isPosition; }
+
+    public int arity() { return arity; }
+    public long lowerBound() { return lowerBound; }
+    public long upperBound() { return upperBound; }
+    public double densePostingListThreshold() { return densePostingListThreshold; }
+    public Optional<TensorType> tensorType() { return tensorType; }
+
+    public Sorting getSorting() { return sorting; }
+
+    public void setRemoveIfZero(boolean remove)                  { this.removeIfZero = remove; }
+    public void setCreateIfNonExistent(boolean create)           { this.createIfNonExistent = create; }
+
+    /**
+     * Sets whether this should be included in the "attributeprefetch" document summary.
+     * True or false to override default, null to use default
+     */
+    public void setPrefetch(Boolean prefetch)                    { this.prefetch = prefetch; }
+    public void setEnableBitVectors(boolean enableBitVectors)    { this.enableBitVectors = enableBitVectors; }
+    public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; }
+    public void setFastSearch(boolean fastSearch)                { this.fastSearch = fastSearch; }
+    public void setHuge(boolean huge)                            { this.huge = huge; }
+    public void setFastAccess(boolean fastAccess)                { this.fastAccess = fastAccess; }
+    public void setPosition(boolean position)                    { this.isPosition = position; }
+    public void setArity(int arity)                              { this.arity = arity; }
+    public void setLowerBound(long lowerBound)                   { this.lowerBound = lowerBound; }
+    public void setUpperBound(long upperBound)                   { this.upperBound = upperBound; }
+    public void setDensePostingListThreshold(double threshold)   { this.densePostingListThreshold = threshold; }
+    public void setTensorType(TensorType tensorType)             { this.tensorType = Optional.of(tensorType); }
+
+    public String         getName()                     { return name; }
+    public Type           getType()                     { return type; }
+    public CollectionType getCollectionType()           { return collectionType; }
+
+    public void  setName(String name)                   { this.name=name; }
+    private void setType(Type type)                     { this.type=type; }
+    public void  setCollectionType(CollectionType type) { this.collectionType=type; }
+
+    /** Converts to the right attribute type from a field datatype */
+    public static Type convertDataType(DataType fieldType) {
+        FieldValue fval = fieldType.createFieldValue();
+        if (fval instanceof StringFieldValue) {
+            return Type.STRING;
+        } else if (fval instanceof IntegerFieldValue) {
+            return Type.INTEGER;
+        } else if (fval instanceof LongFieldValue) {
+            return Type.LONG;
+        } else if (fval instanceof FloatFieldValue) {
+            return Type.FLOAT;
+        } else if (fval instanceof DoubleFieldValue) {
+            return Type.DOUBLE;
+        } else if (fval instanceof ByteFieldValue) {
+            return Type.BYTE;
+        } else if (fval instanceof Raw) {
+            return Type.BYTE;
+        } else if (fval instanceof PredicateFieldValue) {
+            return Type.PREDICATE;
+        } else if (fval instanceof TensorFieldValue) {
+            return Type.TENSOR;
+        } else if (fieldType instanceof CollectionDataType) {
+            return convertDataType(((CollectionDataType) fieldType).getNestedType());
+        } else {
+            throw new IllegalArgumentException("Don't know which attribute type to " +
+                                               "convert " + fieldType + " to");
+        }
+    }
+
+    /** Converts to the right attribute type from a field datatype */
+    public static CollectionType convertCollectionType(DataType fieldType) {
+        if (fieldType instanceof ArrayDataType) {
+            return CollectionType.ARRAY;
+        } else if (fieldType instanceof WeightedSetDataType) {
+            return CollectionType.WEIGHTEDSET;
+        } else if (fieldType instanceof PrimitiveDataType) {
+            return CollectionType.SINGLE;
+        } else {
+            throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType");
+        }
+    }
+
+    /** Converts to the right field type from an attribute type */
+    public static DataType convertAttrType(Type attrType) {
+        if (attrType== Type.STRING) {
+            return DataType.STRING;
+        } else if (attrType== Type.INTEGER) {
+            return DataType.INT;
+        } else if (attrType== Type.LONG) {
+            return DataType.LONG;
+        } else if (attrType== Type.FLOAT) {
+            return DataType.FLOAT;
+        } else if (attrType== Type.DOUBLE) {
+            return DataType.DOUBLE;
+        } else if (attrType == Type.BYTE) {
+            return DataType.BYTE;
+        } else if (attrType == Type.PREDICATE) {
+            return DataType.PREDICATE;
+        } else if (attrType == Type.TENSOR) {
+            return DataType.TENSOR;
+        } else {
+            throw new IllegalArgumentException("Don't know which attribute type to " +
+                    "convert " + attrType + " to");
+        }
+    }
+
+    public DataType getDataType() {
+        DataType dataType = Attribute.convertAttrType(type);
+        if (collectionType.equals(Attribute.CollectionType.ARRAY)) {
+            return DataType.getArray(dataType);
+        } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) {
+            return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero);
+        } else {
+            return dataType;
+        }
+    }
+
+    public int hashCode() {
+        return name.hashCode() +
+                type.hashCode() +
+                collectionType.hashCode() +
+                sorting.hashCode() +
+                (isPrefetch() ? 13 : 0) +
+                (fastSearch ? 17 : 0) +
+                (removeIfZero ? 47 : 0) +
+                (createIfNonExistent ? 53 : 0) +
+                (isPosition ? 61 : 0) +
+                (huge ? 67 : 0) +
+                (enableBitVectors ? 71 : 0) +
+                (enableOnlyBitVector ? 73 : 0) +
+                tensorType.hashCode();
+    }
+
+    public boolean equals(Object object) {
+        if (! (object instanceof Attribute)) return false;
+
+        Attribute other=(Attribute)object;
+        if (!this.name.equals(other.name)) return false;
+        return isCompatible(other);
+    }
+
+    /** Returns whether these attributes describes the same entity, even if they have different names */
+    public boolean isCompatible(Attribute other) {
+        if ( ! this.type.equals(other.type)) return false;
+        if ( ! this.collectionType.equals(other.collectionType)) return false;
+        if (this.isPrefetch() != other.isPrefetch()) return false;
+        if (this.removeIfZero != other.removeIfZero) return false;
+        if (this.createIfNonExistent != other.createIfNonExistent) return false;
+        if (this.enableBitVectors != other.enableBitVectors) return false;
+        if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false;
+        // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now
+        if (this.fastSearch != other.fastSearch) return false;
+        if (this.huge != other.huge) return false;
+        if ( ! this.sorting.equals(other.sorting)) return false;
+        if (!this.tensorType.equals(other.tensorType)) return false;
+
+        return true;
+    }
+
+    public @Override Attribute clone() {
+        try {
+            return (Attribute)super.clone();
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Programming error");
+        }
+    }
+
+    public String toString() {
+        return "attribute '" + name + "' (" + type + ")";
+    }
+
+    public Set<String> getAliases() {
+        return aliases;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java
new file mode 100644
index 00000000000..fc5494d0f53
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.util.OptionalDouble;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+/**
+ * Encapsulates values required for native implementation of boolean search.
+ *
+ * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a>
+ * @since 5.2
+ */
+public final class BooleanIndexDefinition
+{
+    public static final int DEFAULT_ARITY = 8;
+    public static final long DEFAULT_UPPER_BOUND = Long.MAX_VALUE;
+    public static final long DEFAULT_LOWER_BOUND = Long.MIN_VALUE;
+    public static final double DEFAULT_DENSE_POSTING_LIST_THRESHOLD = 0.4;
+
+    private final OptionalInt arity;   // mandatory field value
+    private final OptionalLong lowerBound;
+    private final OptionalLong upperBound;
+    private final OptionalDouble densePostingListThreshold;
+
+    public BooleanIndexDefinition(OptionalInt arity, OptionalLong lowerBound,
+                                  OptionalLong upperBound, OptionalDouble densePostingListThreshold) {
+        this.arity = arity;
+        this.lowerBound = lowerBound;
+        this.upperBound = upperBound;
+        this.densePostingListThreshold = densePostingListThreshold;
+    }
+
+    public int getArity() {
+        return arity.getAsInt();
+    }
+
+    public boolean hasArity() {
+        return arity.isPresent();
+    }
+
+    public long getLowerBound() {
+        return lowerBound.orElse(DEFAULT_LOWER_BOUND);
+    }
+
+    public boolean hasLowerBound() {
+        return lowerBound.isPresent();
+    }
+
+    public long getUpperBound() {
+        return upperBound.orElse(DEFAULT_UPPER_BOUND);
+    }
+
+    public boolean hasUpperBound() {
+        return upperBound.isPresent();
+    }
+
+    public double getDensePostingListThreshold() {
+        return densePostingListThreshold.orElse(DEFAULT_DENSE_POSTING_LIST_THRESHOLD);
+    }
+
+    public boolean hasDensePostingListThreshold() {
+        return densePostingListThreshold.isPresent();
+    }
+
+    @Override
+    public String toString() {
+        return "BooleanIndexDefinition [arity=" + arity + ", lowerBound="
+                + lowerBound + ", upperBound=" + upperBound + ", densePostingListThreshold="
+                + densePostingListThreshold + "]";
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
new file mode 100644
index 00000000000..040a798d3b3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Searchable collection of fields.
+ * 
+ * @author balder
+ */
+public class FieldSet {
+    
+    private final String name;
+    private final Set<String> queryCommands = new LinkedHashSet<>();
+    private final Set<String> fieldNames = new TreeSet<>();
+    private final Set<SDField> fields = new TreeSet<>();
+    private Matching matching = null;
+
+    public FieldSet(String name) { this.name = name; }
+    public String getName() { return name; }
+    public FieldSet addFieldName(String field) { fieldNames.add(field); return this; }
+    public Set<String> getFieldNames() { return fieldNames; }
+    public Set<SDField> fields() { return fields; }
+
+    public Set<String> queryCommands() {
+        return queryCommands;
+    }
+
+    public void setMatching(Matching matching) {
+        this.matching = matching;
+    }
+
+    public Matching getMatching() {
+        return matching;
+    }
+
+    @Override
+    public String toString() { return "fieldset '" + name + "'"; }
+    
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java
new file mode 100644
index 00000000000..3c90bcef513
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+
+/**
+ * Defines how a field should be matched.
+ * Matching objects can be compared based on their content, but they are <i>not</i> immutable.
+ *
+ * @author bratseth
+ */
+public class Matching implements Cloneable, Serializable {
+
+    public static final Type defaultType = Type.TEXT;
+
+    public static enum Type {
+        TEXT("text"),
+        WORD("word"),
+        EXACT("exact"),
+        GRAM("gram");
+        private String name;
+        Type(String name) { this.name = name; }
+        public String getName() { return name; }
+    }
+
+    /** Which match algorithm is used by this matching setup */
+    public enum Algorithm {
+        NORMAL("normal"),
+        PREFIX("prefix"),
+        SUBSTRING("substring"),
+        SUFFIX("suffix");
+        private String name;
+        Algorithm(String name) { this.name = name; }
+        public String getName() { return name; }
+    }
+
+    private Type type = Type.TEXT;
+
+    /** The basic match algorithm */
+    private Algorithm algorithm = Algorithm.NORMAL;
+
+    private boolean typeUserSet = false;
+
+    private boolean algorithmUserSet = false;
+
+    /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */
+    private int gramSize=-1;
+
+    /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */
+    private Integer maxLength;
+
+    private String exactMatchTerminator=null;
+
+    /** Creates a matching of type "text" */
+    public Matching() {}
+
+    public Matching(Type type) {
+        this.type = type;
+    }
+
+    public Type getType() { return type; }
+
+    public void setType(Type type) {
+        this.type = type;
+        typeUserSet = true;
+    }
+
+    public Integer maxLength() { return maxLength; }
+    public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; }
+    public boolean isTypeUserSet() { return typeUserSet; }
+
+    public Algorithm getAlgorithm() { return algorithm; }
+
+    public void setAlgorithm(Algorithm algorithm) {
+        this.algorithm = algorithm;
+        algorithmUserSet = true;
+    }
+
+    public boolean isAlgorithmUserSet() { return algorithmUserSet; }
+
+    public boolean isPrefix() { return algorithm == Algorithm.PREFIX; }
+
+    public boolean isSubstring() { return algorithm == Algorithm.SUBSTRING; }
+
+    public boolean isSuffix() { return algorithm == Algorithm.SUFFIX; }
+
+    /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */
+    public int getGramSize() { return gramSize; }
+
+    public void setGramSize(int gramSize) { this.gramSize=gramSize; }
+
+    /**
+     * Merge data from another matching object
+     */
+    public void merge(Matching m) {
+        if (m.isAlgorithmUserSet()) {
+            this.setAlgorithm(m.getAlgorithm());
+        }
+        if (m.isTypeUserSet()) {
+            this.setType(m.getType());
+            if (m.getType()==Type.GRAM)
+              gramSize=m.gramSize;
+        }
+        if (m.getExactMatchTerminator() != null) {
+            this.setExactMatchTerminator(m.getExactMatchTerminator());
+        }
+    }
+
+    /**
+     * If exact matching is used, this returns the terminator string
+     * which terminates an exact matched sequence in queries. If exact
+     * matching is not used, or no terminator is set, this is null
+     */
+    public String getExactMatchTerminator() { return exactMatchTerminator; }
+
+    /**
+     * Sets the terminator string which terminates an exact matched
+     * sequence in queries (used if type is EXACT).
+     */
+    public void setExactMatchTerminator(String exactMatchTerminator) {
+        this.exactMatchTerminator = exactMatchTerminator;
+    }
+
+    public String toString() {
+        return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + "], [exact-terminator "+exactMatchTerminator+"]";
+    }
+
+    public Matching clone() {
+        try {
+            return (Matching)super.clone();
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Programming error");
+        }
+    }
+
+    public boolean equals(Object o) {
+        if (! (o instanceof Matching)) return false;
+
+        Matching other=(Matching)o;
+        if ( ! other.type.equals(this.type)) return false;
+        if ( ! other.algorithm.equals(this.algorithm)) return false;
+        if ( this.exactMatchTerminator==null && other.exactMatchTerminator!=null) return false;
+        if ( this.exactMatchTerminator!=null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) )
+            return false;
+        if ( gramSize!=other.gramSize) return false;
+        return true;
+    }
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java
new file mode 100644
index 00000000000..945593d550b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+/**
+ * class representing the character normalization
+ * we want to do on query and indexed text.
+ * Levels are strict subsets, so doing accent
+ * removal means doing codepoint normalizing
+ * and case normalizing also.
+ */
+// TODO: Missing author
+public class NormalizeLevel {
+
+    /**
+     * The current levels are as follows:
+     * NONE: no changes to input text
+     * CODEPOINT: convert text into Unicode
+     * Normalization Form Compatibility Composition
+     * LOWERCASE: also convert text into lowercase letters
+     * ACCENT: do both above and remove accents on characters
+     */
+    public enum Level {
+        NONE, CODEPOINT, LOWERCASE, ACCENT
+    }
+
+    private boolean userSpecified = false;
+    private Level level = Level.ACCENT;
+
+    /**
+     * Returns whether accents should be removed from text
+     */
+    public boolean doRemoveAccents() { return level == Level.ACCENT; }
+
+    /**
+     * Construct a default (full) normalizelevel,
+     */
+    public NormalizeLevel() {}
+
+    /**
+     * Construct for a specific level, possibly user specified
+     *
+     * @param level which level to use
+     * @param fromUser whether this was specified by the user
+     */
+    public NormalizeLevel(Level level, boolean fromUser) {
+        this.level = level;
+        this.userSpecified = fromUser;
+    }
+
+    /**
+     * Change the current level to CODEPOINT as inferred
+     * by other features' needs.  If the current level
+     * was user specified it will not change; also this
+     * will not increase the level.
+     */
+    public void inferCodepoint() {
+        if (userSpecified) {
+            // ignore inferred changes if user specified something
+            return;
+        }
+        // do not increase level
+        if (level != Level.NONE) level = Level.CODEPOINT;
+    }
+
+    /**
+     * Change the current level to LOWERCASE as inferred
+     * by other features' needs.  If the current level
+     * was user specified it will not change; also this
+     * will not increase the level.
+     */
+    public void inferLowercase() {
+        if (userSpecified) {
+            // ignore inferred changes if user specified something
+            return;
+        }
+        // do not increase level
+        if (level == Level.NONE) return;
+        if (level == Level.CODEPOINT) return;
+
+        level = Level.LOWERCASE;
+    }
+
+    public Level getLevel() {
+        return level;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java
new file mode 100644
index 00000000000..f2b0b6a9164
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * The rank type of a field. For now this is just a container of a string name.
+ * This class is immutable.
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public enum RankType {
+
+    /** *implicit* default: No type has been set. */
+    DEFAULT,
+
+    // Rank types which can be set explicitly. These are defined for Vespa in NativeRankTypeDefinitionSet
+    IDENTITY, ABOUT, TAGS, EMPTY;
+
+    @Override
+    public String toString() {
+        return "rank type " + name().toLowerCase();
+    }
+
+    /**
+     * Returns the rank type from a string, regardless of its case.
+     *
+     * @param  rankTypeName a rank type name in any casing
+     * @return the rank type found
+     * @throws IllegalArgumentException if not found
+     */
+    public static RankType fromString(String rankTypeName) {
+        try {
+            return RankType.valueOf(rankTypeName.toUpperCase());
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Unknown rank type '" + rankTypeName + "'. Supported rank types are " +
+                                               "'identity', 'about', 'tags' and 'empty'.");
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java
new file mode 100644
index 00000000000..9e0bda7fd5a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+
+/**
+ * The rank settings given in a rank clause in the search definition.
+ *
+ * @author  <a href="mailto:vehardh@yahoo-inc.com">Vegard Havdal</a>
+ */
+public class Ranking implements Cloneable, Serializable {
+
+    private boolean literal = false;
+    private boolean filter = false;
+    private boolean normal = false;
+
+    /**
+     * <p>Returns whether literal (non-stemmed, non-normalized) forms of the words should
+     * be indexed in a separate index which is searched by a automatically added rank term
+     * during searches.</p>
+     *
+     * <p>Default is false.</p>
+     */
+    public boolean isLiteral() { return literal; }
+
+    public void setLiteral(boolean literal) { this.literal = literal; }
+
+    /**
+     * <p>Returns whether this is a filter. Filters will only tell if they are matched or not,
+     * no detailed relevance information will be available about the match.</p>
+     *
+     * <p>Matching a filter is much cheaper for the search engine than matching a regular field.</p>
+     *
+     * <p>Default is false.</p>
+     */
+    public boolean isFilter() { return filter && !normal; }
+
+    public void setFilter(boolean filter) { this.filter = filter; }
+
+    /** Whether user has explicitly requested normal (non-filter) behavior */
+    public boolean isNormal() { return normal; }
+    public void setNormal(boolean n) { this.normal = n; }
+
+    /** Returns true if the given rank settings are the same */
+    public @Override boolean equals(Object o) {
+        if ( ! (o instanceof Ranking)) return false;
+
+        Ranking other=(Ranking)o;
+        if (this.filter != other.filter) return false;
+        if (this.literal != other.literal) return false;
+        if (this.normal != other.normal) return false;
+        return true;
+    }
+
+    public @Override String toString() {
+        return "rank settings [filter: " + filter + ", literal: " + literal + ", normal: "+normal+"]";
+    }
+
+    public @Override Ranking clone() {
+        try {
+            return (Ranking)super.clone();
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Programming error",e);
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
new file mode 100644
index 00000000000..6292dc9ef72
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -0,0 +1,316 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.*;
+import com.yahoo.document.annotation.AnnotationType;
+import com.yahoo.document.annotation.AnnotationTypeRegistry;
+import com.yahoo.documentmodel.NewDocumentType;
+import com.yahoo.documentmodel.VespaDocumentType;
+import com.yahoo.searchdefinition.FieldSets;
+import com.yahoo.searchdefinition.Search;
+
+import java.io.Serializable;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+   <p>A document definition is a list of fields. Documents may inherit other documents,
+   implicitly acquiring their fields as it's own. If a document is not set to inherit
+   any document, it will always inherit the document "document.0".</p>
+
+   @author  <a href="thomasg@yahoo-inc.com">Thomas Gundersen</a>
+   @author  <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a>
+*/
+public class SDDocumentType implements Cloneable, Serializable {
+    public static final SDDocumentType VESPA_DOCUMENT;
+    private Map<DataTypeName, SDDocumentType> inheritedTypes = new HashMap<>();
+    private Map<NewDocumentType.Name, SDDocumentType> ownedTypes = new HashMap<>();
+    private AnnotationTypeRegistry annotationTypes = new AnnotationTypeRegistry();
+    private DocumentType docType;    
+    private DataType structType;
+    // The field sets here are set from the processing step in SD, to ensure that the full Search and this SDDocumentType is built first.
+    private FieldSets fieldSets;
+
+    static {
+        VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName());
+        VESPA_DOCUMENT.addType(createSDDocumentType(PositionDataType.INSTANCE));
+    }
+
+    public SDDocumentType clone() throws CloneNotSupportedException {
+        SDDocumentType type = (SDDocumentType) super.clone();
+        type.docType = docType.clone();
+        type.inheritedTypes.putAll(inheritedTypes);
+        type.structType = structType;
+        return type;
+    }
+
+    /**
+     * For adding structs defined in document scope
+     *
+     * @param dt The struct to add.
+     * @return self, for chaining
+     */
+    public SDDocumentType addType(SDDocumentType dt) {
+        NewDocumentType.Name name = new NewDocumentType.Name(dt.getName());
+        if (getType(name) != null) {
+            throw new IllegalArgumentException("Data type '" + name.toString() + "' has already been used.");
+        }
+        if (name.getName() == docType.getName()) {
+            throw new IllegalArgumentException("Data type '" + name.toString() + "' can not have same name as its defining document.");
+        }
+        ownedTypes.put(name, dt);
+        return this;
+    }
+    public final SDDocumentType getOwnedType(String name) {
+         return getOwnedType(new NewDocumentType.Name(name));
+    }
+    public SDDocumentType getOwnedType(DataTypeName name) {
+        return getOwnedType(name.getName());
+    }
+
+    public SDDocumentType getOwnedType(NewDocumentType.Name name) {
+        return ownedTypes.get(name);
+    }
+
+    public final SDDocumentType getType(String name) {
+        return getType(new NewDocumentType.Name(name));
+    }
+
+    public SDDocumentType getType(NewDocumentType.Name name) {
+        SDDocumentType type = ownedTypes.get(name);
+        if (type == null) {
+            for (SDDocumentType inherited : inheritedTypes.values()) {
+                type = inherited.getType(name);
+                if (type != null) {
+                    return type;
+                }
+            }
+        }
+        return type;
+    }
+
+    public SDDocumentType addAnnotation(AnnotationType annotation) {
+        annotationTypes.register(annotation);
+        return this;
+    }
+
+    /**
+     * Access to all owned datatypes.
+     * @return all types
+     */
+    public Collection<SDDocumentType> getTypes() { return ownedTypes.values(); }
+    public Collection<AnnotationType> getAnnotations() { return annotationTypes.getTypes().values(); }
+    public AnnotationType findAnnotation(String name) { return annotationTypes.getType(name); }
+
+    public Collection<SDDocumentType> getAllTypes() {
+        Collection<SDDocumentType> list = new ArrayList<>();
+        list.addAll(getTypes());
+        for (SDDocumentType inherited : inheritedTypes.values()) {
+            list.addAll(inherited.getAllTypes());
+        }
+        return list;
+    }
+
+    /**
+     * Creates a new document type.
+     * The document type id will be generated as a hash from the document type name.
+     *
+     * @param name The name of the new document type
+    */
+    public SDDocumentType(String name) {
+        this(name,null);
+    }
+
+    public SDDocumentType(DataTypeName name) {
+        this(name.getName());
+    }
+
+    /**
+     * Creates a new document type.
+     * The document type id will be generated as a hash from the document type name.
+     *
+     * @param name The name of the new document type
+     * @param search check for type ID collisions in this search definition
+     */
+    public SDDocumentType(String name, Search search) {
+        docType = new DocumentType(name);
+        docType.getHeaderType().setCompressionConfig(new CompressionConfig());
+        docType.getBodyType().setCompressionConfig(new CompressionConfig());
+        validateId(search);
+        inherit(VESPA_DOCUMENT);
+    }
+
+    public boolean isStruct() { return getStruct() != null; }
+    public DataType getStruct() { return structType; }
+    public SDDocumentType setStruct(DataType structType) {
+        if (structType != null) {
+            this.structType = structType;
+            inheritedTypes.clear();
+        } else {
+            if (docType.getHeaderType() != null) {
+                this.structType = docType.getHeaderType();
+                inheritedTypes.clear();
+            } else {
+                throw new IllegalArgumentException("You can not set a null struct");
+            }
+        }
+        return this;
+    }
+
+    public String getName() { return docType.getName(); }
+    public DataTypeName getDocumentName() { return docType.getDataTypeName(); }
+    public DocumentType getDocumentType() { return docType; }
+
+    public void inherit(DataTypeName name) {
+        if ( ! inheritedTypes.containsKey(name)) {
+            inheritedTypes.put(name, new TemporarySDDocumentType(name));
+        }
+    }
+
+    public void inherit(SDDocumentType type) {
+        if (type != null) {
+            if (!inheritedTypes.containsKey(type.getDocumentName()) || (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) {
+                inheritedTypes.put(type.getDocumentName(), type);
+            }
+        }
+    }
+
+    public Collection<SDDocumentType>  getInheritedTypes() { return inheritedTypes.values(); }
+
+    protected void validateId(Search search) {
+        if (search == null) return;
+        if (search.getDocument(getName()) == null) return;
+        SDDocumentType doc = search.getDocument();
+        throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " +
+                "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'");
+    }
+
+    public void setFieldId(SDField field, int id) {
+        field.setId(id, docType);
+    }
+
+    /** Override getField, as it may need to ask inherited types that isn't registered in document type.
+     * @param name Name of field to get.
+     * @return The field found.
+     */
+    public Field getField(String name) {
+        if (name.contains(".")) {
+            String superFieldName = name.substring(0,name.indexOf("."));
+            String subFieldName = name.substring(name.indexOf(".")+1);
+            Field f = docType.getField(superFieldName);
+            if (f != null) {
+                if (f instanceof SDField) {
+                    SDField superField = (SDField)f;
+                    return superField.getStructField(subFieldName);
+                } else {
+                    throw new IllegalArgumentException("Field "+f.getName()+" is not SDField");
+                }
+            }
+        }
+        Field f = docType.getField(name);
+        if (f == null) {
+            for(SDDocumentType parent : inheritedTypes.values()) {
+                f = parent.getField(name);
+                if (f != null) return f;
+            }
+        }
+        return f;
+    }
+
+    public void addField(Field field) {
+    	verifyInheritance(field);
+        for (Iterator<Field> i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) {
+            if (field.getName().equalsIgnoreCase((i.next()).getName())) {
+                throw new IllegalArgumentException("Duplicate (case insensitively) " + field + " in " + this);
+            }
+        }
+        docType.addField(field);
+    }
+
+    /**
+     * This is SD parse-time inheritance check.
+     *
+     * @param field The field being verified.
+     */
+    private void verifyInheritance(Field field) {
+    	for (SDDocumentType parent : inheritedTypes.values()) {
+            for (Field pField : parent.fieldSet()) {
+            	if (pField.getName().equals(field.getName())) {
+            		if (!pField.getDataType().equals(field.getDataType())) {
+            			throw new IllegalArgumentException("For search '"+getName()+"', field '"+field.getName()+"': " +
+            					"datatype can't be different from that of same field in supertype '"+parent.getName()+"'.");
+            		}
+            	}
+            }
+    	}
+	}
+
+	public SDField addField(String string, DataType dataType) {
+        SDField field = new SDField(this, string, dataType);
+        addField(field);
+        return field;
+    }
+
+    public Field addField(String string, DataType dataType, boolean header, int code) {
+        SDField field = new SDField(this, string, code, dataType, header);
+        addField(field);
+        return field;
+    }
+
+    private Map<String, Field> fieldsInherited() {
+    	Map<String, Field> map = new LinkedHashMap<>();
+        for (SDDocumentType parent : inheritedTypes.values()) {
+            for (Field field : parent.fieldSet()) {
+                map.put(field.getName(), field);
+            }
+        }
+        return map;
+    }
+
+    public Set<Field> fieldSet() {
+        Map<String, Field> map = fieldsInherited();
+        Iterator<Field> it = docType.fieldIteratorThisTypeOnly();
+        while (it.hasNext()) {
+            Field field = it.next();
+            map.put(field.getName(), field);
+        }
+        return new LinkedHashSet<>(map.values());
+    }
+
+    public Iterator<Field> fieldIterator() {
+        return fieldSet().iterator();
+    }
+
+    public int getFieldCount() {
+        return docType.getFieldCount();
+    }
+
+    public String toString() {
+        return "SD document type '" + docType.getName() + "'";
+    }
+
+    private static SDDocumentType createSDDocumentType(StructDataType structType) {
+        SDDocumentType docType = new SDDocumentType(structType.getName());
+        for (Field field : structType.getFields()) {
+            docType.addField(new SDField(docType, field.getName(), field.getDataType()));
+        }
+        docType.setStruct(structType);
+        return docType;
+    }
+    
+    /**
+     * The field sets defined for this type and its {@link Search}
+     * @return fieldsets 
+     */
+    public FieldSets getFieldSets() {
+        return fieldSets;
+    }
+
+    /**
+     * Sets the field sets for this
+     * @param fieldSets field sets to set for this object
+     */
+    public void setFieldSets(FieldSets fieldSets) {
+        this.fieldSets = fieldSets;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
new file mode 100644
index 00000000000..b379abd63ec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -0,0 +1,770 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.*;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.fieldoperation.FieldOperation;
+import com.yahoo.searchdefinition.fieldoperation.FieldOperationContainer;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.indexinglanguage.ExpressionSearcher;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.parser.IndexingInput;
+import com.yahoo.vespa.indexinglanguage.parser.ParseException;
+
+import java.io.Serializable;
+import java.util.*;
+
+/**
+ * The field class represents a document field. It is used in
+ * the Document class to get and set fields. Each SDField has
+ * a name, a numeric ID, a data type, and a boolean that says whether it's
+ * a header field. The numeric ID is used when the fields are stored
+ * in serialized form.
+ *
+ * @author bratseth
+ */
+public class SDField extends Field implements TypedKey, FieldOperationContainer, Serializable {
+
+    /** Use this field for modifying index-structure, even if it doesn't
+        have any indexing code */
+    private boolean indexStructureField = false;
+
+    /** The indexing statements to be applied to this value during indexing */
+    private ScriptExpression indexingScript = new ScriptExpression();
+
+    /** The default rank type for indices of this field */
+    private RankType rankType = RankType.DEFAULT;
+
+    /** Rank settings in a "rank" block for the field. */
+    private Ranking ranking = new Ranking();
+
+    /**
+     * The literal boost of this field. This boost is added to a rank score
+     * when a query term matched as query term exactly (unnormalized and unstemmed).
+     * Non-positive boosts causes no boosting, 0 allows boosts
+     * to be specified in other rank profiles, while negative values
+     * turns the capability off.
+     */
+    private int literalBoost=-1;
+
+    /** The weight of this field. This is a percentage,
+     * so 100 is default to provide the identity transform. */
+    private int weight=100;
+
+    /**
+     * Indicates what kind of matching should be done on this field
+     */
+    private Matching matching=new Matching();
+
+    /** Attribute settings, or null if there are none */
+    private Map<String, Attribute> attributes = new TreeMap<>();
+
+    /**
+     * The stemming setting of this field, or null to use the default.
+     * Default is determined by the owning search definition.
+     */
+    private Stemming stemming=null;
+
+    /** How content of this field should be accent normalized etc. */
+    private NormalizeLevel normalizing = new NormalizeLevel();
+
+    /** Extra query commands of this field */
+    private List<String> queryCommands=new java.util.ArrayList<>(0);
+
+    /** Summary fields defined in this field */
+    private Map<String,SummaryField> summaryFields = new java.util.LinkedHashMap<>(0);
+
+    /** The explicitly index settings on this field */
+    private Map<String, Index> indices=new java.util.LinkedHashMap<>();
+
+    /** True if body or header is set explicitly for this field */
+    private boolean headerOrBodyDefined = false;
+
+    private boolean idOverride = false;
+
+    /** Struct fields defined in this field */
+    private Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0);
+
+    /** The document that this field was declared in, or null*/
+    protected SDDocumentType ownerDocType = null;
+
+    /** The aliases declared for this field. May pertain to indexes or attributes */
+    private Map<String, String> aliasToName = new HashMap<>();
+
+    /** Pending operations that must be applied after parsing, due to use of not-yet-defined structs. */
+    private List<FieldOperation> pendingOperations = new LinkedList<>();
+
+    private boolean isExtraField = false;
+
+    /**
+       Creates a new field. This method is only used to create reserved fields
+       @param name The name of the field
+       @param dataType The datatype of the field
+       @param isHeader Whether this is a "header" field or a "content" field
+       (true = "header").
+    */
+    protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) {
+        super(name, id, dataType, isHeader);
+        populate(populate, repo, name, dataType, isHeader);
+    }
+
+    public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) {
+        this(repo, name, id, dataType, isHeader, true);
+    }
+
+    /**
+       Creates a new field.
+
+       @param name The name of the field
+       @param dataType The datatype of the field
+       @param isHeader Whether this is a "header" field or a "content" field
+       (true = "header").
+    */
+    public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) {
+        super(name,dataType,isHeader);
+        populate(populate, repo, name, dataType, isHeader);
+    }
+
+    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) {
+        populate(populate,repo, name, dataType, isHeader, null, 0);
+    }
+
+    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching,  int recursion) {
+        if (populate || (dataType instanceof MapDataType)) {
+            populateWithStructFields(repo, name, dataType, isHeader, recursion);
+            populateWithStructMatching(repo, name, dataType, fieldMatching);
+        }
+    }
+
+    public SDField(String name, DataType dataType, boolean isHeader) {
+        this(null, name, dataType, isHeader, true);
+    }
+
+    /**
+       Creates a new field.
+
+       @param name The name of the field
+       @param dataType The datatype of the field
+       @param isHeader Whether this is a "header" field or a "content" field
+       (true = "header").
+       @param owner the owning document (used to check for id collisions)
+    */
+    protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) {
+        super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+        this.ownerDocType=owner;
+        populate(populate, repo, name, dataType, isHeader);
+    }
+
+    /**
+       Creates a new field.
+
+       @param name The name of the field
+       @param dataType The datatype of the field
+       @param isHeader Whether this is a "header" field or a "content" field
+       (true = "header").
+       @param owner The owning document (used to check for id collisions)
+       @param fieldMatching The matching object to set for the field
+    */
+    protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) {
+        super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+        this.ownerDocType=owner;
+        if (fieldMatching != null) {
+            this.setMatching(fieldMatching);
+        }
+        populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion);
+    }
+
+    /**
+       Constructor for <b>header</b> fields
+
+       @param name The name of the field
+       @param dataType The datatype of the field
+    */
+    public SDField(SDDocumentType repo,  String name, DataType dataType) {
+        this(repo, name,dataType,true, true);
+    }
+    public SDField(String name, DataType dataType) {
+        this(null, name,dataType);
+    }
+
+    public void setIsExtraField(boolean isExtra) {
+        isExtraField = isExtra;
+    }
+
+    public boolean isExtraField() {
+        return isExtraField;
+    }
+
+    public boolean doesAttributing() {
+        return containsExpression(AttributeExpression.class);
+    }
+
+    public boolean doesIndexing() {
+        return containsExpression(IndexExpression.class);
+    }
+
+    public boolean doesSummarying() {
+        if (usesStruct()) {
+            for (SDField structField : getStructFields()) {
+                if (structField.doesSummarying()) {
+                    return true;
+                }
+            }
+        }
+        return containsExpression(SummaryExpression.class);
+    }
+
+    public boolean doesLowerCasing() {
+        return containsExpression(LowerCaseExpression.class);
+    }
+
+    public <T extends Expression> boolean containsExpression(Class<T> searchFor) {
+        return findExpression(searchFor) != null;
+    }
+
+    public <T extends Expression> T findExpression(Class<T> searchFor) {
+        return new ExpressionSearcher<>(searchFor).searchIn(indexingScript);
+    }
+
+    public void addSummaryFieldSources(SummaryField summaryField) {
+        if (usesStruct()) {
+            /*
+             * How this works for structs: When at least one sub-field in a struct is to
+             * be used for summary, that whole struct field is included in summary.cfg. Then,
+             * vsmsummary.cfg specifies the sub-fields used for each struct field.
+             * So we recurse into each struct, adding the destination classes set for each sub-field
+             * to the main summary-field for the struct field.
+             */
+            for (SDField structField : getStructFields()) {
+                for (SummaryField sumF : structField.getSummaryFields()) {
+                    for (String dest : sumF.getDestinations()) {
+                        summaryField.addDestination(dest);
+                    }
+                }
+                structField.addSummaryFieldSources(summaryField);
+            }
+        } else {
+            if (doesSummarying()) {
+                summaryField.addSource(getName());
+            }
+        }
+    }
+
+    public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) {
+        DataType dt = getFirstStructOrMapRecursive();
+        if (dt == null) {
+            return;
+        }
+        if (dataType instanceof MapDataType) {
+            MapDataType mdt = (MapDataType) dataType;
+
+            SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(),
+                                           isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+            structFields.put("key", keyField);
+
+            SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(),
+                                             isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+            structFields.put("value", valueField);
+        } else {
+            if (recursion >= 10) {
+                return;
+            }
+            if (dataType instanceof CollectionDataType) {
+                dataType = ((CollectionDataType)dataType).getNestedType();
+            }
+            SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
+            if (subType == null) {
+                throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'.");
+            }
+            for (Field field : subType.fieldSet()) {
+                SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(),
+                                               isHeader, subType, new Matching(), true, recursion + 1);
+                structFields.put(field.getName(), subField);
+            }
+        }
+    }
+
+    public void populateWithStructMatching(SDDocumentType sdoc, String name, DataType dataType,
+                                           Matching superFieldMatching) {
+        DataType dt = getFirstStructOrMapRecursive();
+        if (dt != null) {
+            if (dataType instanceof MapDataType) {
+                MapDataType mdt = (MapDataType) dataType;
+
+                Matching keyFieldMatching = new Matching();
+                if (superFieldMatching != null) {
+                    keyFieldMatching.merge(superFieldMatching);
+                }
+                SDField keyField = structFields.get(name.concat(".key"));
+                if (keyField != null) {
+                    keyField.populateWithStructMatching(sdoc, name.concat(".key"), mdt.getKeyType(), keyFieldMatching);
+                    keyField.setMatching(keyFieldMatching);
+                }
+
+                Matching valueFieldMatching = new Matching();
+                if (superFieldMatching != null) {
+                    valueFieldMatching.merge(superFieldMatching);
+                }
+                SDField valueField = structFields.get(name.concat(".value"));
+                if (valueField != null) {
+                    valueField.populateWithStructMatching(sdoc, name.concat(".value"), mdt.getValueType(),
+                                                          valueFieldMatching);
+                    valueField.setMatching(valueFieldMatching);
+                }
+
+            } else {
+
+                if (dataType instanceof CollectionDataType) {
+                    dataType = ((CollectionDataType)dataType).getNestedType();
+                }
+                SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null;
+                if (subType != null) {
+                    for (Field f : subType.fieldSet()) {
+                        if (f instanceof SDField) {
+                            SDField field = (SDField)f;
+                            Matching subFieldMatching = new Matching();
+                            if (superFieldMatching != null) {
+                                subFieldMatching.merge(superFieldMatching);
+                            }
+                            subFieldMatching.merge(field.getMatching());
+                            SDField subField = structFields.get(field.getName());
+                            if (subField != null) {
+                                subField.populateWithStructMatching(sdoc, name.concat(".").concat(field.getName()), field.getDataType(),
+                                                                    subFieldMatching);
+                                subField.setMatching(subFieldMatching);
+                            }
+                        } else {
+                             throw new IllegalArgumentException("Field in struct is not SDField " + f.getName());
+                        }
+                    }
+                } else {
+                    throw new IllegalArgumentException("Could not find struct " + dataType.getName());
+                }
+            }
+        }
+    }
+
+    public void addOperation(FieldOperation op) {
+        pendingOperations.add(op);
+    }
+
+    public void applyOperations(SDField field) {
+        if (pendingOperations.isEmpty()) {
+            return;
+        }
+        ListIterator<FieldOperation> ops = pendingOperations.listIterator();
+        while (ops.hasNext()) {
+            FieldOperation op = ops.next();
+            ops.remove();
+            op.apply(field);
+        }
+    }
+
+    public void applyOperations() {
+        applyOperations(this);
+    }
+
+    public void setId(int fieldId, DocumentType owner) {
+        super.setId(fieldId, owner);
+        idOverride = true;
+    }
+
+    public StructDataType getFirstStructRecursive() {
+        DataType dataType = getDataType();
+        while (true) { // Currently no nesting of collections
+            if (dataType instanceof CollectionDataType) {
+                dataType = ((CollectionDataType)dataType).getNestedType();
+            } else if (dataType instanceof MapDataType) {
+                dataType = ((MapDataType)dataType).getValueType();
+            } else {
+                break;
+            }
+        }
+        return (dataType instanceof StructDataType) ? (StructDataType)dataType : null;
+    }
+
+    public DataType getFirstStructOrMapRecursive() {
+        DataType dataType = getDataType();
+        while (dataType instanceof CollectionDataType) { // Currently no nesting of collections
+            dataType = ((CollectionDataType)dataType).getNestedType();
+        }
+        return (dataType instanceof StructDataType || dataType instanceof MapDataType) ? dataType : null;
+    }
+
+    public boolean usesStruct() {
+        DataType dt = getFirstStructRecursive();
+        return (dt != null);
+    }
+
+    public boolean usesStructOrMap() {
+        DataType dt = getFirstStructOrMapRecursive();
+        return (dt != null);
+    }
+
+    /** Parse an indexing expression which will use the simple linguistics implementatino suitable for testing */
+    public void parseIndexingScript(String script) {
+        parseIndexingScript(script, new SimpleLinguistics());
+    }
+
+    public void parseIndexingScript(String script, Linguistics linguistics) {
+        try {
+            ScriptParserContext config = new ScriptParserContext(linguistics);
+            config.setInputStream(new IndexingInput(script));
+            setIndexingScript(ScriptExpression.newInstance(config));
+        } catch (ParseException e) {
+            throw new RuntimeException("Failed to parser script '" + script + "'.", e);
+        }
+    }
+
+    /** Sets the indexing script of this, or null to not use a script */
+    public void setIndexingScript(ScriptExpression exp) {
+        if (exp == null) {
+            exp = new ScriptExpression();
+        }
+        indexingScript = exp;
+        if (indexingScript.isEmpty()) {
+            return; // TODO: This causes empty expressions not to be propagate to struct fields!! BAD BAD BAD!!
+        }
+        if (!usesStructOrMap()) {
+            new ExpressionVisitor() {
+
+                @Override
+                protected void doVisit(Expression exp) {
+                    if (!(exp instanceof AttributeExpression)) {
+                        return;
+                    }
+                    String fieldName = ((AttributeExpression)exp).getFieldName();
+                    if (fieldName == null) {
+                        fieldName = getName();
+                    }
+                    Attribute attribute = attributes.get(fieldName);
+                    if (attribute == null) {
+                        addAttribute(new Attribute(fieldName, getDataType()));
+                    }
+                }
+            }.visit(indexingScript);
+        }
+        for (SDField structField : getStructFields()) {
+            structField.setIndexingScript(exp);
+        }
+    }
+
+    public ScriptExpression getIndexingScript() {
+        return indexingScript;
+    }
+
+    @SuppressWarnings("deprecation")
+    @Override
+    public void setDataType(DataType type) {
+        if (type.equals(DataType.URI)) { // Different defaults, naturally
+            normalizing.inferLowercase();
+            stemming=Stemming.NONE;
+        }
+        this.dataType = type;
+        if (!idOverride) {
+            this.fieldId = calculateIdV7(null);
+        }
+    }
+
+    public boolean isIndexStructureField() {
+        return indexStructureField;
+    }
+
+    public void setIndexStructureField(boolean indexStructureField) {
+        this.indexStructureField = indexStructureField;
+    }
+
+    /**
+     * Returns an iterator of the index names this should index to
+     * (whether set explicitly or not)
+     */
+    public Iterator<String> getFieldNameAsIterator() { // TODO: Replace usage by getName
+        return Collections.singletonList(getName()).iterator();
+    }
+
+    /** Returns 1 if this is indexed, 0 if it is not indexed */ // TODO: Replace by a boolean method, or something, see hasIndex
+    public int getIndexToCount() {
+        if (getIndexingScript() == null) return 0;
+        if (!doesIndexing()) return 0;
+
+        return 1;
+    }
+
+    /** Sets the literal boost of this field */
+    public void setLiteralBoost(int literalBoost) { this.literalBoost=literalBoost; }
+
+    /**
+     * Returns the literal boost of this field. This boost is added to a literal score
+     * when a query term matched as query term exactly (unnormalized and unstemmed).
+     * Default is non-positive.
+     */
+    public int getLiteralBoost() { return literalBoost; }
+
+    /** Sets the weight of this field */
+    public void setWeight(int weight) { this.weight=weight; }
+
+    /** Returns the weight of this field, or 0 if nothing is set */
+    public int getWeight() { return weight; }
+
+    /**
+     * Returns what kind of matching type should be applied.
+     */
+    public Matching getMatching() { return matching; }
+
+    /**
+     * Sets what kind of matching type should be applied.
+     * (Token matching is default, PREFIX, SUBSTRING, SUFFIX are alternatives)
+     */
+    public void setMatching(Matching matching) { this.matching=matching; }
+
+    /**
+     * Set the matching type for this field and all subfields.
+     */
+    // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right
+    //       Matching object for struct fields as lookup time instead.
+    public void setMatchingType(Matching.Type type) {
+        this.getMatching().setType(type);
+        for (SDField structField : getStructFields()) {
+            structField.setMatchingType(type);
+        }
+    }
+
+    /**
+     * Set matching algorithm for this field and all subfields.
+     */
+    // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right
+    //       Matching object for struct fields as lookup time instead.
+    public void setMatchingAlgorithm(Matching.Algorithm algorithm) {
+        this.getMatching().setAlgorithm(algorithm);
+        for (SDField structField : getStructFields()) {
+            structField.getMatching().setAlgorithm(algorithm);
+        }
+    }
+
+    /** Adds an explicit index defined in this field */
+    public void addIndex(Index index) {
+        indices.put(index.getName(),index);
+    }
+
+    /**
+     * Returns an index, or null if no index with this name has had
+     * some <b>explicit settings</b> applied in this field (even if this returns null,
+     * the index may be implicitly defined by an indexing statement)
+     */
+    public Index getIndex(String name) {
+        return indices.get(name);
+    }
+
+    /**
+     * Returns an index if this field has one (implicitly or
+     * explicitly) targeting the given name.
+     */
+    public boolean existsIndex(String name) {
+        if (indices.get(name) != null) return true;
+        if (name.equals(getName())) {
+            if (doesIndexing()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Defined indices on this field
+     * @return defined indices on this
+     */
+    public Map<String, Index> getIndices() {
+        return indices;
+    }
+
+    /**
+     * Sets the default rank type of this fields indices, and sets this rank type
+     * to all indices explicitly defined here which has no index set.
+     * (This complex behavior is dues to the fact than we would prefer to have rank types
+     * per field, not per index)
+     */
+    public void setRankType(RankType rankType) {
+        this.rankType=rankType;
+        for (Index index : getIndices().values()) {
+            if (index.getRankType()==null)
+                index.setRankType(rankType);
+        }
+
+    }
+
+    /** Returns the rank settings set in a "rank" block for this field. This is never null. */
+    public Ranking getRanking() { return ranking; }
+
+    /** Returns the default rank type of indices of this field, or null if nothing is set */
+    public RankType getRankType() { return this.rankType; }
+
+    /**
+     * <p>Returns the search-time attribute settings of this field
+     * or null if none is set.</p>
+     *
+     * <p>TODO: Make unmodifiable.</p>
+     */
+    public Map<String, Attribute> getAttributes() { return attributes; }
+
+    public void addAttribute(Attribute attribute) {
+        String name = attribute.getName();
+        if (name == null || "".equals(name)) {
+            name = getName();
+            attribute.setName(name);
+        }
+        attributes.put(attribute.getName(),attribute);
+    }
+
+    /**
+     * Returns the stemming setting of this field.
+     * Default is determined by the owning search definition.
+     *
+     * @return the stemming setting of this, or null, to use the default
+     */
+    public Stemming getStemming() { return stemming; }
+
+    /**
+     * Whether this field should be stemmed in this search definition
+     */
+    public Stemming getStemming(Search search) {
+        if (stemming!=null)
+            return stemming;
+        else
+            return search.getStemming();
+    }
+
+    /**
+     * Sets how this field should be stemmed, or set to null to use the default.
+     */
+    public void setStemming(Stemming stemming) {
+        this.stemming=stemming;
+    }
+
+    /**
+     * List of static summary fields
+     * @return list of static summary fields
+     */
+    public Collection<SummaryField> getSummaryFields() { return summaryFields.values(); }
+
+    /**
+     * Add summary field
+     */
+    public void addSummaryField(SummaryField summaryField) {
+        summaryFields.put(summaryField.getName(),summaryField);
+    }
+
+    /**
+     * Returns a summary field defined (implicitly or explicitly) by this field.
+     * Returns null if there is no such summary field defined.
+     */
+    public SummaryField getSummaryField(String name) {
+        return summaryFields.get(name);
+    }
+
+    /**
+     * Returns a summary field defined (implicitly or explicitly) by this field.
+     *
+     * @param create true to create the summary field and add it to this field before returning if it is missing
+     * @return the summary field, or null if not present and create is false
+     */
+    public SummaryField getSummaryField(String name,boolean create) {
+        SummaryField summaryField=summaryFields.get(name);
+        if (summaryField==null && create) {
+            summaryField=new SummaryField(name, getDataType());
+            addSummaryField(summaryField);
+        }
+        return summaryFields.get(name);
+    }
+
+    /** Returns list of static struct fields */
+    public Collection<SDField> getStructFields() { return structFields.values(); }
+
+    /**
+     * Returns a struct field defined in this field,
+     * potentially traversing into nested structs.
+     * Returns null if there is no such struct field defined.
+     */
+    public SDField getStructField(String name) {
+        if (name.contains(".")) {
+            String superFieldName = name.substring(0,name.indexOf("."));
+            String subFieldName = name.substring(name.indexOf(".")+1);
+            SDField superField = structFields.get(superFieldName);
+            if (superField != null) {
+                return superField.getStructField(subFieldName);
+            }
+            return null;
+        }
+        return structFields.get(name);
+    }
+
+    /**
+     * Returns how the content of this field should be accent normalized etc
+     */
+    public NormalizeLevel getNormalizing() { return normalizing; }
+
+    /**
+     * Change how the content of this field should be accent normalized etc
+     */
+    public void setNormalizing(NormalizeLevel level) { normalizing = level; }
+
+    public void addQueryCommand(String name) {
+       queryCommands.add(name);
+    }
+
+    public boolean hasQueryCommand(String name) {
+        return queryCommands.contains(name);
+    }
+
+    /**
+     * A list of query commands
+     * @return a list of strings with query commands.
+     */
+    public List<String> getQueryCommands() {
+        return queryCommands;
+    }
+
+
+    public boolean isHeaderOrBodyDefined() {
+        return headerOrBodyDefined;
+    }
+
+    public void setHeaderOrBodyDefined(boolean headerOrBodySetExplicitly) {
+        this.headerOrBodyDefined = headerOrBodySetExplicitly;
+    }
+
+    /**
+     * The document that this field was declared in, or null
+     *
+     */
+    public SDDocumentType getOwnerDocType() {
+        return ownerDocType;
+    }
+
+    /**
+     * Two fields are equal if they have the same name
+     * No, they are not.
+     */
+    public boolean equals(Object other) {
+        if ( ! (other instanceof SDField)) return false;
+        return super.equals(other);
+    }
+
+    public int hashCode() {
+        return getName().hashCode();
+    }
+
+    public String toString() {
+        return "field '" + getName() + "'";
+    }
+
+    /** The aliases declared for this field */
+    public Map<String, String> getAliasToName() {
+        return aliasToName;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java
new file mode 100644
index 00000000000..a0511dec4b0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.io.Serializable;
+
+/**
+ * A search-time document attribute sort specification(per-document in-memory value).
+ * This belongs to the attribute or field(implicitt attribute).
+ *
+ * @author  <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public final class Sorting implements Cloneable, Serializable {
+
+    // Remember to change hashCode and equals when you add new fields
+    public enum Function {UCA, RAW, LOWERCASE}
+    public enum Strength {PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL}
+    private boolean ascending = true;
+    private Function function = Function.UCA;
+    private String locale = "";
+    private Strength strength = Strength.PRIMARY;
+
+    public boolean isAscending()       { return ascending; }
+    public boolean isDescending()      { return ! ascending; }
+    public String getLocale()          { return locale; }
+    public Function getFunction()      { return function; }
+    public Strength getStrength()      { return strength; }
+
+    public void setAscending()                 { ascending = true; }
+    public void setDescending()                { ascending = false; }
+    public void setFunction(Function function) { this.function = function; }
+    public void setLocale(String locale)       { this.locale = locale; }
+    public void setStrength(Strength strength) { this.strength = strength; }
+
+    public int hashCode() {
+        return locale.hashCode() +
+               strength.hashCode() +
+               function.hashCode() +
+               (isDescending() ? 13 : 0);
+    }
+
+    public boolean equals(Object object) {
+        if (! (object instanceof Sorting)) return false;
+
+        Sorting other=(Sorting)object;
+        return this.locale.equals(other.locale) &&
+               (ascending == other.ascending) &&
+               (function == other.function) &&
+               (strength == other.strength);
+    }
+
+    public @Override Sorting clone() {
+        try {
+            return (Sorting)super.clone();
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Programming error");
+        }
+    }
+
+    public String toString() {
+        return "sorting '" + (isAscending() ? '+' : '-') + function.toString() + "(" + strength.toString() + ", " + locale + ")";
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java
new file mode 100644
index 00000000000..f471201f55e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.language.process.StemMode;
+
+import java.util.logging.Logger;
+
+/**
+ * <p>The stemming setting of a field. This describes how the search engine
+ * should transform content of this field into base forms (stems) to increase
+ * recall (find "car" when you search for "cars" etc.).</p>
+ *
+ * @author bratseth
+ */
+public enum Stemming {
+
+     /** No stemming */
+    NONE("none"),
+
+    /** Stem as much as possible */
+    ALL("all"),
+
+    /** select shortest possible stem */
+    SHORTEST("shortest"),
+
+    /** index (and query?) multiple stems */
+    MULTIPLE("multiple");
+
+    private static Logger log=Logger.getLogger(Stemming.class.getName());
+
+    private final String name;
+
+    /**
+     * Returns the stemming object for the given string.
+     * The legal stemming names are the stemming constants in any capitalization.
+     *
+     * @throws IllegalArgumentException if there is no stemming type with the given name
+     */
+    public static Stemming get(String stemmingName) {
+        try {
+            Stemming stemming = Stemming.valueOf(stemmingName.toUpperCase());
+            if (stemming.equals(ALL)) {
+                log.warning("note: stemming ALL is the same as stemming mode SHORTEST");
+                stemming = SHORTEST;
+            }
+            return stemming;
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("'" + stemmingName + "' is not a valid stemming setting");
+        }
+    }
+
+    private Stemming(String name) {
+        this.name = name;
+    }
+
+    public String getName() { return name; }
+
+    public String toString() {
+        return "stemming " + name;
+    }
+
+    public StemMode toStemMode() {
+        if (this == Stemming.SHORTEST) {
+            return StemMode.SHORTEST;
+        }
+        if (this == Stemming.MULTIPLE) {
+            return StemMode.ALL;
+        }
+        return StemMode.NONE;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java
new file mode 100644
index 00000000000..193ec75acd0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.DataTypeName;
+
+/**
+ * @author balder
+ */
+public class TemporarySDDocumentType extends SDDocumentType {
+    public TemporarySDDocumentType(DataTypeName name) {
+        super(name);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
new file mode 100644
index 00000000000..00012d65434
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.DataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class TemporarySDField extends SDField {
+
+    public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) {
+        super(owner, name, dataType, isHeader, owner, false);
+    }
+
+    public TemporarySDField(String name, DataType dataType) {
+        super(null, name, dataType, true, false);
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java
new file mode 100644
index 00000000000..6a10ebe642d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import com.yahoo.document.DataType;
+
+/**
+ * Common interface for various typed key (or field definitions).
+ * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping
+ *
+ * @author bratseth
+ */
+public interface TypedKey {
+
+    String getName();
+
+    void setDataType(DataType type);
+
+    DataType getDataType();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java
new file mode 100644
index 00000000000..0f8e200c4c6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document.annotation;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.document.annotation.AnnotationType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SDAnnotationType extends AnnotationType {
+    private SDDocumentType sdDocType;
+    private String inherits;
+
+    public SDAnnotationType(String name) {
+        super(name);
+    }
+
+    public SDAnnotationType(String name, SDDocumentType dataType, String inherits) {
+        super(name);
+        this.sdDocType = dataType;
+        this.inherits = inherits;
+    }
+
+    public SDDocumentType getSdDocType() {
+        return sdDocType;
+    }
+
+    public String getInherits() {
+        return inherits;
+    }
+
+    public void inherit(String inherits) {
+        this.inherits = inherits;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java
new file mode 100644
index 00000000000..5b82b068908
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document.annotation;
+
+import com.yahoo.document.annotation.AnnotationReferenceDataType;
+import com.yahoo.document.annotation.AnnotationType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class TemporaryAnnotationReferenceDataType extends AnnotationReferenceDataType {
+    private final String target;
+
+    public TemporaryAnnotationReferenceDataType(String target) {
+        this.target = target;
+    }
+
+    public String getTarget() {
+        return target;
+    }
+
+    @Override
+    public void setAnnotationType(AnnotationType type) {
+        super.setName("annotationreference<" + type.getName() + ">");
+        super.setAnnotationType(type);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java
new file mode 100644
index 00000000000..60aeb3d50cc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AliasOperation implements FieldOperation {
+    private String aliasedName;
+    private String alias;
+
+    public AliasOperation(String aliasedName, String alias) {
+        this.aliasedName = aliasedName;
+        this.alias = alias;
+    }
+
+    public String getAliasedName() {
+        return aliasedName;
+    }
+
+    public void setAliasedName(String aliasedName) {
+        this.aliasedName = aliasedName;
+    }
+
+    public String getAlias() {
+        return alias;
+    }
+
+    public void setAlias(String alias) {
+        this.alias = alias;
+    }
+
+    public void apply(SDField field) {
+        if (aliasedName == null) {
+            aliasedName = field.getName();
+        }
+        field.getAliasToName().put(alias, aliasedName);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java
new file mode 100644
index 00000000000..19ee857c627
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.tensor.TensorType;
+
+import java.util.Optional;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class AttributeOperation implements FieldOperation, FieldOperationContainer {
+    private final String name;
+    private Boolean huge;
+    private Boolean fastSearch;
+    private Boolean fastAccess;
+    private Boolean prefetch;
+    private Boolean enableBitVectors;
+    private Boolean enableOnlyBitVector;
+    //TODO: Husk sorting!!
+    private boolean doAlias = false;
+    private String alias;
+    private String aliasedName;
+    private Optional<TensorType> tensorType = Optional.empty();
+
+    public AttributeOperation(String name) {
+        this.name = name;
+    }
+
+    public void addOperation(FieldOperation op) {
+        //TODO: Implement this method:
+
+    }
+
+    public void applyOperations(SDField field) {
+        //TODO: Implement this method:
+
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Boolean getHuge() {
+        return huge;
+    }
+
+    public void setHuge(Boolean huge) {
+        this.huge = huge;
+    }
+
+    public Boolean getFastSearch() {
+        return fastSearch;
+    }
+
+    public void setFastSearch(Boolean fastSearch) {
+        this.fastSearch = fastSearch;
+    }
+
+    public Boolean getFastAccess() {
+        return fastAccess;
+    }
+
+    public void setFastAccess(Boolean fastAccess) {
+        this.fastAccess = fastAccess;
+    }
+
+    public Boolean getPrefetch() {
+        return prefetch;
+    }
+
+    public void setPrefetch(Boolean prefetch) {
+        this.prefetch = prefetch;
+    }
+
+    public Boolean getEnableBitVectors() {
+        return enableBitVectors;
+    }
+
+    public void setEnableBitVectors(Boolean enableBitVectors) {
+        this.enableBitVectors = enableBitVectors;
+    }
+
+    public Boolean getEnableOnlyBitVector() {
+        return enableOnlyBitVector;
+    }
+
+    public void setEnableOnlyBitVector(Boolean enableOnlyBitVector) {
+        this.enableOnlyBitVector = enableOnlyBitVector;
+    }
+
+    public boolean isDoAlias() {
+        return doAlias;
+    }
+
+    public void setDoAlias(boolean doAlias) {
+        this.doAlias = doAlias;
+    }
+
+    public String getAlias() {
+        return alias;
+    }
+
+    public void setAlias(String alias) {
+        this.alias = alias;
+    }
+
+    public String getAliasedName() {
+        return aliasedName;
+    }
+
+    public void setAliasedName(String aliasedName) {
+        this.aliasedName = aliasedName;
+    }
+
+    public void setTensorType(TensorType tensorType) {
+        this.tensorType = Optional.of(tensorType);
+    }
+
+    public void apply(SDField field) {
+        Attribute attribute = field.getAttributes().get(name);
+        if (attribute == null) {
+            attribute = new Attribute(name, field.getDataType());
+            field.addAttribute(attribute);
+        }
+
+        if (huge != null) {
+            attribute.setHuge(huge);
+        }
+        if (fastSearch != null) {
+            attribute.setFastSearch(fastSearch);
+        }
+        if (fastAccess != null) {
+            attribute.setFastAccess(fastAccess);
+        }
+        if (prefetch != null) {
+            attribute.setPrefetch(prefetch);
+        }
+        if (enableBitVectors != null) {
+            attribute.setEnableBitVectors(enableBitVectors);
+        }
+        if (enableOnlyBitVector != null) {
+            attribute.setEnableOnlyBitVector(enableOnlyBitVector);
+        }
+        if (doAlias) {
+            field.getAliasToName().put(alias, aliasedName);
+        }
+        if (tensorType.isPresent()) {
+            attribute.setTensorType(tensorType.get());
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java
new file mode 100644
index 00000000000..b830407ea2e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class BodyOperation implements FieldOperation {
+    public void apply(SDField field) {
+        field.setHeader(false);
+        field.setHeaderOrBodyDefined(true);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java
new file mode 100644
index 00000000000..f6c7277869e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class BoldingOperation implements FieldOperation {
+
+    private final boolean bold;
+
+    public BoldingOperation(boolean bold) {
+        this.bold = bold;
+    }
+
+    public void apply(SDField field) {
+        SummaryField summaryField = field.getSummaryField(field.getName(), true);
+        summaryField.addSource(field.getName());
+        summaryField.addDestination("default");
+        summaryField.setTransform(bold ? summaryField.getTransform().bold() : summaryField.getTransform().unbold());
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
new file mode 100644
index 00000000000..20caf0b13c7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface FieldOperation {
+    public void apply(SDField field);
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
new file mode 100644
index 00000000000..30d16e369b1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public interface FieldOperationContainer {
+    public void addOperation(FieldOperation op);
+    public void applyOperations(SDField field);
+    public String getName();
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java
new file mode 100644
index 00000000000..48d2711b39e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class HeaderOperation implements FieldOperation {
+    public void apply(SDField field) {
+        field.setHeader(true);
+        field.setHeaderOrBodyDefined(true);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java
new file mode 100644
index 00000000000..25318e2db85
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IdOperation implements FieldOperation {
+    private SDDocumentType document;
+    private int fieldId;
+
+    public SDDocumentType getDocument() {
+        return document;
+    }
+
+    public void setDocument(SDDocumentType document) {
+        this.document = document;
+    }
+
+    public int getFieldId() {
+        return fieldId;
+    }
+
+    public void setFieldId(int fieldId) {
+        this.fieldId = fieldId;
+    }
+
+    public void apply(SDField field) {
+         document.setFieldId(field, fieldId);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
new file mode 100644
index 00000000000..8ab065ce419
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Index.Type;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.OptionalDouble;
+import java.util.OptionalInt;
+import java.util.OptionalLong;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexOperation implements FieldOperation {
+    private String indexName;
+    private Optional<Boolean> prefix = Optional.empty();
+    private List<String> aliases = new LinkedList<>();
+    private Optional<String> stemming = Optional.empty();
+    private Optional<Type> type = Optional.empty();
+
+    private OptionalInt arity = OptionalInt.empty(); // For predicate data type in boolean search
+    private OptionalLong lowerBound = OptionalLong.empty();
+    private OptionalLong upperBound = OptionalLong.empty();
+    private OptionalDouble densePostingListThreshold = OptionalDouble.empty();
+
+    public String getIndexName() {
+        return indexName;
+    }
+
+    public void setIndexName(String indexName) {
+        this.indexName = indexName;
+    }
+
+    public boolean getPrefix() {
+        return prefix.get();
+    }
+
+    public void setPrefix(Boolean prefix) {
+        this.prefix = Optional.of(prefix);
+    }
+
+    public void addAlias(String alias) {
+        aliases.add(alias);
+    }
+
+    public String getStemming() {
+        return stemming.get();
+    }
+
+    public void setStemming(String stemming) {
+        this.stemming = Optional.of(stemming);
+    }
+
+    public void apply(SDField field) {
+        Index index = field.getIndex(indexName);
+
+        if (index == null) {
+            index = new Index(indexName);
+            field.addIndex(index);
+        }
+
+        applyToIndex(index);
+    }
+
+    public void applyToIndex(Index index) {
+        if (prefix.isPresent()) {
+            index.setPrefix(prefix.get());
+        }
+        for (String alias : aliases) {
+            index.addAlias(alias);
+        }
+        if (stemming.isPresent()) {
+            index.setStemming(Stemming.get(stemming.get()));
+        }
+        if (type.isPresent()) {
+            index.setType(type.get());
+        }
+        if (arity.isPresent() || lowerBound.isPresent() ||
+                upperBound.isPresent() || densePostingListThreshold.isPresent()) {
+            index.setBooleanIndexDefiniton(
+                    new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold));
+        }
+    }
+
+    public Type getType() {
+        return type.get();
+    }
+
+    public void setType(Type type) {
+        this.type = Optional.of(type);
+    }
+
+    public void setArity(int arity) {
+        this.arity = OptionalInt.of(arity);
+    }
+
+    public void setLowerBound(long value) {
+        this.lowerBound = OptionalLong.of(value);
+    }
+
+    public void setUpperBound(long value) {
+        this.upperBound = OptionalLong.of(value);
+    }
+
+    public void setDensePostingListThreshold(double densePostingListThreshold) {
+        this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
new file mode 100644
index 00000000000..af3852bffb8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.searchdefinition.parser.SimpleCharStream;
+import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexingOperation implements FieldOperation {
+
+    private final ScriptExpression script;
+
+    public IndexingOperation(ScriptExpression script) {
+        this.script = script;
+    }
+
+    public void apply(SDField field) {
+        field.setIndexingScript(script);
+    }
+
+    /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */
+    public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException {
+        return fromStream(input, multiLine, new SimpleLinguistics());
+    }
+
+    public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine, Linguistics linguistics)
+            throws ParseException {
+        ScriptParserContext config = new ScriptParserContext(linguistics);
+        config.setAnnotatorConfig(new AnnotatorConfig());
+        config.setInputStream(input);
+        ScriptExpression exp;
+        try {
+            if (multiLine) {
+                exp = ScriptExpression.newInstance(config);
+            } else {
+                exp = new ScriptExpression(StatementExpression.newInstance(config));
+            }
+        } catch (com.yahoo.vespa.indexinglanguage.parser.ParseException e) {
+            ParseException t = new ParseException("Error reported by IL parser: " + e.getMessage());
+            t.initCause(e);
+            throw t;
+        }
+        return new IndexingOperation(exp);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
new file mode 100644
index 00000000000..a416895a6a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class IndexingRewriteOperation implements FieldOperation {
+    public void apply(SDField field) {
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java
new file mode 100644
index 00000000000..9128778b7ea
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class MatchOperation implements FieldOperation {
+    private Matching.Type matchingType;
+    private Integer gramSize;
+    private Integer maxLength;
+    private Matching.Algorithm matchingAlgorithm;
+    private String exactMatchTerminator;
+
+    public void setMatchingType(Matching.Type matchingType) {
+        this.matchingType = matchingType;
+    }
+
+    public void setGramSize(Integer gramSize) {
+        this.gramSize = gramSize;
+    }
+    public void setMaxLength(Integer maxLength) {
+        this.maxLength = maxLength;
+    }
+
+    public void setMatchingAlgorithm(Matching.Algorithm matchingAlgorithm) {
+        this.matchingAlgorithm = matchingAlgorithm;
+    }
+
+    public void setExactMatchTerminator(String exactMatchTerminator) {
+        this.exactMatchTerminator = exactMatchTerminator;
+    }
+
+    public void apply(SDField field) {
+        if (matchingType != null) {
+            field.setMatchingType(matchingType);
+        }
+        if (gramSize != null) {
+            field.getMatching().setGramSize(gramSize);
+        }
+        if (maxLength != null) {
+            field.getMatching().maxLength(maxLength);
+        }
+        if (matchingAlgorithm != null) {
+            field.setMatchingAlgorithm(matchingAlgorithm);
+        }
+        if (exactMatchTerminator != null) {
+            field.getMatching().setExactMatchTerminator(exactMatchTerminator);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java
new file mode 100644
index 00000000000..e0d15cabbe0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.NormalizeLevel;
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class NormalizingOperation implements FieldOperation {
+    private NormalizeLevel.Level level;
+
+    public NormalizingOperation(String setting) {
+        if ("none".equals(setting)) {
+            this.level = NormalizeLevel.Level.NONE;
+        } else if ("codepoint".equals(setting)) {
+            this.level = NormalizeLevel.Level.CODEPOINT;
+        } else if ("lowercase".equals(setting)) {
+            this.level = NormalizeLevel.Level.LOWERCASE;
+        } else if ("accent".equals(setting)) {
+            this.level = NormalizeLevel.Level.ACCENT;
+        } else if ("all".equals(setting)) {
+            this.level = NormalizeLevel.Level.ACCENT;
+        } else {
+            throw new IllegalArgumentException("invalid normalizing setting: "+setting);
+        }
+    }
+
+    public void apply(SDField field) {
+        field.setNormalizing(new NormalizeLevel(level, true));
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java
new file mode 100644
index 00000000000..e4af5660e7b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.List;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class QueryCommandOperation implements FieldOperation {
+    private List<String> queryCommands = new java.util.ArrayList<>(0);
+
+    public void addQueryCommand(String name) {
+       queryCommands.add(name);
+    }
+
+    public void apply(SDField field) {
+        for (String command : queryCommands) {
+            field.addQueryCommand(command);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java
new file mode 100644
index 00000000000..42b1e5514a1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class RankOperation implements FieldOperation {
+
+    private Boolean literal = null;
+    private Boolean filter = null;
+    private Boolean normal = null;
+
+    public Boolean getLiteral() { return literal; }
+    public void setLiteral(Boolean literal) { this.literal = literal; }
+
+    public Boolean getFilter() { return filter; }
+    public void setFilter(Boolean filter) { this.filter = filter; }
+
+    public Boolean getNormal() { return normal; }
+    public void setNormal(Boolean n) { this.normal = n; }
+
+    public void apply(SDField field) {
+        if (literal != null) {
+            field.getRanking().setLiteral(literal);
+        }
+        if (filter != null) {
+            field.getRanking().setFilter(filter);
+        }
+        if (normal != null) {
+            field.getRanking().setNormal(normal);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java
new file mode 100644
index 00000000000..640ccc48818
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class RankTypeOperation implements FieldOperation {
+    private String indexName;
+    private RankType type;
+
+    public String getIndexName() {
+        return indexName;
+    }
+    public void setIndexName(String indexName) {
+        this.indexName = indexName;
+    }
+
+    public RankType getType() {
+        return type;
+    }
+    public void setType(RankType type) {
+        this.type = type;
+    }
+
+    public void apply(SDField field) {
+        if (indexName == null) {
+            field.setRankType(type); // Set default if the index is not specified.
+        } else {
+            Index index = field.getIndex(indexName);
+            if (index == null) {
+                index = new Index(indexName);
+                field.addIndex(index);
+            }
+            index.setRankType(type);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java
new file mode 100644
index 00000000000..e9225f0def5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java
@@ -0,0 +1,91 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Sorting;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SortingOperation implements FieldOperation {
+    private String attributeName;
+    private Boolean ascending;
+    private Boolean descending;
+    private Sorting.Function function;
+    private Sorting.Strength strength;
+    private String locale;
+
+    public SortingOperation(String attributeName) {
+        this.attributeName = attributeName;
+    }
+
+    public String getAttributeName() {
+        return attributeName;
+    }
+
+    public Boolean getAscending() {
+        return ascending;
+    }
+
+    public void setAscending() {
+        this.ascending = true;
+    }
+
+    public Boolean getDescending() {
+        return descending;
+    }
+
+    public void setDescending() {
+        this.descending = true;
+    }
+
+    public Sorting.Function getFunction() {
+        return function;
+    }
+
+    public void setFunction(Sorting.Function function) {
+        this.function = function;
+    }
+
+    public Sorting.Strength getStrength() {
+        return strength;
+    }
+
+    public void setStrength(Sorting.Strength strength) {
+        this.strength = strength;
+    }
+
+    public String getLocale() {
+        return locale;
+    }
+
+    public void setLocale(String locale) {
+        this.locale = locale;
+    }
+
+    public void apply(SDField field) {
+        Attribute attribute = field.getAttributes().get(attributeName);
+        if (attribute == null) {
+            attribute = new Attribute(attributeName, field.getDataType());
+            field.addAttribute(attribute);
+        }
+        Sorting sorting = attribute.getSorting();
+
+        if (ascending != null) {
+            sorting.setAscending();
+        }
+        if (descending != null) {
+            sorting.setDescending();
+        }
+        if (function != null) {
+            sorting.setFunction(function);
+        }
+        if (strength != null) {
+            sorting.setStrength(strength);
+        }
+        if (locale != null) {
+            sorting.setLocale(locale);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java
new file mode 100644
index 00000000000..19b760386a1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StemmingOperation implements FieldOperation {
+    private String setting;
+
+    public String getSetting() {
+        return setting;
+    }
+
+    public void setSetting(String setting) {
+        this.setting = setting;
+    }
+
+    public void apply(SDField field) {
+        field.setStemming(Stemming.get(setting));
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java
new file mode 100644
index 00000000000..f0c3d964934
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class StructFieldOperation implements FieldOperation, FieldOperationContainer {
+    private String structFieldName;
+    private List<FieldOperation> pendingOperations = new LinkedList<>();
+
+    public StructFieldOperation(String structFieldName) {
+        this.structFieldName = structFieldName;
+    }
+
+    public void apply(SDField field) {
+        SDField structField = field.getStructField(structFieldName);
+        if (structField == null ) {
+            throw new IllegalArgumentException("Struct field '" + structFieldName + "' has not been defined in struct " +
+                                               "for field '" + field.getName() + "'.");
+        }
+
+        applyOperations(structField);
+    }
+
+    public void addOperation(FieldOperation op) {
+        pendingOperations.add(op);
+    }
+
+    public void applyOperations(SDField field) {
+        if (pendingOperations.isEmpty()) {
+            return;
+        }
+        ListIterator<FieldOperation> ops = pendingOperations.listIterator();
+        while (ops.hasNext()) {
+            FieldOperation op = ops.next();
+            ops.remove();
+            op.apply(field);
+        }
+    }
+
+    public String getName() {
+        return structFieldName;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java
new file mode 100644
index 00000000000..f320ef649b4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java
@@ -0,0 +1,81 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SummaryInFieldLongOperation extends SummaryInFieldOperation {
+    private DataType type;
+    private Boolean bold;
+    private Set<String> destinations = new java.util.LinkedHashSet<>();
+    private List<SummaryField.Property> properties = new ArrayList<>();
+
+    public SummaryInFieldLongOperation(String name) {
+        super(name);
+    }
+
+    public SummaryInFieldLongOperation() {
+        super(null);
+    }
+
+    public void setType(DataType type) {
+        this.type = type;
+    }
+
+    public void setBold(Boolean bold) {
+        this.bold = bold;
+    }
+
+    public void addDestination(String destination) {
+        destinations.add(destination);
+    }
+
+    public Iterator<String> destinationIterator() {
+        return destinations.iterator();
+    }
+
+
+    public void addProperty(SummaryField.Property property) {
+        properties.add(property);
+    }
+
+    public void apply(SDField field) {
+        if (type == null) {
+            type = field.getDataType();
+        }
+        SummaryField summary = new SummaryField(name, type);
+        applyToSummary(summary);
+        field.addSummaryField(summary);
+    }
+
+    public void applyToSummary(SummaryField summary) {
+        if (transform != null) {
+            summary.setTransform(transform);
+        }
+
+        if (bold != null) {
+            summary.setTransform(bold ? summary.getTransform().bold() : summary.getTransform().unbold());
+        }
+
+        for (SummaryField.Source source : sources) {
+            summary.addSource(source);
+        }
+
+        for (String destination : destinations) {
+            summary.addDestination(destination);
+        }
+
+        for (SummaryField.Property prop : properties) {
+            summary.addProperty(prop.getName(), prop.getValue());
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java
new file mode 100644
index 00000000000..e1abbcc93b5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public abstract class SummaryInFieldOperation implements FieldOperation {
+    protected String name;
+    protected SummaryTransform transform;
+    protected Set<SummaryField.Source> sources = new java.util.LinkedHashSet<>();
+
+    public SummaryInFieldOperation(String name) {
+        this.name = name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setTransform(SummaryTransform transform) {
+        this.transform = transform;
+    }
+
+    public SummaryTransform getTransform() {
+        return transform;
+    }
+
+    public void addSource(String name) {
+        sources.add(new SummaryField.Source(name));
+    }
+
+    public void addSource(SummaryField.Source source) {
+        sources.add(source);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java
new file mode 100644
index 00000000000..1ffb8d9a83e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SummaryInFieldShortOperation extends SummaryInFieldOperation {
+    public SummaryInFieldShortOperation(String name) {
+        super(name);
+    }
+
+    public void apply(SDField field) {
+        SummaryField ret = field.getSummaryField(name);
+        if (ret == null) {
+            ret = new SummaryField(name, field.getDataType());
+            ret.addSource(field.getName());
+            ret.addDestination("default");
+        }
+        ret.setImplicit(false);
+
+        ret.setTransform(transform);
+        for (SummaryField.Source source : sources) {
+            ret.addSource(source);
+        }
+        field.addSummaryField(ret);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java
new file mode 100644
index 00000000000..f392b2fdcc8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class SummaryToOperation implements FieldOperation {
+    private Set<String> destinations = new java.util.LinkedHashSet<>();
+    private String name;
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public void addDestination(String destination) {
+        destinations.add(destination);
+    }
+
+    public void apply(SDField field) {
+        SummaryField summary;
+        summary = field.getSummaryField(name);
+        if (summary == null) {
+            summary = new SummaryField(field);
+            summary.addSource(field.getName());
+            summary.addDestination("default");
+            field.addSummaryField(summary);
+        }
+        summary.setImplicit(false);
+
+        for (String destination : destinations) {
+            summary.addDestination(destination);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java
new file mode 100644
index 00000000000..3cbd557b850
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class WeightOperation implements FieldOperation {
+    private int weight;
+
+    public int getWeight() {
+        return weight;
+    }
+
+    public void setWeight(int weight) {
+        this.weight = weight;
+    }
+
+    public void apply(SDField field) {
+        field.setWeight(weight);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java
new file mode 100644
index 00000000000..c1721a4dbb5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.fieldoperation;
+
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.document.WeightedSetDataType;
+
+/**
+ * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ */
+public class WeightedSetOperation implements FieldOperation {
+
+    private Boolean createIfNonExistent;
+    private Boolean removeIfZero;
+
+    public Boolean getCreateIfNonExistent() {
+        return createIfNonExistent;
+    }
+
+    public void setCreateIfNonExistent(Boolean createIfNonExistent) {
+        this.createIfNonExistent = createIfNonExistent;
+    }
+
+    public Boolean getRemoveIfZero() {
+        return removeIfZero;
+    }
+
+    public void setRemoveIfZero(Boolean removeIfZero) {
+        this.removeIfZero = removeIfZero;
+    }
+
+    public void apply(SDField field) {
+        WeightedSetDataType ctype = (WeightedSetDataType) field.getDataType();
+
+        if (createIfNonExistent != null) {
+            field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), createIfNonExistent,
+                                                      ctype.removeIfZero()));
+        }
+
+        ctype = (WeightedSetDataType) field.getDataType();
+        if (removeIfZero != null) {
+            field.setDataType(DataType.getWeightedSet(ctype.getNestedType(),
+                                                      ctype.createIfNonExistent(), removeIfZero));
+        }
+
+        ctype = (WeightedSetDataType) field.getDataType();
+        for (Object o : field.getAttributes().values()) {
+            Attribute attribute = (Attribute) o;
+            attribute.setRemoveIfZero(ctype.removeIfZero());
+            attribute.setCreateIfNonExistent(ctype.createIfNonExistent());
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "WeightedSetOperation{" +
+               "createIfNonExistent=" + createIfNonExistent +
+               ", removeIfZero=" + removeIfZero +
+               '}';
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java
new file mode 100644
index 00000000000..b925b4f2caa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.parser;
+
+import com.yahoo.javacc.FastCharStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings("deprecation")
+public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream,
+                                                                com.yahoo.vespa.indexinglanguage.parser.CharStream
+{
+    public SimpleCharStream(String input) {
+        super(input);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
new file mode 100644
index 00000000000..5d39c70e3b9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * This processor creates a {@link com.yahoo.searchdefinition.document.SDDocumentType} for each {@link Search} object which holds all the data that search
+ * associates with a document described in a search definition file. This includes all extra fields, summary fields and
+ * implicit fields. All non-indexed and non-summary fields are discarded.
+ */
+public class AddExtraFieldsToDocument extends Processor {
+
+    public AddExtraFieldsToDocument(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        SDDocumentType document = search.getDocument();
+        if (document != null) {
+            for (Field field : search.extraFieldList()) {
+                addSdField(search, document, (SDField)field);
+            }
+            for (SummaryField field : search.getSummary("default").getSummaryFields()) {
+                addSummaryField(search, document, field);
+            }
+        }
+    }
+
+    private void addSdField(Search search, SDDocumentType document, SDField field) {
+        if (field.getIndexToCount() == 0 && field.getAttributes().isEmpty()) {
+            return;
+        }
+        for (Attribute atr : field.getAttributes().values()) {
+            if (atr.getName().equals(field.getName() + "_position")) {
+                DataType type = PositionDataType.INSTANCE;
+                if (atr.getCollectionType().equals(Attribute.CollectionType.ARRAY)) {
+                    type = DataType.getArray(type);
+                }
+                addField(search, document, new SDField(document, atr.getName(), type));
+            } else if (!atr.getName().equals(field.getName())) {
+                addField(search, document, new SDField(document, atr.getName(), atr.getDataType()));
+            }
+        }
+        addField(search, document, field);
+    }
+
+    private void addSummaryField(Search search, SDDocumentType document, SummaryField field) {
+        Field docField = document.getField(field.getName());
+        if (docField == null) {
+            SDField newField = search.getField(field.getName());
+            if (newField == null) {
+                newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true);
+                newField.setIsExtraField(true);
+            }
+            document.addField(newField);
+        } else if (!docField.getDataType().equals(field.getDataType())) {
+            throw newProcessException(search, field, "Summary field has conflicting type.");
+        }
+    }
+
+    private void addField(Search search, SDDocumentType document, Field field) {
+        if (document.getField(field.getName()) != null) {
+            throw newProcessException(search, field, "Field shadows another.");
+        }
+        document.addField(field);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java
new file mode 100644
index 00000000000..9df77c09373
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Checks that attribute properties only are set for attributes that have data (are created by an indexing statement).
+ *
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class AttributeProperties extends Processor {
+
+    public AttributeProperties(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            String fieldName = field.getName();
+
+            // For each attribute, check if the attribute has been created
+            // by an indexing statement.
+            for (Attribute attribute : field.getAttributes().values()) {
+                if (attributeCreated(field, attribute.getName())) {
+                    continue;
+                }
+                // Check other fields or statements that may have created this attribute.
+                boolean created = false;
+                for (SDField f : search.allFieldsList()) {
+                    // Checking against the field we are looking at
+                    if (!f.getName().equals(fieldName)) {
+                        if (attributeCreated(f, attribute.getName())) {
+                            created = true;
+                            break;
+                        }
+                    }
+                }
+                if (!created) {
+                    throw new IllegalArgumentException("Attribute '" + attribute.getName() + "' in field '" +
+                                                       field.getName() + "' is not created by the indexing statement");
+                }
+            }
+        }
+    }
+
+    /**
+     * Checks if the attribute has been created bye an indexing statement in this field.
+     *
+     * @param field         a searchdefinition field
+     * @param attributeName name of the attribute
+     * @return true if the attribute has been created by this field, else false
+     */
+    static boolean attributeCreated(SDField field, String attributeName) {
+        if (!field.doesAttributing()) {
+            return false;
+        }
+        for (Attribute attribute : field.getAttributes().values()) {
+            if (attribute.getName().equals(attributeName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java
new file mode 100644
index 00000000000..774160793cb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Fields that derive to attribute(s) and no indices should use the WORD indexing form,
+ * in a feeble attempt to match the most peoples expectations as closely as possible.
+ *
+ * @author vegardh
+ *
+ */
+public class AttributesImplicitWord extends Processor {
+
+    public AttributesImplicitWord(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (fieldImplicitlyWordMatch(field)) {
+                field.getMatching().setType(Matching.Type.WORD);
+            }
+        }
+    }
+
+    private boolean fieldImplicitlyWordMatch(SDField field) {
+        // numeric types should not trigger exact-match query parsing
+        DataType dt = field.getDataType().getPrimitiveType();
+        if (dt != null && dt instanceof NumericDataType) {
+            return false;
+        }
+        return (field.getIndexToCount() == 0
+                && field.getAttributes().size() > 0
+                && field.getIndices().isEmpty()
+                && !field.getMatching().isTypeUserSet());
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java
new file mode 100644
index 00000000000..2e2e665eb34
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Checks that bolding or dynamic summary is turned on only for text fields. Throws exception if it is turned on for any
+ * other fields (otherwise will cause indexing failure)
+ *
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class Bolding extends Processor {
+
+    public Bolding(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            for (SummaryField summary : field.getSummaryFields()) {
+                if (summary.getTransform().isBolded() &&
+                    !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI)))
+                {
+                    throw new IllegalArgumentException("'bolding: on' for non-text field " +
+                                                       "'" + field.getName() + "'" +
+                                                       " (" + summary.getDataType() + ")" +
+                                                       " is not allowed");
+                } else if (summary.getTransform().isDynamic() &&
+                           !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI)))
+                {
+                    throw new IllegalArgumentException("'summary: dynamic' for non-text field " +
+                                                       "'" + field.getName() + "'" +
+                                                       " (" + summary.getDataType() + ")" +
+                                                       " is not allowed");
+                }
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java
new file mode 100644
index 00000000000..4ad9b5304a5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document
+ * @author vegardh
+ *
+ */
+public class BuiltInFieldSets extends Processor {
+
+    private static final String DOC_FIELDSET_NAME = "[document]";
+    public static final String SEARCH_FIELDSET_NAME = "[search]";     // Public due to oddities in position handling.
+    public static final String INTERNAL_FIELDSET_NAME = "[internal]"; // This one populated from misc places
+
+    public BuiltInFieldSets(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        addDocumentFieldSet();
+        addSearchFieldSet();
+        // "Hook" the field sets on search onto the document types, since we will include them
+        // on the document configs
+        search.getDocument().setFieldSets(search.fieldSets());
+    }
+
+    private void addSearchFieldSet() {
+        for (SDField searchField : search.extraFieldList()) {
+            search.fieldSets().addBuiltInFieldSetItem(SEARCH_FIELDSET_NAME, searchField.getName());
+        }
+    }
+
+    private void addDocumentFieldSet() {
+        for (Field docField : search.getDocument().fieldSet()) {
+            search.fieldSets().addBuiltInFieldSetItem(DOC_FIELDSET_NAME, docField.getName());
+        }
+    }
+
+    
+    
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
new file mode 100644
index 00000000000..0aabd4dc192
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java
@@ -0,0 +1,201 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.ArrayDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+
+/**
+ * Adds a "fieldName.zcurve" long attribute and a "fieldName.distance" summary field to all position type fields.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class CreatePositionZCurve extends Processor {
+
+    public CreatePositionZCurve(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            DataType fieldType = field.getDataType();
+            if ( ! isSupportedPositionType(fieldType)) continue;
+
+            if (field.doesIndexing()) {
+                fail(search, field, "Indexing of data type '" + fieldType.getName() + "' is not supported, " +
+                                    "replace 'index' statement with 'attribute'.");
+            }
+
+            if ( ! field.doesAttributing()) continue;
+
+            boolean doesSummary = field.doesSummarying();
+
+            String fieldName = field.getName();
+            field.getAttributes().remove(fieldName);
+
+            String zName = PositionDataType.getZCurveFieldName(fieldName);
+            SDField zCurveField = createZCurveField(field, zName);
+            search.addExtraField(zCurveField);
+            search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, zCurveField.getName());
+
+            // configure summary
+            Collection<String> summaryTo = removeSummaryTo(field);
+            ensureCompatibleSummary(field, zName,
+                                    PositionDataType.getPositionSummaryFieldName(fieldName),
+                                    DataType.getArray(DataType.STRING), // will become "xmlstring"
+                                    SummaryTransform.POSITIONS, summaryTo);
+            ensureCompatibleSummary(field, zName,
+                                    PositionDataType.getDistanceSummaryFieldName(fieldName),
+                                    DataType.INT,
+                                    SummaryTransform.DISTANCE, summaryTo);
+            // clear indexing script
+            field.setIndexingScript(null);
+            SDField posX = field.getStructField(PositionDataType.FIELD_X);
+            if (posX != null) {
+                posX.setIndexingScript(null);
+            }
+            SDField posY = field.getStructField(PositionDataType.FIELD_Y);
+            if (posY != null) {
+                posY.setIndexingScript(null);
+            }
+            if (doesSummary) ensureCompatibleSummary(field, zName,
+                                                     field.getName(),
+                                                     field.getDataType(),
+                                                     SummaryTransform.GEOPOS, summaryTo);
+        }
+    }
+
+    private SDField createZCurveField(SDField inputField, String fieldName) {
+        if (search.getField(fieldName) != null || search.getAttribute(fieldName) != null) {
+            throw newProcessException(search, null, "Incompatible position attribute '" + fieldName +
+                                                    "' already created.");
+        }
+        boolean isArray = inputField.getDataType() instanceof ArrayDataType;
+        SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG);
+        Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY :
+                                                                            Attribute.CollectionType.SINGLE);
+        attribute.setPosition(true);
+        attribute.setFastSearch(true);
+        field.addAttribute(attribute);
+
+        ScriptExpression script = inputField.getIndexingScript();
+        script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script);
+        script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script);
+        field.setIndexingScript(script);
+        return field;
+    }
+
+    private void ensureCompatibleSummary(SDField field, String sourceName, String summaryName, DataType summaryType,
+                                         SummaryTransform summaryTransform, Collection<String> summaryTo) {
+        SummaryField summary = search.getSummaryField(summaryName);
+        if (summary == null) {
+            summary = new SummaryField(summaryName, summaryType, summaryTransform);
+            summary.addDestination("default");
+            summary.addDestinations(summaryTo);
+            field.addSummaryField(summary);
+        } else if (!summary.getDataType().equals(summaryType)) {
+            fail(search, field, "Incompatible summary field '" + summaryName + "' type "+summary.getDataType()+" already created.");
+        } else if (summary.getTransform() == SummaryTransform.NONE) {
+            summary.setTransform(summaryTransform);
+            summary.addDestination("default");
+            summary.addDestinations(summaryTo);
+        } else if (summary.getTransform() != summaryTransform) {
+            deployLogger.log(Level.WARNING, "Summary field " + summaryName + " has wrong transform: " + summary.getTransform());
+            return;
+        }
+        SummaryField.Source source = new SummaryField.Source(sourceName);
+        summary.getSources().clear();
+        summary.addSource(source);
+    }
+
+    private Set<String> removeSummaryTo(SDField field) {
+        Set<String> summaryTo = new HashSet<>();
+        Collection<SummaryField> summaryFields = field.getSummaryFields();
+        for (SummaryField summary : summaryFields) {
+            summaryTo.addAll(summary.getDestinations());
+        }
+        summaryFields.clear();
+        return summaryTo;
+    }
+
+    private static boolean isSupportedPositionType(DataType dataType) {
+        if (dataType instanceof ArrayDataType) {
+            dataType = ((ArrayDataType)dataType).getNestedType();
+        }
+        return dataType.equals(PositionDataType.INSTANCE);
+    }
+
+    private static class RemoveSummary extends ExpressionConverter {
+
+        final String find;
+
+        RemoveSummary(String find) {
+            this.find = find;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            if (!(exp instanceof SummaryExpression)) {
+                return false;
+            }
+            String fieldName = ((SummaryExpression)exp).getFieldName();
+            return fieldName == null || fieldName.equals(find);
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            return null;
+        }
+    }
+
+    private static class PerformZCurve extends ExpressionConverter {
+
+        final String find;
+        final String replace;
+        final boolean isArray;
+
+        PerformZCurve(SDField find, String replace) {
+            this.find = find.getName();
+            this.replace = replace;
+            this.isArray = find.getDataType() instanceof ArrayDataType;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            if (!(exp instanceof AttributeExpression)) {
+                return false;
+            }
+            String fieldName = ((AttributeExpression)exp).getFieldName();
+            return fieldName == null || fieldName.equals(find);
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            return new StatementExpression(
+                    isArray ? new ForEachExpression(new ZCurveExpression()) :
+                    new ZCurveExpression(), new AttributeExpression(replace));
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java
new file mode 100644
index 00000000000..6d5b3ff8936
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+public class DeprecateAttributePrefetch extends Processor {
+
+    public DeprecateAttributePrefetch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            for (Attribute a : field.getAttributes().values()) {
+                if (Boolean.TRUE.equals(a.getPrefetchValue())) {
+                    warn(search, field, "Attribute prefetch is deprecated. Use an explicitly defined document summary with all desired fields defined as attribute.");
+                }
+                if (Boolean.FALSE.equals(a.getPrefetchValue())) {
+                    warn(search, field, "Attribute prefetch is deprecated. no-prefetch can be removed.");
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java
new file mode 100644
index 00000000000..dbe83143189
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.MapDataType;
+import com.yahoo.document.PrimitiveDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Non-primitive key types for map and weighted set forbidden (though OK in document model)
+ * @author vegardh
+ *
+ */
+public class DisallowComplexMapAndWsetKeyTypes extends Processor {
+
+    public DisallowComplexMapAndWsetKeyTypes(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+    	// TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do
+    	// not start using the static stuff on SDDocumentTypes any more.
+        for (SDField field : search.allFieldsList()) {
+            if (field.getDataType() instanceof WeightedSetDataType) {
+                DataType nestedType = ((WeightedSetDataType)field.getDataType()).getNestedType();
+                if (!(nestedType instanceof PrimitiveDataType)) {
+                    fail(search, field, "Weighted set must have a primitive key type.");
+                }
+            } else if (field.getDataType() instanceof MapDataType) {
+                if (!(((MapDataType)field.getDataType()).getKeyType() instanceof PrimitiveDataType)) {
+                    fail(search, field, "Map key type must be a primitive type");
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java
new file mode 100644
index 00000000000..49701f3e443
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Created by balder on 3/10/15.
+ */
+public class DiversitySettingsValidator extends Processor {
+    public DiversitySettingsValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) {
+            if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) {
+                validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity());
+            }
+        }
+    }
+    private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) {
+        String attributeName = settings.getAttribute();
+        new AttributeValidator(search.getName(), rankProfile.getName(),
+                search.getAttribute(attributeName), attributeName).validate();
+    }
+
+    private static class AttributeValidator extends  MatchPhaseSettingsValidator.AttributeValidator {
+        public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) {
+            super(searchName, rankProfileName, attribute, attributeName);
+        }
+
+        protected void validateThatAttributeIsSingleAndNotPredicate() {
+            if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) ||
+                 attribute.getType().equals(Attribute.Type.PREDICATE))
+            {
+                failValidation("must be single value numeric, or enumerated attribute, but it is '"
+                        + attribute.getDataType().getName() + "'");
+            }
+        }
+
+        @Override
+        public void validate() {
+            validateThatAttributeExists();
+            validateThatAttributeIsSingleAndNotPredicate();
+        }
+
+        @Override
+        public String getValidationType() {
+            return "diversity";
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java
new file mode 100644
index 00000000000..d53be4bd18d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.indexinglanguage.ExpressionSearcher;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of exact matching
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class ExactMatch extends Processor {
+
+    public static final String DEFAULT_EXACT_TERMINATOR = "@@";
+
+    public ExactMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            Matching.Type matching = field.getMatching().getType();
+            if (matching.equals(Matching.Type.EXACT) ||
+                matching.equals(Matching.Type.WORD))
+            {
+                implementExactMatch(field, search);
+            } else if (field.getMatching().getExactMatchTerminator() != null) {
+                warn(search, field, "exact-terminator requires 'exact' matching to have any effect.");
+            }
+        }
+    }
+
+    private void implementExactMatch(SDField field, Search search) {
+        field.setStemming(Stemming.NONE);
+        field.getNormalizing().inferLowercase();
+
+        if (field.getMatching().getType().equals(Matching.Type.WORD)) {
+            field.addQueryCommand("word");
+        } else { // exact
+            String exactTerminator = DEFAULT_EXACT_TERMINATOR;
+            if (field.getMatching().getExactMatchTerminator() != null &&
+                ! field.getMatching().getExactMatchTerminator().equals(""))
+            {
+                exactTerminator = field.getMatching().getExactMatchTerminator();
+            } else {
+                warn(search, field,
+                     "With 'exact' matching, an exact-terminator is needed (using \""
+                     + exactTerminator +"\" as terminator)");
+            }
+            field.addQueryCommand("exact " + exactTerminator);
+
+            // The following part illustrates how nice it would have been with canonical
+            // representation of indices
+            if (field.doesIndexing()) {
+                exactMatchSettingsForField(field);
+            }
+        }
+        ScriptExpression script = field.getIndexingScript();
+        if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) {
+            field.setIndexingScript((ScriptExpression)new MyProvider(search).convert(field.getIndexingScript()));
+        }
+    }
+
+    private void exactMatchSettingsForField(SDField field) {
+        field.getRanking().setFilter(true);
+    }
+
+    private static class MyProvider extends TypedTransformProvider {
+
+        MyProvider(Search search) {
+            super(ExactExpression.class, search);
+        }
+
+        @Override
+        protected boolean requiresTransform(Expression exp, DataType fieldType) {
+            return exp instanceof OutputExpression;
+        }
+
+        @Override
+        protected Expression newTransform(DataType fieldType) {
+            Expression exp = new ExactExpression();
+            if (fieldType instanceof CollectionDataType) {
+                exp = new ForEachExpression(exp);
+            }
+            return exp;
+        }
+    }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java
new file mode 100644
index 00000000000..e809fe90c7f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.NormalizeLevel;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Computes the right "index commands" for each fieldset in a search definition.
+ *
+ *
+ * @author vegardh
+ * @author bratseth
+ */
+public class FieldSetValidity extends Processor {
+
+    public FieldSetValidity(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) {
+            checkFieldNames(search, fieldSet);
+            checkMatching(search, fieldSet);
+            checkNormalization(search, fieldSet);
+            checkStemming(search, fieldSet);
+        }
+    }
+
+    private void checkFieldNames(Search search, FieldSet fieldSet) {
+        for (String fld : fieldSet.getFieldNames()) {
+            SDField field = search.getField(fld);
+            if (field == null) {
+                throw new IllegalArgumentException("For search '" + search.getName() + "': Field '"+ fld + "' in " + fieldSet + " does not exist.");
+            }
+        }
+    }
+
+    private void checkMatching(Search search, FieldSet fieldSet) {
+        Matching fsMatching = null;
+        for (String fld : fieldSet.getFieldNames()) {
+            SDField field = search.getField(fld);
+            Matching fieldMatching = field.getMatching();
+            if (fsMatching==null) {
+                fsMatching = fieldMatching;
+            } else {
+                if (fsMatching!=null && !fsMatching.equals(fieldMatching)) {
+                    warn(search, field, "The matching settings for the fields in " + fieldSet + " are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues.");
+                    return;
+                }
+            }
+        }
+    }
+
+    private void checkNormalization(Search search, FieldSet fieldSet) {
+        NormalizeLevel.Level fsNorm = null;
+        for (String fld : fieldSet.getFieldNames()) {
+            SDField field = search.getField(fld);
+            NormalizeLevel.Level fieldNorm = field.getNormalizing().getLevel();
+            if (fsNorm==null) {
+                fsNorm = fieldNorm;
+            } else {
+                if (fsNorm!=null && !fsNorm.equals(fieldNorm)) {
+                    warn(search, field, "The normalization settings for the fields in " + fieldSet + " are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues.");
+                    return;
+                }
+            }
+        }
+    }
+
+    private void checkStemming(Search search, FieldSet fieldSet) {
+        Stemming fsStemming = null;
+        for (String fld : fieldSet.getFieldNames()) {
+            SDField field = search.getField(fld);
+            Stemming fieldStemming = field.getStemming();
+            if (fsStemming==null) {
+                fsStemming = fieldStemming;
+            } else {
+                if (fsStemming!=null && !fsStemming.equals(fieldStemming)) {
+                    warn(search, field, "The stemming settings for the fields in the fieldset '"+fieldSet.getName()+"' are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues.");
+                    return;
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java
new file mode 100644
index 00000000000..1df8c642750
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.logging.Level;
+
+/**
+ * Takes the fields and indexes that are of type rank filter, and stores those names on all rank profiles
+ * @author vegardh
+ *
+ */
+public class FilterFieldNames extends Processor {
+
+    public FilterFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField f : search.allFieldsList()) {
+            if (f.getRanking().isFilter()) {
+                filterField(f.getName());
+            }
+        }
+        for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) {
+            Set<String> filterFields = new LinkedHashSet<>();
+            findFilterFields(search, profile, filterFields);
+            for (Iterator<String> itr = filterFields.iterator(); itr.hasNext(); ) {
+                String fieldName = itr.next();
+                profile.filterFields().add(fieldName);
+                profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE, RankType.EMPTY);
+            }
+        }
+    }
+
+    private void filterField(String f) {
+        for (RankProfile rp : rankProfileRegistry.localRankProfiles(search)) {
+            rp.filterFields().add(f);
+        }
+    }
+
+    private void findFilterFields(Search search, RankProfile profile, Set<String> filterFields) {
+        for (Iterator<RankProfile.RankSetting> itr = profile.declaredRankSettingIterator(); itr.hasNext(); ) {
+            RankProfile.RankSetting setting = itr.next();
+            if (setting.getType().equals(RankProfile.RankSetting.Type.PREFERBITVECTOR) &&
+                    ((Boolean)setting.getValue()).booleanValue())
+            {
+                String fieldName = setting.getFieldName();
+                if (search.getField(fieldName) != null) {
+                    if (!profile.filterFields().contains(fieldName)) {
+                        filterFields.add(fieldName);
+                    }
+                } else {
+                    deployLogger.log(Level.WARNING, "For rank profile '" + profile.getName() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'");
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
new file mode 100644
index 00000000000..fa7725843b1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java
@@ -0,0 +1,222 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import java.util.logging.Level;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.PositionDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Makes implicitly defined summaries into explicit summaries
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class ImplicitSummaries extends Processor {
+
+    public ImplicitSummaries(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        DocumentSummary defaultSummary=search.getSummary("default");
+        if (defaultSummary==null) {
+            defaultSummary=new DocumentSummary("default");
+            search.addSummary(defaultSummary);
+        }
+
+        for (SDField field : search.allFieldsList()) {
+            collectSummaries(field,search);
+        }
+
+        for (DocumentSummary documentSummary : search.getSummaries().values()) {
+            documentSummary.purgeImplicits();
+        }
+    }
+
+    private void addSummaryFieldSources(SummaryField summaryField, SDField sdField) {
+        sdField.addSummaryFieldSources(summaryField);
+    }
+
+    private void collectSummaries(SDField field,Search search) {
+        SummaryField addedSummaryField=null;
+
+        // Implicit
+        String fieldName = field.getName();
+        SummaryField fieldSummaryField=field.getSummaryField(fieldName);
+        if (fieldSummaryField == null && field.doesSummarying()) {
+            fieldSummaryField=new SummaryField(fieldName, field.getDataType());
+            fieldSummaryField.setImplicit(true);
+            addSummaryFieldSources(fieldSummaryField, field);
+            fieldSummaryField.addDestination("default");
+            field.addSummaryField(fieldSummaryField);
+            addedSummaryField = fieldSummaryField;
+        }
+        if (fieldSummaryField != null) {
+            for (String dest : fieldSummaryField.getDestinations()) {
+                DocumentSummary summary = search.getSummary(dest);
+                if (summary != null) {
+                    summary.add(fieldSummaryField);
+                }
+            }
+        }
+
+        // Attribute prefetch
+        for (Attribute attribute : field.getAttributes().values()) {
+            if (attribute.getName().equals(fieldName)) {
+                if (addedSummaryField != null) {
+                    addedSummaryField.setTransform(SummaryTransform.ATTRIBUTE);
+                }
+                if (attribute.isPrefetch()) {
+                    addPrefetchAttribute(attribute, field, search);
+                }
+            }
+        }
+
+        // Position attributes
+        if (field.doesSummarying()) {
+            for (Attribute attribute : field.getAttributes().values()) {
+                if (!attribute.isPosition()) {
+                    continue;
+                }
+                DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(search);
+                attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)));
+                attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)));
+            }
+        }
+
+        // Explicits
+        for (SummaryField summaryField : field.getSummaryFields()) {
+            // Make sure we fetch from attribute here too
+            Attribute attribute=field.getAttributes().get(fieldName);
+            if (attribute!=null && summaryField.getTransform()==SummaryTransform.NONE) {
+                summaryField.setTransform(SummaryTransform.ATTRIBUTE);
+            }
+
+            if (isValid(summaryField,search)) {
+                addToDestinations(summaryField, search);
+            }
+        }
+
+    }
+
+    private DocumentSummary getOrCreateAttributePrefetchSummary(Search search) {
+        DocumentSummary summary = search.getSummary("attributeprefetch");
+        if (summary == null) {
+            summary = new DocumentSummary("attributeprefetch");
+            search.addSummary(summary);
+        }
+        return summary;
+    }
+
+
+    private void addPrefetchAttribute(Attribute attribute,SDField field,Search search) {
+        if (attribute.getPrefetchValue()==null) { // Prefetch by default - unless any summary makes this dynamic
+            // Check if there is an implicit dynamic definition
+            SummaryField fieldSummaryField=field.getSummaryField(attribute.getName());
+            if (fieldSummaryField!=null && fieldSummaryField.getTransform().isDynamic()) return;
+
+            // Check if an explicit class makes it dynamic (first is enough, as all must be the same, checked later)
+            SummaryField explicitSummaryField=search.getExplicitSummaryField(attribute.getName());
+            if (explicitSummaryField!=null && explicitSummaryField.getTransform().isDynamic()) return;
+        }
+
+        DocumentSummary summary = getOrCreateAttributePrefetchSummary(search);
+        SummaryField attributeSummaryField = new SummaryField(attribute.getName(), attribute.getDataType());
+        attributeSummaryField.addSource(attribute.getName());
+        attributeSummaryField.addDestination("attributeprefetch");
+        attributeSummaryField.setTransform(SummaryTransform.ATTRIBUTE);
+        summary.add(attributeSummaryField);
+    }
+
+    // Returns whether this is valid. Warns if invalid and ignorable. Throws if not ignorable.
+    private boolean isValid(SummaryField summaryField,Search search) {
+        if (summaryField.getTransform() == SummaryTransform.DISTANCE ||
+            summaryField.getTransform() == SummaryTransform.POSITIONS)
+        {
+            int sourceCount = summaryField.getSourceCount();
+            if (sourceCount != 1) {
+                throw newProcessException(search.getName(), summaryField.getName(),
+                                          "Expected 1 source field, got " + sourceCount + ".");
+            }
+            String sourceName = summaryField.getSingleSource();
+            if (search.getAttribute(sourceName) == null) {
+                throw newProcessException(search.getName(), summaryField.getName(),
+                                          "Summary source attribute '" + sourceName + "' not found.");
+            }
+            return true;
+        }
+
+        String fieldName = summaryField.getSourceField();
+        SDField sourceField = search.getField(fieldName);
+        if (sourceField == null) {
+            throw newProcessException(search, summaryField, "Source field '" + fieldName + "' does not exist.");
+        }
+        if (!sourceField.doesSummarying() &&
+            !summaryField.getTransform().equals(SummaryTransform.ATTRIBUTE) &&
+            !summaryField.getTransform().equals(SummaryTransform.GEOPOS))
+        {
+            // Summary transform attribute may indicate that the ilscript was rewritten to remove summary
+            // by another search that uses this same field in inheritance.
+            deployLogger.log(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField +
+                        " is not creating a summary value in its indexing statement");
+            return false;
+        }
+
+        if (summaryField.getTransform().isDynamic()
+            && summaryField.getName().equals(sourceField.getName())
+            && sourceField.doesAttributing()) {
+            Attribute attribute=sourceField.getAttributes().get(sourceField.getName());
+            if (attribute!=null) {
+                String destinations="document summary 'default'";
+                if (summaryField.getDestinations().size()>0) {
+                    destinations = "document summaries " + summaryField.getDestinations();
+                }
+                deployLogger.log(Level.WARNING, "Will fetch the disk summary value of " + sourceField + " in " + destinations +
+                        " since this summary field uses a dynamic summary value (snippet/bolding): Dynamic summaries and bolding " +
+                        "is not supported with summary values fetched from in-memory attributes yet. If you want to see partial updates " +
+                        "to this attribute, remove any bolding and dynamic snippeting from this field");
+                // Note: The dynamic setting has already overridden the attribute map setting,
+                // so we do not need to actually do attribute.setSummary(false) here
+                // Also, we can not do this, since it makes it impossible to fetch this attribute
+                // in another summary
+            }
+        }
+
+        return true;
+    }
+
+    private void addToDestinations(SummaryField summaryField,Search search) {
+        if (summaryField.getDestinations().size()==0) {
+            addToDestination("default",summaryField,search);
+        }
+        else {
+            for (String destinationName : summaryField.getDestinations())
+                addToDestination(destinationName,summaryField,search);
+        }
+    }
+
+    private void addToDestination(String destinationName,SummaryField summaryField,Search search) {
+        DocumentSummary destination=search.getSummary(destinationName);
+        if (destination==null) {
+            destination=new DocumentSummary(destinationName);
+            search.addSummary(destination);
+            destination.add(summaryField);
+        }
+        else {
+            SummaryField existingField=
+                destination.getSummaryField(summaryField.getName());
+            SummaryField merged=summaryField.mergeWith(existingField);
+            destination.add(merged);
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java
new file mode 100644
index 00000000000..b95f2469cd8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * This processor adds all implicit summary fields to all registered document summaries. If another field has already
+ * been registered with one of the implicit names, this processor will throw an {@link IllegalStateException}.
+ */
+public class ImplicitSummaryFields extends Processor {
+
+    public ImplicitSummaryFields(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (DocumentSummary docsum : search.getSummaries().values()) {
+            addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES));
+            addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES));
+        }
+    }
+
+    private void addField(DocumentSummary docsum, SummaryField field) {
+        if (docsum.getSummaryField(field.getName()) != null) {
+            throw new IllegalStateException("Summary class '" + docsum.getName() + "' uses reserved field name '" +
+                                            field.getName() + "'.");
+        }
+        docsum.add(field);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java
new file mode 100644
index 00000000000..2c18c58d3dc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Because of the way the parser works (allowing any token as identifier),
+ * it is not practical to limit the syntax of field names there, do it here.
+ * Important to disallow dash, has semantic in IL.
+ * @author vegardh
+ *
+ */
+public class IndexFieldNames extends Processor {
+
+    private static final String FIELD_NAME_REGEXP = "[a-zA-Z]\\w*";
+
+    public IndexFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (!field.getName().matches(FIELD_NAME_REGEXP) && !legalDottedPositionField(field)) {
+                fail(search, field, " Not a legal field name. Legal expression: " + FIELD_NAME_REGEXP);
+            }
+        }
+    }
+
+    /**
+     * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. For 6.0, rename
+     * to _position and _distance and delete this method.
+     * @param field an {@link com.yahoo.searchdefinition.document.SDField}
+     * @return true if allowed
+     */
+    private boolean legalDottedPositionField(SDField field) {
+        return field.getName().endsWith(".position") || field.getName().endsWith(".distance");
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java
new file mode 100644
index 00000000000..fecc5e2309d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+
+/**
+ * Fail if:
+ * 1) There are index: settings without explicit index names (name same as field name)
+ * 2) All the index-to indexes differ from the field name.
+ * @author vegardh
+ *
+ */
+public class IndexSettingsNonFieldNames extends Processor {
+
+    public IndexSettingsNonFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            boolean fieldNameUsed = false;
+            for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) {
+                String iName = (String)(i.next());
+                if (iName.equals(field.getName())) {
+                    fieldNameUsed = true;
+                }
+            }
+            if (!fieldNameUsed) {
+                for (Index index : field.getIndices().values()) {
+                    if (index.getName().equals(field.getName())) {
+                        throw new IllegalArgumentException("Error in " + field + " in " + search +
+                        ": When all index names differ from field name, index parameter settings must specify index name explicitly.");
+                    }
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java
new file mode 100644
index 00000000000..3148273ddc1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * @author balder
+ */
+public class IndexTo2FieldSet extends Processor {
+
+    public IndexTo2FieldSet(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            processField(field);
+        }
+    }
+    private void processField(SDField field) {
+        if (field.usesStructOrMap()) {
+            for (SDField subField : field.getStructFields()) {
+                processField(subField);
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
new file mode 100644
index 00000000000..c2e889f7993
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
@@ -0,0 +1,110 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * <p>This processor modifies all indexing scripts so that they input the value of the owning field by default. It also
+ * ensures that all fields used as input exist.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IndexingInputs extends Processor {
+
+    public IndexingInputs(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            ScriptExpression script = field.getIndexingScript();
+            if (script == null) {
+                continue;
+            }
+            String fieldName = field.getName();
+            script = (ScriptExpression)new DefaultToCurrentField(fieldName).convert(script);
+            script = (ScriptExpression)new EnsureInputExpression(fieldName).convert(script);
+            new VerifyInputExpression(search, field).visit(script);
+            field.setIndexingScript(script);
+        }
+    }
+
+    private static class DefaultToCurrentField extends ExpressionConverter {
+
+        final String fieldName;
+
+        DefaultToCurrentField(String fieldName) {
+            this.fieldName = fieldName;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            return exp instanceof InputExpression && ((InputExpression)exp).getFieldName() == null;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            return new InputExpression(fieldName);
+        }
+    }
+
+    private static class EnsureInputExpression extends ExpressionConverter {
+
+        final String fieldName;
+
+        EnsureInputExpression(String fieldName) {
+            this.fieldName = fieldName;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            return exp instanceof StatementExpression;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            if (exp.requiredInputType() != null) {
+                return new StatementExpression(new InputExpression(fieldName), exp);
+            } else {
+                return exp;
+            }
+        }
+    }
+
+    private class VerifyInputExpression extends ExpressionVisitor {
+
+        private final Search search;
+        private final SDField field;
+
+        public VerifyInputExpression(Search search, SDField field) {
+            this.search = search;
+            this.field = field;
+        }
+
+        @Override
+        protected void doVisit(Expression exp) {
+            if (!(exp instanceof InputExpression)) {
+                return;
+            }
+            SDDocumentType docType = search.getDocument();
+            String inputField = ((InputExpression)exp).getFieldName();
+            if (docType.getField(inputField) != null) {
+                return;
+            }
+            fail(search, field, "Indexing script refers to field '" + inputField + "' which does not exist " +
+                                          "in document type '" + docType.getName() + "'.");
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java
new file mode 100644
index 00000000000..55e7dabf9ba
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java
@@ -0,0 +1,141 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.*;
+
+/**
+ * <p>This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents
+ * any output expression from writing to any field except for the owning field. Finally, for <tt>SummaryExpression</tt>,
+ * this processor expands to write all appropriate summary fields.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IndexingOutputs extends Processor {
+
+    public IndexingOutputs(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            ScriptExpression script = field.getIndexingScript();
+            if (script == null) {
+                continue;
+            }
+            Set<String> summaryFields = new TreeSet<>();
+            findSummaryTo(search, field, summaryFields, summaryFields);
+            MyConverter converter = new MyConverter(search, field, summaryFields);
+            field.setIndexingScript((ScriptExpression)converter.convert(script));
+        }
+    }
+
+    public void findSummaryTo(Search search, SDField field, Set<String> dynamicSummary,
+                                     Set<String> staticSummary) {
+        Map<String, SummaryField> summaryFields = search.getSummaryFields(field);
+        if (summaryFields.isEmpty()) {
+            fillSummaryToFromField(field, dynamicSummary, staticSummary);
+        } else {
+            fillSummaryToFromSearch(search, field, summaryFields, dynamicSummary, staticSummary);
+        }
+    }
+
+    private void fillSummaryToFromSearch(Search search, SDField field, Map<String, SummaryField> summaryFields,
+                                                Set<String> dynamicSummary, Set<String> staticSummary) {
+        for (SummaryField summaryField : summaryFields.values()) {
+            fillSummaryToFromSummaryField(search, field, summaryField, dynamicSummary, staticSummary);
+        }
+    }
+
+    private void fillSummaryToFromSummaryField(Search search, SDField field, SummaryField summaryField, Set<String> dynamicSummary, Set<String> staticSummary) {
+        SummaryTransform summaryTransform = summaryField.getTransform();
+        String summaryName = summaryField.getName();
+        if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) {
+            // Avoid writing to summary fields that have more than a single input field, as that is handled by the
+            // summary rewriter in the search core.
+            return;
+        }
+        if (summaryTransform.isDynamic()) {
+            DataType fieldType = field.getDataType();
+            if (fieldType != DataType.URI && fieldType != DataType.STRING) {
+                warn(search, field, "Dynamic summaries are only supported for fields of type " +
+                                    "string, ignoring summary field '" + summaryField.getName() +
+                                    "' for sd field '" + field.getName() + "' of type " +
+                                    fieldType.getName() + ".");
+                return;
+            }
+            dynamicSummary.add(summaryName);
+        } else if (summaryTransform != SummaryTransform.ATTRIBUTE) {
+            staticSummary.add(summaryName);
+        }
+    }
+
+    private static void fillSummaryToFromField(SDField field, Set<String> dynamicSummary, Set<String> staticSummary) {
+        for (SummaryField summaryField : field.getSummaryFields()) {
+            String summaryName = summaryField.getName();
+            if (summaryField.getTransform().isDynamic()) {
+                dynamicSummary.add(summaryName);
+            } else {
+                staticSummary.add(summaryName);
+            }
+        }
+    }
+
+    private class MyConverter extends ExpressionConverter {
+
+        final Search search;
+        final Field field;
+        final Set<String> summaryFields;
+
+        MyConverter(Search search, Field field, Set<String> summaryFields) {
+            this.search = search;
+            this.field = field;
+            this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            if (!(exp instanceof OutputExpression)) {
+                return false;
+            }
+            String fieldName = ((OutputExpression)exp).getFieldName();
+            if (fieldName == null) {
+                return true; // inject appropriate field name
+            }
+            if (!fieldName.equals(field.getName())) {
+                fail(search, field, "Indexing expression '" + exp + "' attempts to write to a field other than '" +
+                                    field.getName() + "'.");
+            }
+            return false;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            List<Expression> ret = new LinkedList<>();
+            if (exp instanceof AttributeExpression) {
+                ret.add(new AttributeExpression(field.getName()));
+            } else if (exp instanceof IndexExpression) {
+                ret.add(new IndexExpression(field.getName()));
+            } else if (exp instanceof SummaryExpression) {
+                for (String fieldName : summaryFields) {
+                    ret.add(new SummaryExpression(fieldName));
+                }
+            } else {
+                throw new UnsupportedOperationException(exp.getClass().getName());
+            }
+            return new StatementExpression(ret);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
new file mode 100644
index 00000000000..af04deb5347
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java
@@ -0,0 +1,149 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class IndexingValidation extends Processor {
+
+    public IndexingValidation(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        VerificationContext ctx = new VerificationContext(new MyAdapter(search));
+        for (SDField field : search.allFieldsList()) {
+            ScriptExpression script = field.getIndexingScript();
+            try {
+                script.verify(ctx);
+                MyConverter converter = new MyConverter();
+                for (StatementExpression exp : script) {
+                    converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch
+                }
+            } catch (VerificationException e) {
+                fail(search, field, "For expression '" + e.getExpression() + "': " + e.getMessage());
+            }
+        }
+    }
+
+    private static class MyConverter extends ExpressionConverter {
+
+        final Set<String> outputs = new HashSet<>();
+        final Set<String> prevNames = new HashSet<>();
+
+        @Override
+        protected ExpressionConverter branch() {
+            MyConverter ret = new MyConverter();
+            ret.outputs.addAll(outputs);
+            ret.prevNames.addAll(prevNames);
+            return ret;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            if (exp instanceof OutputExpression) {
+                String fieldName = ((OutputExpression)exp).getFieldName();
+                if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) {
+                    throw new VerificationException(exp, "Attempting to assign conflicting values to field '" +
+                                                         fieldName + "'.");
+                }
+                outputs.add(fieldName);
+                prevNames.add(fieldName);
+            }
+            if (exp.createdOutputType() != null) {
+                prevNames.clear();
+            }
+            return false;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    private static class MyAdapter implements FieldTypeAdapter {
+
+        final Search search;
+
+        public MyAdapter(Search search) {
+            this.search = search;
+        }
+
+        @Override
+        public DataType getInputType(Expression exp, String fieldName) {
+            SDField field = search.getDocumentField(fieldName);
+            if (field == null) {
+                throw new VerificationException(exp, "Input field '" + fieldName + "' not found.");
+            }
+            return field.getDataType();
+        }
+
+        @Override
+        public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+            String fieldDesc;
+            DataType fieldType;
+            if (exp instanceof AttributeExpression) {
+                Attribute attribute = search.getAttribute(fieldName);
+                if (attribute == null) {
+                    throw new VerificationException(exp, "Attribute '" + fieldName + "' not found.");
+                }
+                fieldDesc = "attribute";
+                fieldType = attribute.getDataType();
+            } else if (exp instanceof IndexExpression) {
+                SDField field = search.getField(fieldName);
+                if (field == null) {
+                    throw new VerificationException(exp, "Index field '" + fieldName + "' not found.");
+                }
+                fieldDesc = "index field";
+                fieldType = field.getDataType();
+            } else if (exp instanceof SummaryExpression) {
+                SummaryField field = search.getSummaryField(fieldName);
+                if (field == null) {
+                    throw new VerificationException(exp, "Summary field '" + fieldName + "' not found.");
+                }
+                fieldDesc = "summary field";
+                fieldType = field.getDataType();
+            } else {
+                throw new UnsupportedOperationException();
+            }
+            if (!fieldType.isAssignableFrom(valueType) &&
+                !fieldType.isAssignableFrom(createCompatType(valueType)))
+            {
+                throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc +
+                                                     " '" + fieldName + "' which is " + fieldType.getName() + ".");
+            }
+        }
+
+        private static DataType createCompatType(DataType origType) {
+            if (origType instanceof ArrayDataType) {
+                return DataType.getArray(createCompatType(((ArrayDataType)origType).getNestedType()));
+            } else if (origType instanceof MapDataType) {
+                MapDataType mapType = (MapDataType)origType;
+                return DataType.getMap(createCompatType(mapType.getKeyType()),
+                                       createCompatType(mapType.getValueType()));
+            } else if (origType instanceof WeightedSetDataType) {
+                return DataType.getWeightedSet(createCompatType(((WeightedSetDataType)origType).getNestedType()));
+            } else if (origType == PositionDataType.INSTANCE) {
+                return DataType.LONG;
+            } else {
+                return origType;
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
new file mode 100644
index 00000000000..ff8be71f8c3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class IndexingValues extends Processor {
+
+    public IndexingValues(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (Field field : search.getDocument().fieldSet()) {
+            SDField sdField = (SDField)field;
+            if (!sdField.isExtraField()) {
+                new RequireThatDocumentFieldsAreImmutable(field).convert(sdField.getIndexingScript());
+            }
+        }
+    }
+
+    private class RequireThatDocumentFieldsAreImmutable extends ExpressionConverter {
+
+        final Field field;
+        Expression mutatedBy;
+
+        RequireThatDocumentFieldsAreImmutable(Field field) {
+            this.field = field;
+        }
+
+        @Override
+        public ExpressionConverter branch() {
+            return clone();
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            if (exp instanceof OutputExpression && mutatedBy != null) {
+                throw newProcessException(search, field,
+                                          "Indexing expression '" + mutatedBy + "' modifies the value of the " +
+                                          "document field '" + field.getName() + "'. This is no longer supported -- " +
+                                          "declare such fields outside the document.");
+            }
+            if (exp instanceof InputExpression && ((InputExpression)exp).getFieldName().equals(field.getName())) {
+                mutatedBy = null;
+            } else if (exp.createdOutputType() != null) {
+                mutatedBy = exp;
+            }
+            return false;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            throw new UnsupportedOperationException();
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java
new file mode 100644
index 00000000000..a29896ce3e8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Replaces the 'index' statement of all numerical fields to 'attribute' because we no longer support numerical indexes.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class IntegerIndex2Attribute extends Processor {
+
+    public IntegerIndex2Attribute(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) {
+                // Avoid changing for example RISE fields
+                if (field.getIndex(field.getName()) != null && !(field.getIndex(field.getName()).getType().equals(Index.Type.VESPA))) continue;
+                ScriptExpression script = field.getIndexingScript();
+                Set<String> attributeNames = new HashSet<>();
+                new MyVisitor(attributeNames).visit(script);
+                field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script));
+                warn(search, field, "Changed to attribute because numerical indexes (field has type "+field.getDataType().getName()+") is not currently supported." +
+                		" Index-only settings may fail. Ignore this warning for streaming search.");
+            }
+        }
+    }
+
+    private static class MyVisitor extends ExpressionVisitor {
+
+        final Set<String> attributeNames;
+
+        public MyVisitor(Set<String> attributeNames) {
+            this.attributeNames = attributeNames;
+        }
+
+        @Override
+        protected void doVisit(Expression exp) {
+            if (exp instanceof AttributeExpression) {
+                attributeNames.add(((AttributeExpression)exp).getFieldName());
+            }
+        }
+    }
+
+    private static class MyConverter extends ExpressionConverter {
+
+        final Set<String> attributeNames;
+
+        public MyConverter(Set<String> attributeNames) {
+            this.attributeNames = attributeNames;
+        }
+
+        @Override
+        protected boolean shouldConvert(Expression exp) {
+            return exp instanceof IndexExpression;
+        }
+
+        @Override
+        protected Expression doConvert(Expression exp) {
+            String indexName = ((IndexExpression)exp).getFieldName();
+            if (attributeNames.contains(indexName)) {
+                return null;
+            }
+            return new AttributeExpression(indexName);
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java
new file mode 100644
index 00000000000..e4e96bb51ab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+
+/**
+ * Expresses literal boosts in terms of extra indices with rank boost.
+ * One extra index named <i>indexname</i>_exact is added for each index having
+ * a fields with literal-boosts of zero or more (zero to support other
+ * rank profiles setting a literal boost). Complete boost values in to fields
+ * are translated to rank boosts to the implementation indices.
+ * These indices has no positional
+ * or phrase support and contains concatenated versions of each field value
+ * of complete-boosted fields indexed to <i>indexname</i>. A search for indexname
+ * will be rewritten to also search <i>indexname</i>_exaxt
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class LiteralBoost extends Processor {
+
+    public LiteralBoost(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    /** Adds extra search fields and indices to express literal boosts */
+    @Override
+    public void process() {
+        checkRankModifierRankType(search);
+        addLiteralBoostsToFields(search);
+        reduceFieldLiteralBoosts(search);
+    }
+
+    /** Checks if literal boost is given using rank: , and set the actual literal boost accordingly. */
+    private void checkRankModifierRankType(Search search) {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getLiteralBoost() > -1) continue; // Let explicit value take precedence
+            if (field.getRanking().isLiteral())
+                field.setLiteralBoost(100);
+        }
+    }
+
+    /**
+     * Ensures there are field boosts for all literal boosts mentioned in rank profiles.
+     * This is required because boost indices will only be generated by looking
+     * at field boosts
+     */
+    private void addLiteralBoostsToFields(Search search) {
+        Iterator i = matchingRankSettingsIterator(search, RankProfile.RankSetting.Type.LITERALBOOST);
+        while (i.hasNext()) {
+            RankProfile.RankSetting setting = (RankProfile.RankSetting)i.next();
+            SDField field = search.getField(setting.getFieldName());
+            if (field == null) continue;
+            if (field.getLiteralBoost() < 0)
+                field.setLiteralBoost(0);
+        }
+    }
+
+    private void reduceFieldLiteralBoosts(Search search) {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getLiteralBoost()<0) continue;
+            reduceFieldLiteralBoost(field,search);
+        }
+    }
+
+    private void reduceFieldLiteralBoost(SDField field,Search search) {
+        SDField literalField = addField(search, field, "literal",
+                "{ input " + field.getName() + " | tokenize | index " + field.getName() + "_literal; }",
+                "literal-boost");
+        literalField.setWeight(field.getWeight() + field.getLiteralBoost());
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java
new file mode 100644
index 00000000000..856b0cf3348
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Takes the aliases set on field by parser and sets them on correct Index or Attribute
+ * @author vegardh
+ *
+ */
+public class MakeAliases extends Processor {
+
+    public MakeAliases(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        List<String> usedAliases = new ArrayList<>();
+        for (SDField field : search.allFieldsList()) {
+            for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) {
+                String alias = e.getKey();
+                String name = e.getValue();
+                String errMsg = "For search '"+search.getName()+"': alias '"+alias+"' ";
+                if (search.existsIndex(alias)) {
+                    throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an index.");
+                }
+                if (search.getAttribute(alias)!=null) {
+                    throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an attribute.");
+                }
+                if (usedAliases.contains(alias)) {
+                    throw new IllegalArgumentException(errMsg+"specified more than once.");
+                }
+                usedAliases.add(alias);
+
+                Index index = field.getIndex(name);
+                Attribute attribute = field.getAttributes().get(name);
+                if (index != null) {
+                    index.addAlias(alias); // alias will be for index in this case, since it is the one used in a search
+                } else if (attribute != null && !field.doesIndexing()) {
+                    attribute.getAliases().add(alias);
+                } else {
+                    index = new Index(name);
+                    index.addAlias(alias);
+                    field.addIndex(index);
+                }
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java
new file mode 100644
index 00000000000..a9f013daa98
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * <p>All summary fields which are not attributes
+ * must currently be present in the default summary class,
+ * since the default summary class also defines the docsum.dat format.
+ * This processor adds any missing summaries to the default summary.
+ * When that is decoupled from the actual summaries returned, this
+ * processor can be removed. Note: the StreamingSummary also takes advantage of
+ * the fact that default is the superset.</p>
+ *
+ * <p>All other summary logic should work unchanged without this processing step
+ * except that IndexStructureValidator.validateSummaryFields must be changed to
+ * consider all summaries, not just the default, i.e change to
+ * if (search.getSummaryField(expr.getFieldName()) == null)</p>
+ *
+ * <p>This must be done after other summary processors.</p>
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public class MakeDefaultSummaryTheSuperSet extends Processor {
+
+    public MakeDefaultSummaryTheSuperSet(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        DocumentSummary defaultSummary=search.getSummary("default");
+        for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) {
+            if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue;
+            if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue;
+
+            defaultSummary.add(summaryField.clone());
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java
new file mode 100644
index 00000000000..f21670e6f78
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.Matching.Type;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Warn on inconsistent match settings for any index
+ *
+ * @author vegardh
+ */
+public class MatchConsistency extends Processor {
+
+    public MatchConsistency(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        Map<String, Matching.Type> types = new HashMap<>();
+        for (SDField field : search.allFieldsList()) {
+            new MyVisitor(search, field, types).visit(field.getIndexingScript());
+        }
+    }
+
+    void checkMatching(Search search, SDField field, Map<String, Type> types, String indexTo) {
+        Type prevType = types.get(indexTo);
+        if (prevType == null) {
+            types.put(indexTo, field.getMatching().getType());
+        } else if (!field.getMatching().getType().equals(prevType)) {
+            warn(search, field, "The matching type for index '" + indexTo + "' (got " + field.getMatching().getType() +
+                                ") is inconsistent with that given for the same index in a previous field (had " +
+                                prevType + ").");
+        }
+    }
+
+    private class MyVisitor extends ExpressionVisitor {
+
+        final Search search;
+        final SDField field;
+        final Map<String, Type> types;
+
+        public MyVisitor(Search search, SDField field, Map<String, Type> types) {
+            this.search = search;
+            this.field = field;
+            this.types = types;
+        }
+
+        @Override
+        protected void doVisit(Expression exp) {
+            if (exp instanceof IndexExpression) {
+                checkMatching(search, field, types, ((IndexExpression)exp).getFieldName());
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java
new file mode 100644
index 00000000000..efceeca0af0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java
@@ -0,0 +1,92 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Validates the match phase settings for all registered rank profiles.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class MatchPhaseSettingsValidator extends Processor {
+
+    public MatchPhaseSettingsValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) {
+            RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings();
+            if (settings != null) {
+                validateMatchPhaseSettings(rankProfile, settings);
+            }
+        }
+    }
+
+    private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) {
+        String attributeName = settings.getAttribute();
+        new AttributeValidator(search.getName(), rankProfile.getName(),
+                search.getAttribute(attributeName), attributeName).validate();
+    }
+
+    public static class AttributeValidator {
+
+        private final String searchName;
+        private final String rankProfileName;
+        protected final Attribute attribute;
+        private final String attributeName;
+
+        public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) {
+            this.searchName = searchName;
+            this.rankProfileName = rankProfileName;
+            this.attribute = attribute;
+            this.attributeName = attributeName;
+        }
+
+        public void validate() {
+            validateThatAttributeExists();
+            validateThatAttributeIsSingleNumeric();
+            validateThatAttributeIsFastSearch();
+        }
+
+        protected void validateThatAttributeExists() {
+            if (attribute == null) {
+                failValidation("does not exists");
+            }
+        }
+
+        protected void validateThatAttributeIsSingleNumeric() {
+            if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) ||
+                 attribute.getType().equals(Attribute.Type.STRING) ||
+                 attribute.getType().equals(Attribute.Type.PREDICATE))
+            {
+                failValidation("must be single value numeric, but it is '"
+                        + attribute.getDataType().getName() + "'");
+            }
+        }
+
+        protected void validateThatAttributeIsFastSearch() {
+            if (!attribute.isFastSearch()) {
+                failValidation("must be fast-search, but it is not");
+            }
+        }
+
+        protected void failValidation(String what) {
+            throw new IllegalArgumentException(createMessagePrefix() + what);
+        }
+
+        public String getValidationType() { return "match-phase"; }
+
+        private String createMessagePrefix() {
+            return "In search definition '" + searchName +
+                    "', rank-profile '" + rankProfileName +
+                    "': " + getValidationType() + " attribute '" + attributeName + "' ";
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java
new file mode 100644
index 00000000000..61a11fd3a4e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.processing.multifieldresolver.IndexCommandResolver;
+import com.yahoo.searchdefinition.processing.multifieldresolver.RankTypeResolver;
+import com.yahoo.searchdefinition.processing.multifieldresolver.StemmingResolver;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Ensures that there are no conflicting types or field settings
+ * in multifield indices, either by changing settings or by splitting
+ * conflicting fields in multiple ones with different settings.
+ *
+ * @author bratseth
+ */
+public class MultifieldIndexHarmonizer extends Processor {
+
+    /** A map from index names to a List of fields going to that index */
+    private Map<String,List<SDField>> indexToFields=new java.util.HashMap<>();
+
+    public MultifieldIndexHarmonizer(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        populateIndexToFields(search);
+        resolveAllConflicts(search);
+    }
+
+    private void populateIndexToFields(Search search) {
+        for (SDField field : search.allFieldsList() ) {
+            if (!field.doesIndexing()) {
+                continue;
+            }
+            for (Iterator j = field.getFieldNameAsIterator(); j.hasNext();) {
+                String indexName = (String)j.next();
+                addIndexField(indexName, field);
+            }
+        }
+    }
+
+    private void addIndexField(String indexName,SDField field) {
+        List<SDField> fields=indexToFields.get(indexName);
+        if (fields==null) {
+            fields=new java.util.ArrayList<>();
+            indexToFields.put(indexName,fields);
+        }
+        fields.add(field);
+    }
+
+    private void resolveAllConflicts(Search search) {
+        for (Map.Entry<String, List<SDField>> entry : indexToFields.entrySet()) {
+            String indexName = entry.getKey();
+            List<SDField> fields = entry.getValue();
+            if (fields.size() == 1) continue; // It takes two to make a conflict
+            resolveConflicts(indexName, fields, search);
+        }
+    }
+
+    /**
+     * Resolves all conflicts for one index
+     *
+     * @param indexName the name of the index in question
+     * @param fields all the fields indexed to this index
+     * @param search the search definition having this
+     */
+    private void resolveConflicts(String indexName,List<SDField> fields,Search search) {
+        new StemmingResolver(indexName, fields, search, deployLogger).resolve();
+        new IndexCommandResolver(indexName, fields, search, deployLogger).resolve();
+        new RankTypeResolver(indexName, fields, search, deployLogger).resolve();
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java
new file mode 100644
index 00000000000..8e5122cc158
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of "gram" matching - splitting the incoming text and the queries into
+ * n-grams for matching. This will also validate the gram settings.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class NGramMatch extends Processor {
+
+    public static final int DEFAULT_GRAM_SIZE = 2;
+
+    public NGramMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getMatching().getType().equals(Matching.Type.GRAM))
+                implementGramMatch(search, field);
+            else if (field.getMatching().getGramSize()>=0)
+                throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'");
+        }
+    }
+
+    private void implementGramMatch(Search search, SDField field) {
+        if (field.doesAttributing())
+            throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' not 'attribute' in indexing");
+
+        int n = field.getMatching().getGramSize();
+        if (n<0)
+            n=DEFAULT_GRAM_SIZE; // not set - use default gram size
+        if (n==0)
+            throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1");
+        field.getNormalizing().inferCodepoint();
+        field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing
+        field.addQueryCommand("ngram " + n);
+        field.setIndexingScript((ScriptExpression)new MyProvider(search, n).convert(field.getIndexingScript()));
+    }
+
+    private static class MyProvider extends TypedTransformProvider {
+
+        final int ngram;
+
+        MyProvider(Search search, int ngram) {
+            super(NGramExpression.class, search);
+            this.ngram = ngram;
+        }
+
+        @Override
+        protected boolean requiresTransform(Expression exp, DataType fieldType) {
+            return exp instanceof OutputExpression;
+        }
+
+        @Override
+        protected Expression newTransform(DataType fieldType) {
+            Expression exp = new NGramExpression(null, ngram);
+            if (fieldType instanceof CollectionDataType) {
+                exp = new ForEachExpression(exp);
+            }
+            return exp;
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
new file mode 100644
index 00000000000..078e9400ec5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.indexinglanguage.ExpressionOptimizer;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Run ExpressionOptimizer on all scripts, to get rid of expressions that have no effect.
+ */
+public class OptimizeIlscript extends Processor {
+    public OptimizeIlscript(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            ScriptExpression script = field.getIndexingScript();
+            if (script == null) {
+                continue;
+            }
+            field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script));
+            if (!field.getIndexingScript().toString().equals(script.toString())) {
+                warn(search, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString());
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java
new file mode 100644
index 00000000000..a156a98a584
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java
@@ -0,0 +1,138 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.IntegerFieldValue;
+import com.yahoo.document.datatypes.LongFieldValue;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Validates the predicate fields.
+ *
+ * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a>
+ */
+public class PredicateProcessor extends Processor {
+
+    public PredicateProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getDataType() == DataType.PREDICATE) {
+                if (field.doesIndexing()) {
+                    fail(search, field, "Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded.");
+                }
+                if (field.doesAttributing()) {
+                    Attribute attribute = field.getAttributes().get(field.getName());
+                    for (Index index : field.getIndices().values()) {
+                        BooleanIndexDefinition booleanDefinition = index.getBooleanIndexDefiniton();
+                        if (booleanDefinition == null || !booleanDefinition.hasArity()) {
+                            fail(search, field, "Missing arity value in predicate field.");
+                        }
+                        if (booleanDefinition.getArity() < 2) {
+                            fail(search, field, "Invalid arity value in predicate field, must be greater than 1.");
+                        }
+                        double threshold = booleanDefinition.getDensePostingListThreshold();
+                        if (threshold <= 0 || threshold > 1) {
+                            fail(search, field, "Invalid dense-posting-list-threshold value in predicate field. " +
+                                    "Value must be in range (0..1].");
+                        }
+
+                        attribute.setArity(booleanDefinition.getArity());
+                        attribute.setLowerBound(booleanDefinition.getLowerBound());
+                        attribute.setUpperBound(booleanDefinition.getUpperBound());
+
+                        attribute.setDensePostingListThreshold(threshold);
+                        addPredicateOptimizationIlScript(field, booleanDefinition);
+                    }
+                    DocumentSummary summary = search.getSummary("attributeprefetch");
+                    if (summary != null) {
+                        summary.remove(attribute.getName());
+                    }
+                    for (SummaryField summaryField : search.getSummaryFields(field).values()) {
+                        summaryField.setTransform(SummaryTransform.NONE);
+                    }
+                }
+            } else if (field.getDataType().getPrimitiveType() == DataType.PREDICATE) {
+                fail(search, field, "Collections of predicates are not allowed.");
+            } else if (field.getDataType() == DataType.RAW && field.doesIndexing()) {
+                fail(search, field, "Indexing of RAW fields are not supported. If you are using RAW fields for boolean search, use predicate data type instead.");
+            } else {
+                // if field is not a predicate, disallow predicate-related index parameters
+                for (Index index : field.getIndices().values()) {
+                    if (index.getBooleanIndexDefiniton() != null) {
+                        BooleanIndexDefinition def = index.getBooleanIndexDefiniton();
+                        if (def.hasArity()) {
+                            fail(search, field, "Arity parameter is used only for predicate type fields.");
+                        } else if (def.hasLowerBound() || def.hasUpperBound()) {
+                            fail(search, field, "Parameters lower-bound and upper-bound are used only for predicate type fields.");
+                        } else if (def.hasDensePostingListThreshold()) {
+                            fail(search, field, "Parameter dense-posting-list-threshold is used only for predicate type fields.");
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private void addPredicateOptimizationIlScript(SDField field, BooleanIndexDefinition booleanIndexDefiniton) {
+        Expression script = field.getIndexingScript();
+        if (script == null) {
+            return;
+        }
+        script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script);
+
+        ExpressionConverter converter = new PredicateOutputTransformer(search);
+        field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script)));
+    }
+
+    private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) {
+        List<Expression> expressions = new ArrayList<>();
+        expressions.add(new SetValueExpression(new IntegerFieldValue(options.getArity())));
+        expressions.add(new SetVarExpression("arity"));
+        if (options.hasLowerBound()) {
+            expressions.add(new SetValueExpression(new LongFieldValue(options.getLowerBound())));
+            expressions.add(new SetVarExpression("lower_bound"));
+        }
+        if (options.hasUpperBound()) {
+            expressions.add(new SetValueExpression(new LongFieldValue(options.getUpperBound())));
+            expressions.add(new SetVarExpression("upper_bound"));
+        }
+        return new StatementExpression(expressions);
+    }
+
+    private static class PredicateOutputTransformer extends TypedTransformProvider {
+
+        PredicateOutputTransformer(Search search) {
+            super(OptimizePredicateExpression.class, search);
+        }
+
+        @Override
+        protected boolean requiresTransform(Expression exp, DataType fieldType) {
+            return exp instanceof OutputExpression && fieldType == DataType.PREDICATE;
+        }
+
+        @Override
+        protected Expression newTransform(DataType fieldType) {
+            return new OptimizePredicateExpression();
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
new file mode 100644
index 00000000000..74f359602bd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Executor of processors. This defines the right order of processor execution.
+ *
+ * @author bratseth
+ */
+public class Processing {
+
+    /**
+     * Runs all search processors on the given {@link Search} object. These will modify the search object, <b>possibly
+     * exchanging it with another</b>, as well as its document types.
+     *
+     * @param search The search to process.
+     * @param deployLogger The log to log messages and warnings for application deployment to
+     * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
+     * @param queryProfiles The query profiles contained in the application this search is part of.
+     */
+    public static void process(Search search,
+                               DeployLogger deployLogger,
+                               RankProfileRegistry rankProfileRegistry,
+                               QueryProfiles queryProfiles) {
+        search.process();
+        new UrlFieldValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new BuiltInFieldSets(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SearchMustHaveDocument(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new ReservedDocumentNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new IndexFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new IntegerIndex2Attribute(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new MakeAliases(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SetLanguage(search, deployLogger, rankProfileRegistry, queryProfiles).process(); // Needs to come before UriHack, see ticket 6405470
+        new UriHack(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new LiteralBoost(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new IndexTo2FieldSet(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new TagType(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new IndexingInputs(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new OptimizeIlscript(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new ValidateFieldWithIndexSettingsCreatesIndex(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new AttributesImplicitWord(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new CreatePositionZCurve(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new WordMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new DeprecateAttributePrefetch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new ImplicitSummaries(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new ImplicitSummaryFields(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SummaryConsistency(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SummaryNamesFieldCollisions(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new MakeDefaultSummaryTheSuperSet(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new Bolding(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new AttributeProperties(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SetRankTypeEmptyOnFilters(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new IndexSettingsNonFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SummaryDynamicStructsArrays(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new StringSettingsOnNonStringFields(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new ExactMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new NGramMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new TextMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new MultifieldIndexHarmonizer(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new FilterFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new MatchConsistency(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new ValidateFieldTypes(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new DisallowComplexMapAndWsetKeyTypes(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new SortingSettings(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new FieldSetValidity(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new AddExtraFieldsToDocument(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new PredicateProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new MatchPhaseSettingsValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new DiversitySettingsValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new TensorFieldProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new RankProfileTypeSettingsProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+
+        // These two should be last.
+        new IndexingValidation(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+        new IndexingValues(search, deployLogger, rankProfileRegistry, queryProfiles).process();
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
new file mode 100644
index 00000000000..a3e0484b2fd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java
@@ -0,0 +1,131 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Abstract superclass of all search definition processors.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public abstract class Processor {
+
+    protected final Search search;
+    protected DeployLogger deployLogger;
+    protected final RankProfileRegistry rankProfileRegistry;
+    protected final QueryProfiles queryProfiles;
+
+    /**
+     * Base constructor
+     * @param search the search to process
+     * @param deployLogger Logger du use when logging deploy output.
+     * @param rankProfileRegistry Registry with all rank profiles, used for lookup and insertion.
+     * @param queryProfiles The query profiles contained in the application this search is part of.
+     */
+    public Processor(Search search, DeployLogger deployLogger,
+                     RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        this.search = search;
+        this.deployLogger = deployLogger;
+        this.rankProfileRegistry = rankProfileRegistry;
+        this.queryProfiles = queryProfiles;
+    }
+
+    /**
+     * Processes the input search definition by <b>modifying</b> the input search and its documents, and returns the
+     * input search definition.
+     */
+    public abstract void process();
+
+    /**
+     * Convenience method for adding a no-strings-attached implementation field for a regular field
+     *
+     * @param search       the search definition in question
+     * @param field        the field to add an implementation field for
+     * @param suffix       the suffix of the added implementation field (without the underscore)
+     * @param indexing     the indexing statement of the field
+     * @param queryCommand the query command of the original field, or null if none
+     * @return the implementation field which is added to the search
+     */
+    protected SDField addField(Search search, SDField field, String suffix, String indexing, String queryCommand) {
+        SDField implementationField = search.getField(field.getName() + "_" + suffix);
+        if (implementationField != null) {
+            deployLogger.log(Level.WARNING, "Implementation field " + implementationField + " added twice");
+        } else {
+            implementationField = new SDField(search.getDocument(), field.getName() + "_" + suffix, DataType.STRING);
+        }
+        implementationField.setRankType(RankType.EMPTY);
+        implementationField.setStemming(Stemming.NONE);
+        implementationField.getNormalizing().inferCodepoint();
+        implementationField.parseIndexingScript(indexing);
+        for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) {
+            String indexName = (String)i.next();
+            String implementationIndexName = indexName + "_" + suffix;
+            Index implementationIndex = new Index(implementationIndexName);
+            search.addIndex(implementationIndex);
+        }
+        if (queryCommand != null) {
+            field.addQueryCommand(queryCommand);
+        }
+        search.addExtraField(implementationField);
+        search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, implementationField.getName());
+        return implementationField;
+    }
+
+    /**
+     * Returns an iterator of all the rank settings with given type in all the rank  profiles in this search
+     * definition.
+     */
+    protected Iterator<RankProfile.RankSetting> matchingRankSettingsIterator(
+            Search search, RankProfile.RankSetting.Type type)
+    {
+        List<RankProfile.RankSetting> someRankSettings = new java.util.ArrayList<>();
+
+        for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) {
+            for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) {
+                RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next();
+                if (setting.getType().equals(type)) {
+                    someRankSettings.add(setting);
+                }
+            }
+        }
+        return someRankSettings.iterator();
+    }
+
+    protected String formatError(String searchName, String fieldName, String msg) {
+        return "For search '" + searchName + "', field '" + fieldName + "': " + msg;
+    }
+
+    protected RuntimeException newProcessException(String searchName, String fieldName, String msg) {
+        return new IllegalArgumentException(formatError(searchName, fieldName, msg));
+    }
+
+    protected RuntimeException newProcessException(Search search, Field field, String msg) {
+        return newProcessException(search.getName(), field.getName(), msg);
+    }
+
+    public void fail(Search search, Field field, String msg) {
+        throw newProcessException(search, field, msg);
+    }
+
+    protected void warn(String searchName, String fieldName, String msg) {
+        String fullMsg = formatError(searchName, fieldName, msg);
+        deployLogger.log(Level.WARNING, fullMsg);
+    }
+
+    protected void warn(Search search, Field field, String msg) {
+        warn(search.getName(), field.getName(), msg);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java
new file mode 100644
index 00000000000..74bf2d35bac
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ReservedDocumentNames extends Processor {
+
+    private static final Set<String> RESERVED_NAMES = new HashSet<>();
+    static {
+        for (SDDocumentType dataType : SDDocumentType.VESPA_DOCUMENT.getTypes()) {
+            RESERVED_NAMES.add(dataType.getName());
+        }
+    }
+
+    public ReservedDocumentNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        String docName = search.getDocument().getName();
+        if (RESERVED_NAMES.contains(docName)) {
+            throw new IllegalArgumentException("For search '" + search.getName() + "': Document name '" + docName +
+                                               "' is reserved.");
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java
new file mode 100644
index 00000000000..5b45aecc257
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java
@@ -0,0 +1,28 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * A search must have a document definition of the same name inside of it, otherwise crashes may occur as late as
+ * during feeding
+ * @author vegardh
+ *
+ */
+public class SearchMustHaveDocument extends Processor {
+
+    public SearchMustHaveDocument(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        if (search.getDocument()==null) {
+            throw new IllegalArgumentException("For search '" + search.getName() + "': A search specification must have an equally named document inside of it.");
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
new file mode 100644
index 00000000000..764bb1602a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Check that no text field appears before a field that sets language.
+ *
+ * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a>
+ */
+public class SetLanguage extends Processor {
+
+    public SetLanguage(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        List<String> textFieldsWithoutLanguage = new ArrayList<>();
+
+        for (SDField field : search.allFieldsList()) {
+            if (fieldMustComeAfterLanguageSettingField(field)) {
+                textFieldsWithoutLanguage.add(field.getName());
+            }
+            if (field.containsExpression(SetLanguageExpression.class) && !textFieldsWithoutLanguage.isEmpty()) {
+                StringBuffer fieldString = new StringBuffer();
+                for (String fieldName : textFieldsWithoutLanguage) {
+                    fieldString.append(fieldName).append(" ");
+                }
+                warn(search, field, "Field '" + field.getName() + "' sets the language for this document, " +
+                        "and should be defined as the first field in the searchdefinition. If you have both header and body fields, this field "+
+                        "should be header, if you require it to affect subsequent header fields and/or any body fields. " +
+                        "Preceding text fields that will not have their language set: " +
+                        fieldString.toString() +
+                        " (This warning is omitted for any subsequent fields that also do set_language.)");
+                return;
+            }
+        }
+    }
+
+    private boolean fieldMustComeAfterLanguageSettingField(SDField field) {
+        return (!field.containsExpression(SetLanguageExpression.class) &&
+                (field.getDataType() == DataType.STRING));
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java
new file mode 100644
index 00000000000..bc676b64a71
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * All rank: filter fields should have rank type empty.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SetRankTypeEmptyOnFilters extends Processor {
+
+    public SetRankTypeEmptyOnFilters(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getRanking().isFilter()) {
+                field.setRankType(RankType.EMPTY);
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java
new file mode 100644
index 00000000000..6e77f48b4a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Sorting;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Validate conflicting settings for sorting
+ * @author vegardh
+ *
+ */
+public class SortingSettings extends Processor {
+
+    public SortingSettings(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            for (Attribute attribute : field.getAttributes().values()) {
+                Sorting sorting = attribute.getSorting();
+                if (sorting.getFunction()!=Sorting.Function.UCA) {
+                    if (sorting.getStrength()!=null && sorting.getStrength()!=Sorting.Strength.PRIMARY) {
+                        warn(search, field, "Sort strength only works for sort function 'uca'.");
+                    }
+                    if (sorting.getLocale()!=null && !"".equals(sorting.getLocale())) {
+                        warn(search, field, "Sort locale only works for sort function 'uca'.");
+                    }
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java
new file mode 100644
index 00000000000..48c9af5556a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+public class StringSettingsOnNonStringFields extends Processor {
+
+    public StringSettingsOnNonStringFields(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (!doCheck(field)) continue;
+            if (field.getMatching().isTypeUserSet()) {
+                warn(search, field, "Matching type "+field.getMatching().getType()+" is only allowed for string fields.");
+            }
+            if (field.getRanking().isLiteral()) {
+                warn(search, field, "Rank type literal only applies to string fields");
+            }
+        }
+    }
+
+    private boolean doCheck(SDField field) {
+        if (field.getDataType() instanceof NumericDataType) return true;
+        if (field.getDataType() instanceof CollectionDataType) {
+            if (((CollectionDataType)field.getDataType()).getNestedType() instanceof NumericDataType) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java
new file mode 100644
index 00000000000..20eeb5f0810
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.WeightedSetDataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Ensure that summary field transforms for fields having the same name
+ * are consistent across summary classes
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SummaryConsistency extends Processor {
+
+    public SummaryConsistency(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (DocumentSummary summary : search.getSummaries().values()) {
+            if (summary.getName().equals("default")) continue;
+            for (SummaryField summaryField : summary.getSummaryFields() ) {
+                assertConsistency(summaryField,search);
+                makeAttributeTransformIfAppropriate(summaryField,search);
+            }
+        }
+    }
+
+    /** If the source is an attribute, make this use the attribute transform */
+    private void makeAttributeTransformIfAppropriate(SummaryField summaryField,Search search) {
+        if (summaryField.getTransform()!=SummaryTransform.NONE) return;
+        Attribute attribute=search.getAttribute(summaryField.getSingleSource());
+        if (attribute==null) return;
+        summaryField.setTransform(SummaryTransform.ATTRIBUTE);
+    }
+
+    private void assertConsistency(SummaryField summaryField,Search search) {
+        SummaryField existingDefault=search.getSummary("default").getSummaryField(summaryField.getName()); // Compare to default
+        if (existingDefault!=null) {
+            assertConsistentTypes(existingDefault,summaryField);
+            makeConsistentWithDefaultOrThrow(existingDefault,summaryField);
+        }
+        else {
+            // If no default, compare to whichever definition of the field
+            SummaryField existing=search.getExplicitSummaryField(summaryField.getName());
+            if (existing==null) return;
+            assertConsistentTypes(existing,summaryField);
+            makeConsistentOrThrow(existing,summaryField,search);
+        }
+    }
+
+    private void assertConsistentTypes(SummaryField field1,SummaryField field2) {
+        if (field1.getDataType() instanceof WeightedSetDataType && field2.getDataType() instanceof WeightedSetDataType &&
+            ((WeightedSetDataType)field1.getDataType()).getNestedType().equals(((WeightedSetDataType)field2.getDataType()).getNestedType()))
+            return; // Disregard create-if-nonexistent and create-if-zero distinction
+        if ( ! field1.getDataType().equals(field2.getDataType()))
+            throw new IllegalArgumentException(field1.toLocateString() + " is inconsistent with " + field2.toLocateString() + ": All declarations of the same summary field must have the same type");
+    }
+
+    private void makeConsistentOrThrow(SummaryField field1, SummaryField field2,Search search) {
+        if (field2.getTransform()==SummaryTransform.ATTRIBUTE && field1.getTransform()==SummaryTransform.NONE) {
+            Attribute attribute=search.getAttribute(field1.getName());
+            if (attribute != null) {
+                field1.setTransform(SummaryTransform.ATTRIBUTE);
+            }
+        }
+
+        if (field2.getTransform().equals(SummaryTransform.NONE)) {
+            field2.setTransform(field1.getTransform());
+        }
+        else { // New field sets an explicit transform - must be the same
+            assertEqualTransform(field1,field2);
+        }
+    }
+    private void makeConsistentWithDefaultOrThrow(SummaryField defaultField,SummaryField newField) {
+        if (newField.getTransform().equals(SummaryTransform.NONE)) {
+            newField.setTransform(defaultField.getTransform());
+        }
+        else { // New field sets an explicit transform - must be the same
+            assertEqualTransform(defaultField,newField);
+        }
+    }
+
+
+    private void assertEqualTransform(SummaryField field1,SummaryField field2) {
+        if (!field2.getTransform().equals(field1.getTransform())) {
+            throw new IllegalArgumentException("Conflicting summary transforms. " + field2 +" is already defined as " +
+                                       field1 + ". A field with the same name " +
+                                       "can not have different transforms in different summary classes");
+        }
+    }
+
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java
new file mode 100644
index 00000000000..f97d5345cab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Fail if:
+ * An SD field explicitly says summary:dynamic , but the field is wset, array or struct.
+ * If there is an explicitly defined summary class, saying dynamic in one of its summary
+ * fields is always legal.
+ * @author vegardh
+ *
+ */
+public class SummaryDynamicStructsArrays extends Processor {
+
+    public SummaryDynamicStructsArrays(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            DataType type = field.getDataType();
+            if (type instanceof ArrayDataType || type instanceof WeightedSetDataType
+                    || type instanceof StructDataType) {
+                for (SummaryField sField : field.getSummaryFields()) {
+                    if (sField.getTransform().equals(SummaryTransform.DYNAMICTEASER)) {
+                        throw new IllegalArgumentException("For field '"+field.getName()+"': dynamic summary is illegal " +
+                                        "for fields of type struct, array or weighted set. Use an explicit summary class with explicit summary fields sourcing from" +
+                                        " the array/struct/weighted set.");
+                    }
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java
new file mode 100644
index 00000000000..0d76ada0d52
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryTransform;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Verifies that the source fields actually refers to a valid field.
+ *
+ * @author balder
+ *
+ */
+public class SummaryFieldsMustHaveValidSource extends Processor {
+    SummaryFieldsMustHaveValidSource(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+    @Override
+    public void process() {
+        for (DocumentSummary summary : search.getSummaries().values()) {
+            for (SummaryField summaryField : summary.getSummaryFields()) {
+                if (summaryField.getSources().isEmpty()) {
+                    if ((summaryField.getTransform() != SummaryTransform.RANKFEATURES) &&
+                        (summaryField.getTransform() != SummaryTransform.SUMMARYFEATURES))
+                    {
+                        verifySource(summaryField.getName(), summaryField, summary);
+                    }
+                } else if (summaryField.getSourceCount() == 1) {
+                    verifySource(summaryField.getSingleSource(), summaryField, summary);
+                } else {
+                    for (SummaryField.Source source : summaryField.getSources()) {
+                        if ( ! source.getName().equals(summaryField.getName()) ) {
+                            verifySource(source.getName(), summaryField, summary);
+                        }
+                    }
+                }
+            }
+        }
+
+    }
+
+    private boolean isValid(String source, SummaryField summaryField, DocumentSummary summary) {
+        return  isDocumentField(source) ||
+                (isNotInThisSummaryClass(summary, source) && isSummaryField(source)) ||
+                (isInThisSummaryClass(summary, source) && !source.equals(summaryField.getName())) ||
+                ("documentid".equals(source));
+    }
+
+    private void verifySource(String source, SummaryField summaryField, DocumentSummary summary) {
+        if ( ! isValid(source, summaryField, summary) ) {
+            throw new IllegalArgumentException("For search '" + search.getName() + "', summary class '" + summary.getName() + "'," +
+                    " summary field '" + summaryField.getName() + "': there is no valid source '" + source + "'.");
+        }
+    }
+
+    private boolean isNotInThisSummaryClass(DocumentSummary summary, String name) {
+        return summary.getSummaryField(name) == null;
+    }
+    private boolean isInThisSummaryClass(DocumentSummary summary, String name) {
+        return summary.getSummaryField(name) != null;
+    }
+    private boolean isDocumentField(String name) {
+        return search.getField(name) != null;
+    }
+
+    private boolean isSummaryField(String name) {
+        return search.getSummaryField(name) != null;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java
new file mode 100644
index 00000000000..8d99611d697
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.documentmodel.SummaryField.Source;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Verifies that equally named summary fields in different summary classes don't use different fields for source.
+ * The summarymap config doesn't model this.
+ *
+ * @author vegardh
+ *
+ */
+public class SummaryNamesFieldCollisions extends Processor {
+
+    public SummaryNamesFieldCollisions(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>();
+        for (DocumentSummary summary : search.getSummaries().values()) {
+            if ("default".equals(summary.getName())) continue;
+            for (SummaryField summaryField : summary.getSummaryFields() ) {
+                if (summaryField.isImplicit()) continue;
+                Pair<String, String> prevClassAndSource = fieldToClassAndSource.get(summaryField.getName());
+                for (Source source : summaryField.getSources()) {
+                    if (prevClassAndSource!=null) {
+                        String prevClass = prevClassAndSource.getFirst();
+                        String prevSource = prevClassAndSource.getSecond();
+                        if (!prevClass.equals(summary.getName())) {
+                            if (!prevSource.equals(source.getName())) {
+                                throw new IllegalArgumentException("For search '"+search.getName()+"', summary class '"+summary.getName()+"'," +
+                                		" summary field '"+summaryField.getName()+"':" +
+                                		" Can not use source '"+source.getName()+"' for this summary field, an equally named field in summary class '" +
+                                        prevClass + "' uses a different source: '"+prevSource+"'.");
+                            }
+                        }
+                    } else {
+                        fieldToClassAndSource.put(summaryField.getName(), new Pair<>(summary.getName(), source.getName()));
+                    }
+                }
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java
new file mode 100644
index 00000000000..3e211f4f632
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of the tag datatype
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class TagType extends Processor {
+
+    public TagType(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) {
+                implementTagType(field);
+            }
+        }
+    }
+
+    private void implementTagType(SDField field) {
+        field.setDataType(DataType.getWeightedSet(DataType.STRING,true,true));
+        // Don't set matching and ranking if this field is not attribute nor index
+        if (!field.doesIndexing() && !field.doesAttributing()) return;
+        Matching m = field.getMatching();
+        if (!m.isTypeUserSet()) {
+            m.setType(Matching.Type.WORD);
+        }
+        if (field.getRankType()==null || field.getRankType()== RankType.DEFAULT)
+            field.setRankType((RankType.TAGS));
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
new file mode 100644
index 00000000000..eb4efbe377f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Class that processes and validates tensor fields.
+ *
+ * @author <a href="geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class TensorFieldProcessor extends Processor {
+
+    public TensorFieldProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getDataType() == DataType.TENSOR) {
+                warnUseOfTensorFieldAsAttribute(field);
+                validateIndexingScripsForTensorField(field);
+                validateAttributeSettingForTensorField(field);
+            } else {
+                validateDataTypeForField(field);
+            }
+        }
+    }
+
+    private void warnUseOfTensorFieldAsAttribute(SDField field) {
+        if (field.doesAttributing()) {
+            // TODO (geirst): Remove when no longer beta
+            warn(search, field, "An attribute of type 'tensor' is currently beta, and re-feeding data between Vespa versions might be required.");
+        }
+    }
+
+    private void validateIndexingScripsForTensorField(SDField field) {
+        if (field.doesIndexing()) {
+            fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field.");
+        }
+    }
+
+    private void validateAttributeSettingForTensorField(SDField field) {
+        if (field.doesAttributing()) {
+            Attribute attribute = field.getAttributes().get(field.getName());
+            if (attribute != null && attribute.isFastSearch()) {
+                fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'.");
+            }
+        }
+    }
+
+    private void validateDataTypeForField(SDField field) {
+        if (field.getDataType().getPrimitiveType() == DataType.TENSOR) {
+            fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java
new file mode 100644
index 00000000000..21723639ece
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java
@@ -0,0 +1,122 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.indexinglanguage.ExpressionConverter;
+import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression;
+import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class TextMatch extends Processor {
+
+    public TextMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.getMatching().getType() != Matching.Type.TEXT) {
+                continue;
+            }
+            ScriptExpression script = field.getIndexingScript();
+            if (script == null) {
+                continue;
+            }
+            DataType fieldType = field.getDataType();
+            if (fieldType instanceof CollectionDataType) {
+                fieldType = ((CollectionDataType)fieldType).getNestedType();
+            }
+            if (fieldType != DataType.STRING) {
+                continue;
+            }
+            Set<String> dynamicSummary = new TreeSet<>();
+            Set<String> staticSummary = new TreeSet<>();
+            new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(search, field, dynamicSummary, staticSummary);
+            MyVisitor visitor = new MyVisitor(dynamicSummary);
+            visitor.visit(script);
+            if (!visitor.requiresTokenize) {
+                continue;
+            }
+            ExpressionConverter converter = new MyStringTokenizer(search, findAnnotatorConfig(search, field));
+            field.setIndexingScript((ScriptExpression)converter.convert(script));
+        }
+    }
+
+    private static AnnotatorConfig findAnnotatorConfig(Search search, SDField field) {
+        AnnotatorConfig ret = new AnnotatorConfig();
+        Stemming activeStemming = field.getStemming();
+        if (activeStemming == null) {
+            activeStemming = search.getStemming();
+        }
+        ret.setStemMode(activeStemming.toStemMode());
+        ret.setRemoveAccents(field.getNormalizing().doRemoveAccents());
+        return ret;
+    }
+
+    private static class MyVisitor extends ExpressionVisitor {
+
+        final Set<String> dynamicSummaryFields;
+        boolean requiresTokenize = false;
+
+        MyVisitor(Set<String> dynamicSummaryFields) {
+            this.dynamicSummaryFields = dynamicSummaryFields;
+        }
+
+        @Override
+        protected void doVisit(Expression exp) {
+            if (exp instanceof IndexExpression) {
+                requiresTokenize = true;
+            }
+            if (exp instanceof SummaryExpression &&
+                dynamicSummaryFields.contains(((SummaryExpression)exp).getFieldName()))
+            {
+                requiresTokenize = true;
+            }
+        }
+    }
+
+    private static class MyStringTokenizer extends TypedTransformProvider {
+
+        final AnnotatorConfig annotatorCfg;
+
+        MyStringTokenizer(Search search, AnnotatorConfig annotatorCfg) {
+            super(TokenizeExpression.class, search);
+            this.annotatorCfg = annotatorCfg;
+        }
+
+        @Override
+        protected boolean requiresTransform(Expression exp, DataType fieldType) {
+            return exp instanceof OutputExpression;
+        }
+
+        @Override
+        protected Expression newTransform(DataType fieldType) {
+            Expression exp = new TokenizeExpression(null, annotatorCfg);
+            if (fieldType instanceof CollectionDataType) {
+                exp = new ForEachExpression(exp);
+            }
+            return exp;
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java
new file mode 100644
index 00000000000..0fa9cbfa05f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java
@@ -0,0 +1,61 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Field;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.vespa.indexinglanguage.ValueTransformProvider;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class TypedTransformProvider extends ValueTransformProvider {
+
+    private final Search search;
+    private DataType fieldType;
+
+    TypedTransformProvider(Class<? extends Expression> transformClass, Search search) {
+        super(transformClass);
+        this.search = search;
+    }
+
+    @Override
+    protected final boolean requiresTransform(Expression exp) {
+        if (exp instanceof OutputExpression) {
+            String fieldName = ((OutputExpression)exp).getFieldName();
+            if (exp instanceof AttributeExpression) {
+                Attribute attribute = search.getAttribute(fieldName);
+                if (attribute == null) {
+                    throw new IllegalArgumentException("Attribute '" + fieldName + "' not found.");
+                }
+                fieldType = attribute.getDataType();
+            } else if (exp instanceof IndexExpression) {
+                Field field = search.getField(fieldName);
+                if (field == null) {
+                    throw new IllegalArgumentException("Index field '" + fieldName + "' not found.");
+                }
+                fieldType = field.getDataType();
+            } else if (exp instanceof SummaryExpression) {
+                Field field = search.getSummaryField(fieldName);
+                if (field == null) {
+                    throw new IllegalArgumentException("Summary field '" + fieldName + "' not found.");
+                }
+                fieldType = field.getDataType();
+            } else {
+                throw new UnsupportedOperationException();
+            }
+        }
+        return requiresTransform(exp, fieldType);
+    }
+
+    @Override
+    protected final Expression newTransform() {
+        return newTransform(fieldType);
+    }
+
+    protected abstract boolean requiresTransform(Expression exp, DataType fieldType);
+
+    protected abstract Expression newTransform(DataType fieldType);
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
new file mode 100644
index 00000000000..8f9b88b0268
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.document.*;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author balder
+ */
+public class UriHack extends Processor {
+    private static final List<String> URL_SUFFIX =
+            Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname");
+    public UriHack(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (field.doesIndexing()) {
+                DataType fieldType = field.getDataType();
+                if (fieldType instanceof CollectionDataType) {
+                    fieldType = ((CollectionDataType)fieldType).getNestedType();
+                }
+                if (fieldType == DataType.URI) {
+                    processField(search, field);
+                }
+            }
+        }
+    }
+
+    private void processField(Search search, SDField uriField) {
+        String uriName = uriField.getName();
+        uriField.setStemming(Stemming.NONE);
+        DataType generatedType = DataType.STRING;
+        if (uriField.getDataType() instanceof ArrayDataType) {
+            generatedType = new ArrayDataType(DataType.STRING);
+        } else if (uriField.getDataType() instanceof WeightedSetDataType) {
+            WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType();
+            generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero());
+        }
+        for (String suffix : URL_SUFFIX) {
+            String partName = uriName + "." + suffix;
+            // I wonder if this is explicit in qrs or implicit in backend?
+            // search.addFieldSetItem(uriName, partName);
+            SDField partField = new SDField(partName, generatedType, true);
+            partField.setIndexStructureField(uriField.doesIndexing());
+            partField.setRankType(uriField.getRankType());
+            partField.setStemming(Stemming.NONE);
+            partField.getNormalizing().inferLowercase();
+            if (uriField.getIndex(suffix) != null) {
+                partField.addIndex(uriField.getIndex(suffix));
+            }
+            search.addExtraField(partField);
+            search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, partField.getName());
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java
new file mode 100644
index 00000000000..b01a82b0131
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * @author bratseth
+ */
+public class UrlFieldValidator extends Processor {
+
+    public UrlFieldValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if  ( ! field.getDataType().equals(DataType.URI)) continue;
+
+            if (field.doesAttributing())
+                throw new IllegalArgumentException("Error in " + field + " in " + search + ": " +
+                                                   "uri type fields cannot be attributes");
+        }
+
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
new file mode 100644
index 00000000000..38c7338be3c
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.documentmodel.DocumentSummary;
+import com.yahoo.vespa.documentmodel.SummaryField;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This Processor checks to make sure all fields with the same name have the same {@link DataType}. This check
+ * explicitly disregards whether a field is an index field, an attribute or a summary field. This is a requirement if we
+ * hope to move to a model where index fields, attributes and summary fields share a common field class.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ValidateFieldTypes extends Processor {
+
+    public ValidateFieldTypes(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        String searchName = search.getName();
+        Map<String, DataType> fieldTypes = new HashMap<>();
+        for (SDField field : search.allFieldsList()) {
+            checkFieldType(searchName, "index field", field.getName(), field.getDataType(), fieldTypes);
+            for (Map.Entry<String, Attribute> entry : field.getAttributes().entrySet()) {
+                checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), fieldTypes);
+            }
+        }
+        for (DocumentSummary summary : search.getSummaries().values()) {
+            for (SummaryField field : summary.getSummaryFields()) {
+                checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), fieldTypes);
+            }
+        }
+    }
+
+    private void checkFieldType(String searchName, String fieldDesc, String fieldName, DataType fieldType,
+                                Map<String, DataType> fieldTypes)
+    {
+        DataType prevType = fieldTypes.get(fieldName);
+        if (prevType == null) {
+            fieldTypes.put(fieldName, fieldType);
+        } else if (!equalTypes(prevType, fieldType)) {
+            throw newProcessException(searchName, fieldName, "Duplicate field name with different types. Expected " + prevType.getName() + " for " + fieldDesc +
+                                                             " '" + fieldName + "', got " + fieldType.getName() + ".");
+        }
+    }
+    
+    private boolean equalTypes(DataType d1, DataType d2) {
+        if ("tag".equals(d1.getName())) {
+            return "tag".equals(d2.getName()) || "WeightedSet<string>".equals(d2.getName());
+        }
+        if ("tag".equals(d2.getName())) {
+            return "tag".equals(d1.getName()) || "WeightedSet<string>".equals(d1.getName());
+        }
+        return d1.equals(d2);
+    }
+    
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java
new file mode 100644
index 00000000000..a5c7d25532d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.Ranking;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * Check that fields with index settings actually creates an index or attribute
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor {
+
+    public ValidateFieldWithIndexSettingsCreatesIndex(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        Matching defaultMatching = new Matching();
+        Ranking defaultRanking = new Ranking();
+        for (SDField field : search.allFieldsList()) {
+            if (field.doesIndexing()) {
+                continue;
+            }
+            if (field.doesAttributing()) {
+                continue;
+            }
+            if (!(field.getRanking().equals(defaultRanking))) {
+                fail(search, field, "Fields which are not creating an index or attribute can not contain rank settings.");
+            }
+            if (!(field.getMatching().equals(defaultMatching))) {
+                fail(search, field, "Fields which are not creating an index or attribute can not contain match settings.");
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java
new file mode 100644
index 00000000000..24b811fc7fc
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+/**
+ * The implementation of word matching - with word matching the field is assumed to contain a single "word" - some
+ * contiguous sequence of word and number characters - but without changing the data at the indexing side (as with text
+ * matching) to enforce this. Word matching is thus almost like exact matching on the indexing side (no action taken),
+ * and like text matching on the query side. This may be suitable for attributes, where people both expect the data to
+ * be left as in the input document, and trivially written queries to work by default. However, this may easily lead to
+ * data which cannot be matched at all as the indexing and query side does not agree.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class WordMatch extends Processor {
+
+    public WordMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    public void process() {
+        for (SDField field : search.allFieldsList()) {
+            if (!field.getMatching().getType().equals(Matching.Type.WORD)) {
+                continue;
+            }
+            field.setStemming(Stemming.NONE);
+            field.getNormalizing().inferLowercase();
+            field.addQueryCommand("word");
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java
new file mode 100644
index 00000000000..158515fd05e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java
@@ -0,0 +1,62 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Resolver-class for harmonizing index-commands in multifield indexes
+ */
+public class IndexCommandResolver extends MultiFieldResolver {
+
+    /** Commands which don't have to be harmonized between fields */
+    private static List<String> ignoredCommands = new ArrayList<>();
+
+    /** Commands which must be harmonized between fields */
+    private static List<String> harmonizedCommands = new ArrayList<>();
+
+    static {
+        String[] ignore = { "complete-boost", "literal-boost", "highlight" };
+        ignoredCommands.addAll(Arrays.asList(ignore));
+        String[] harmonize = { "stemming", "normalizing" };
+        harmonizedCommands.addAll(Arrays.asList(harmonize));
+    }
+
+    public IndexCommandResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+        super(indexName, fields, search, logger);
+    }
+
+    /**
+     * Check index-commands for each field, report and attempt to fix any
+     * inconsistencies
+     */
+    public void resolve() {
+        for (SDField field : fields) {
+            for (String command : field.getQueryCommands()) {
+                if (!ignoredCommands.contains(command))
+                    checkCommand(command);
+            }
+        }
+    }
+
+    private void checkCommand(String command) {
+        for (SDField field : fields) {
+            if (!field.hasQueryCommand(command)) {
+                if (harmonizedCommands.contains(command)) {
+                    deployLogger.log(Level.WARNING, command + " must be added to all fields going to the same index (" + indexName + ")" +
+                            ", adding to field " + field.getName());
+                    field.addQueryCommand(command);
+                } else {
+                    deployLogger.log(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() +
+                            "\' doesn't contain command \'" + command+"\'");
+                }
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java
new file mode 100644
index 00000000000..943a788fc75
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+import java.util.List;
+
+/**
+ * Abstract superclass of all multifield conflict resolvers
+ */
+public abstract class MultiFieldResolver {
+
+    protected String indexName;
+    protected List<SDField> fields;
+    protected Search search;
+
+    protected DeployLogger deployLogger;
+
+    public MultiFieldResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+        this.indexName = indexName;
+        this.fields = fields;
+        this.search = search;
+        this.deployLogger = logger;
+    }
+
+    /**
+     * Checks the list of fields for specific conflicts, and reports and/or
+     * attempts to correct them
+     */
+    public abstract void resolve();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
new file mode 100644
index 00000000000..d98dd97ff83
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.TensorFieldType;
+import com.yahoo.searchdefinition.RankProfile;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.processing.Processor;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Class that processes a search instance and sets type settings on all rank profiles.
+ *
+ * Currently, type settings are limited to the type of tensor attribute fields and tensor query features.
+ *
+ * @author <a href="geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class RankProfileTypeSettingsProcessor extends Processor {
+
+    private static final Pattern queryFeaturePattern = Pattern.compile("query\\((\\w+)\\)$");
+
+    public RankProfileTypeSettingsProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process() {
+        processAttributeFields();
+        processQueryProfileTypes();
+
+    }
+
+    private void processAttributeFields() {
+        for (SDField field : search.allFieldsList()) {
+            Attribute attribute = field.getAttributes().get(field.getName());
+            if (attribute != null && attribute.tensorType().isPresent()) {
+                addAttributeTypeToRankProfiles(attribute.getName(), attribute.tensorType().get().toString());
+            }
+        }
+    }
+
+    private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) {
+        for (RankProfile profile : rankProfileRegistry.allRankProfiles()) {
+            profile.addAttributeType(attributeName, attributeType);
+        }
+    }
+
+    private void processQueryProfileTypes() {
+        for (QueryProfileType queryProfileType : queryProfiles.getRegistry().getTypeRegistry().allComponents()) {
+            for (Map.Entry<String, FieldDescription> fieldDescEntry : queryProfileType.fields().entrySet()) {
+                processFieldDescription(fieldDescEntry.getValue());
+            }
+        }
+    }
+
+    private void processFieldDescription(FieldDescription fieldDescription) {
+        String fieldName = fieldDescription.getName();
+        FieldType fieldType = fieldDescription.getType();
+        if (fieldType instanceof TensorFieldType) {
+            TensorFieldType tensorFieldType = (TensorFieldType)fieldType;
+            Matcher matcher = queryFeaturePattern.matcher(fieldName);
+            if (tensorFieldType.type().isPresent() && matcher.matches()) {
+                String queryFeature = matcher.group(1);
+                addQueryFeatureTypeToRankProfiles(queryFeature, tensorFieldType.type().get().toString());
+            }
+        }
+    }
+
+    private void addQueryFeatureTypeToRankProfiles(String queryFeature, String queryFeatureType) {
+        for (RankProfile profile : rankProfileRegistry.allRankProfiles()) {
+            profile.addQueryFeatureType(queryFeature, queryFeatureType);
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java
new file mode 100644
index 00000000000..787a70862a9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.searchdefinition.document.RankType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.Search;
+
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Checks if fields have defined different rank types for the same
+ * index (typically in an index-to statement), and if they have
+ * output a warning and use the first ranktype.
+ *
+ * @author <a href="musum@yahoo-inc.com">Harald Musum</a>
+ */
+public class RankTypeResolver extends MultiFieldResolver {
+
+    public RankTypeResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+        super(indexName, fields, search, logger);
+    }
+
+    public void resolve() {
+        RankType rankType = null;
+        if (fields.size() > 0) {
+            boolean first = true;
+            for (SDField field : fields) {
+                if (first) {
+                    rankType = fields.get(0).getRankType();
+                    first = false;
+                } else if (!field.getRankType().equals(rankType)) {
+                    deployLogger.log(Level.WARNING, "In field '" + field.getName() + "' " +
+                            field.getRankType() + " for index '" + indexName +
+                            "' conflicts with " + rankType +
+                            " defined for the same index in field '" +
+                            field.getName() + "'. Using " +
+                            rankType + ".");
+                    field.setRankType(rankType);
+                }
+            }
+        }
+    }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java
new file mode 100644
index 00000000000..f1e1899391d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.language.Linguistics;
+import com.yahoo.searchdefinition.Index;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.Stemming;
+import com.yahoo.searchdefinition.processing.BuiltInFieldSets;
+import com.yahoo.vespa.indexinglanguage.expressions.*;
+import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
+
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Class resolving conflicts when fields with different stemming-settings are
+ * combined into the same index
+ */
+public class StemmingResolver extends MultiFieldResolver {
+
+    public StemmingResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) {
+        super(indexName, fields, search, logger);
+    }
+
+    @Override
+    public void resolve() {
+        checkStemmingForIndexFields(indexName, fields);
+    }
+
+    private void checkStemmingForIndexFields(String indexName, List<SDField> fields) {
+        Stemming stemming = null;
+        SDField stemmingField = null;
+        for (SDField field : fields) {
+            if (stemming == null && stemmingField==null) {
+                stemming = field.getStemming(search);
+                stemmingField = field;
+            } else if (stemming != field.getStemming(search)) {
+                deployLogger.log(Level.WARNING, "Field '" + field.getName() + "' has " + field.getStemming(search) +
+                        ", whereas field '" + stemmingField.getName() + "' has " + stemming +
+                        ". All fields indexing to the index '" + indexName  + "' must have the same stemming." +
+                        " This should be corrected as it will make indexing fail in a few cases.");
+            }
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java
new file mode 100644
index 00000000000..9915e11b28d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Classes in this package (processors) implements some search
+ * definition features by reducing them to simpler features.
+ * The processors are run after parsing of the search definition,
+ * before creating the derived model.
+ *
+ * For simplicity, features should always be implemented here
+ * rather than in the derived model if possible.
+ *
+ * New processors must be added to the list in Processing.
+ */
+@com.yahoo.api.annotations.PackageMarker
+package com.yahoo.searchdefinition.processing;