Publish

473 files changed, 49280 insertions, 0 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
new file mode 100644
index 00000000000..20831e743b9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -0,0 +1,1060 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.collections.Tuple2;
+import com.yahoo.component.Version;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.fs4.MapEncoder;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.fastsearch.DocumentDatabase;
+import com.yahoo.prelude.query.Highlight;
+import com.yahoo.prelude.query.QueryException;
+import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.properties.PropertyMap;
+import com.yahoo.search.query.Model;
+import com.yahoo.search.query.ParameterParser;
+import com.yahoo.search.query.Presentation;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.SessionId;
+import com.yahoo.search.query.Sorting;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileFieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry;
+import com.yahoo.search.query.properties.DefaultProperties;
+import com.yahoo.search.query.properties.QueryProperties;
+import com.yahoo.search.query.properties.QueryPropertyAliases;
+import com.yahoo.search.query.properties.RequestContextProperties;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.federation.FederationSearcher;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.query.Sorting.AttributeSorter;
+import com.yahoo.search.query.Sorting.FieldOrder;
+import com.yahoo.search.query.Sorting.Order;
+import com.yahoo.search.query.context.QueryContext;
+import com.yahoo.search.query.profile.ModelObjectMap;
+import com.yahoo.search.query.profile.QueryProfileProperties;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.yql.NullItemException;
+import com.yahoo.search.yql.VespaSerializer;
+import com.yahoo.search.yql.YqlParser;
+
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+
+/**
+ * A search query containing all the information required to produce a Result.
+ * <p>
+ * The Query contains:
+ * <ul>
+ * <li>the selection criterion received in the request - which may be a structured boolean tree of operators,
+ * an annotated piece of natural language text received from a user, or a combination of both
+ * <li>a set of field containing the additional general parameters of a query - number of hits,
+ * ranking, presentation etc.
+ * <li>a Map of properties, which can be of any object type
+ * </ul>
+ *
+ * <p>
+ * The properties has three sources
+ * <ol>
+ * <li>They may be set in some Searcher component already executed for this Query - the properties acts as
+ * a blackboard for communicating arbitrary objects between Searcher components.
+ * <li>Properties set in the search Request received - the properties acts as a way to parametrize Searcher
+ * components from the Request.
+ * <li>Properties defined in the selected {@link com.yahoo.search.query.profile.QueryProfile} - this provides
+ * defaults for the parameters to Searcher components. Note that by using query profile types, the components may
+ * define the set of parameters they support.
+ * </ol>
+ * When looked up, the properties are accessed in the priority order listed above.
+ * <p>
+ * The identity of a query is determined by its content.
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author bratseth
+ */
+public class Query extends com.yahoo.processing.Request implements Cloneable {
+
+    // Note to developers: If you think you should add something here you are probably wrong
+    // To add state to the query: Do properties.set("myNewState",new MyNewState()) instead.
+
+    /** The type of the query */
+    public enum Type {
+
+        ALL(0,"all"),
+        ANY(1,"any"),
+        PHRASE(2,"phrase"),
+        ADVANCED(3,"adv"),
+        WEB(4,"web"),
+        PROGRAMMATIC(5, "prog"),
+        YQL(6, "yql");
+
+        private final int intValue;
+        private final String stringValue;
+
+        Type(int intValue,String stringValue) {
+            this.intValue = intValue;
+            this.stringValue = stringValue;
+        }
+
+        /** Converts a type argument value into a query type */
+        public static Type getType(String typeString) {
+            for (Type type:Type.values())
+                if(type.stringValue.equals(typeString))
+                    return type;
+            return ALL;
+        }
+
+        public int asInt() { return intValue; }
+
+        public String toString() { return stringValue; }
+
+    }
+
+    //--------------  Query properties treated as fields in Query ---------------
+
+    /** The offset from the most relevant hits found from this query */
+    private int offset = 0;
+
+    /** The number of hits to return */
+    private int hits = 10;
+
+    /** The query context level, 0 means no tracing */
+    private int traceLevel = 0;
+
+    // The timeout to be used when dumping rank features
+    private static final long dumpTimeout = (6 * 60 * 1000); // 6 minutes
+    private static final long defaultTimeout = 5000;
+    /** The timeout of the query, in milliseconds */
+    private long timeout = defaultTimeout;
+
+
+    /** Whether this query is forbidden to access cached information */
+    private boolean noCache=false;
+
+    /** Whether or not grouping should use a session cache */
+    private boolean groupingSessionCache=false;
+
+    //--------------  Generic property containers --------------------------------
+
+    /**
+     * The synchronous view of the JDisc request causing this query.
+     *
+     * @since 5.1
+     */
+    private final HttpRequest httpRequest;
+
+    /** The context, or null if there is no context */
+    private QueryContext context = null;
+
+    /** Used for downstream session caches */
+    private SessionId sessionId = null;
+
+    //--------------- Owned sub-objects containing query properties ----------------
+
+    /** The ranking requested in this query */
+    private Ranking ranking = new Ranking(this);
+
+    /** The query query and/or query program declaration */
+    private Model model = new Model(this);
+
+    /** How results of this query should be presented */
+    private Presentation presentation = new Presentation(this);
+
+    //---------------- Tracing ----------------------------------------------------
+
+    private static Logger log = Logger.getLogger(Query.class.getName());
+
+    /** The time this query was created */
+    private long startTime;
+
+    /** Error conditions stemming from the query itself */
+    private List<ErrorMessage> errors = new ArrayList<>(0);
+
+    //---------------- Static property handling ------------------------------------
+
+    public static final CompoundName OFFSET = new CompoundName("offset");
+    public static final CompoundName HITS = new CompoundName("hits");
+
+    public static final CompoundName SEARCH_CHAIN = new CompoundName("searchChain");
+    public static final CompoundName TRACE_LEVEL = new CompoundName("traceLevel");
+    public static final CompoundName NO_CACHE = new CompoundName("noCache");
+    public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache");
+    public static final CompoundName TIMEOUT = new CompoundName("timeout");
+
+    private static QueryProfileType argumentType;
+    static {
+        argumentType=new QueryProfileType("native");
+        argumentType.setBuiltin(true);
+
+        argumentType.addField(new FieldDescription(OFFSET.toString(), "integer", "offset start"));
+        argumentType.addField(new FieldDescription(HITS.toString(), "integer", "hits count"));
+        // TODO: Should this be added to com.yahoo.search.query.properties.QueryProperties? If not, why not?
+        argumentType.addField(new FieldDescription(SEARCH_CHAIN.toString(), "string"));
+        argumentType.addField(new FieldDescription(TRACE_LEVEL.toString(), "integer", "tracelevel"));
+        argumentType.addField(new FieldDescription(NO_CACHE.toString(), "boolean", "nocache"));
+        argumentType.addField(new FieldDescription(GROUPING_SESSION_CACHE.toString(), "boolean", "groupingSessionCache"));
+        argumentType.addField(new FieldDescription(TIMEOUT.toString(), "string", "timeout"));
+        argumentType.addField(new FieldDescription(FederationSearcher.SOURCENAME.toString(),"string"));
+        argumentType.addField(new FieldDescription(FederationSearcher.PROVIDERNAME.toString(),"string"));
+        argumentType.addField(new FieldDescription(Presentation.PRESENTATION,new QueryProfileFieldType(Presentation.getArgumentType())));
+        argumentType.addField(new FieldDescription(Ranking.RANKING,new QueryProfileFieldType(Ranking.getArgumentType())));
+        argumentType.addField(new FieldDescription(Model.MODEL,new QueryProfileFieldType(Model.getArgumentType())));
+        argumentType.freeze();
+    }
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    /** The aliases of query properties, these are always the same */
+    // Note: Don't make static for now as GSM calls this through reflection
+    private static Map<String,CompoundName> propertyAliases;
+    static {
+        Map<String,CompoundName> propertyAliasesBuilder = new HashMap<>();
+        addAliases(Query.getArgumentType(), propertyAliasesBuilder);
+        addAliases(Ranking.getArgumentType(), propertyAliasesBuilder);
+        addAliases(Model.getArgumentType(), propertyAliasesBuilder);
+        addAliases(Presentation.getArgumentType(), propertyAliasesBuilder);
+        propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder);
+    }
+    private static void addAliases(QueryProfileType arguments,Map<String,CompoundName> aliases) {
+        String prefix=getPrefix(arguments);
+        for (FieldDescription field : arguments.fields().values()) {
+            for (String alias : field.getAliases())
+                aliases.put(alias,new CompoundName(prefix+field.getName()));
+        }
+    }
+    private static String getPrefix(QueryProfileType type) {
+        if (type.getId().getName().equals("native")) return ""; // The arguments of this directly
+        return type.getId().getName() + ".";
+    }
+
+    public static void addNativeQueryProfileTypesTo(QueryProfileTypeRegistry registry) {
+        // Add modifiable copies to allow query profile types in this to add to these
+        registry.register(Query.getArgumentType().unfrozen());
+        registry.register(Ranking.getArgumentType().unfrozen());
+        registry.register(Model.getArgumentType().unfrozen());
+        registry.register(Presentation.getArgumentType().unfrozen());
+        registry.register(DefaultProperties.argumentType.unfrozen());
+    }
+
+    //---------------- Construction ------------------------------------
+
+    /**
+     * Constructs an empty (null) query
+     */
+    public Query() {
+        this("");
+    }
+
+    /**
+     * Construct a query from a string formatted in the http style, e.g <code>?query=test&amp;offset=10&amp;hits=13</code>
+     * The query must be uri encoded.
+     */
+    public Query(String query) {
+        this(query, null);
+    }
+
+    /**
+     * Construct a query from a string formatted in the http style, e.g <code>?query=test&amp;offset=10&amp;hits=13</code>
+     * The query must be uri encoded.
+     */
+    public Query(String query, CompiledQueryProfile queryProfile) {
+        this(HttpRequest.createTestRequest(query, com.yahoo.jdisc.http.HttpRequest.Method.GET), queryProfile);
+    }
+
+    /**
+     * Creates a query from a request
+     *
+     * @param request the HTTP request from which this is created
+     * @param queryProfile the query profile to use for this query, or null if none.
+     */
+    public Query(HttpRequest request, CompiledQueryProfile queryProfile) {
+        super(new QueryPropertyAliases(propertyAliases));
+        this.httpRequest = request;
+        init(request.propertyMap(), queryProfile);
+    }
+
+    /**
+     * Creates a query from a request
+     *
+     * @param request the HTTP request from which this is created
+     */
+    public Query(HttpRequest request) {
+        this(request, null);
+    }
+
+    private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile) {
+        startTime = System.currentTimeMillis();
+        if (queryProfile != null) {
+            // Move all request parameters to the query profile just to validate that the parameter settings are legal
+            Properties queryProfileProperties=new QueryProfileProperties(queryProfile);
+            properties().chain(queryProfileProperties);
+            // TODO: Just checking legality rather than actually setting would be faster
+            setPropertiesFromRequestMap(requestMap, properties()); // Adds errors to the query for illegal set attempts
+
+            // Create the full chain
+            properties().chain(new QueryProperties(this, queryProfile.getRegistry())).
+                         chain(new ModelObjectMap()).
+                         chain(new RequestContextProperties(requestMap)).
+                         chain(queryProfileProperties).
+                         chain(new DefaultProperties());
+
+            // Pass the values from the query profile which maps through a field in the Query object model
+            // through the property chain to cause those values to be set in the Query object model
+            setFieldsFrom(queryProfileProperties, requestMap);
+        }
+        else { // bypass these complications if there is no query profile to get values from and validate against
+            properties().
+                    chain(new QueryProperties(this, new CompiledQueryProfileRegistry())).
+                    chain(new PropertyMap()).
+                    chain(new DefaultProperties());
+            setPropertiesFromRequestMap(requestMap, properties());
+        }
+
+        properties().setParentQuery(this);
+        traceProperties();
+    }
+
+    public Query(Query query) {
+        this(query, query.getStartTime());
+    }
+
+    private Query(Query query, long startTime) {
+        super(query.properties().clone());
+        this.startTime = startTime;
+        this.httpRequest = query.httpRequest;
+        query.copyPropertiesTo(this);
+    }
+
+    /**
+     * Creates a new query from another query, but with time sensitive
+     * fields reset.
+     *
+     * @return new query
+     */
+    public static Query createNewQuery(Query query) {
+        return new Query(query, System.currentTimeMillis());
+    }
+
+    /**
+     * Calls properties().set on each value in the given properties which is declared in this query or
+     * one of its dependent objects. This will ensure the appropriate setters are called on this and all
+     * dependent objects for the appropriate subset of the given property values
+     */
+    private void setFieldsFrom(Properties properties, Map<String,String> context) {
+        setFrom(properties,Query.getArgumentType(), context);
+        setFrom(properties,Model.getArgumentType(), context);
+        setFrom(properties,Presentation.getArgumentType(), context);
+        setFrom(properties,Ranking.getArgumentType(), context);
+    }
+
+    /**
+     * For each field in the given query profile type, take the corresponding value from originalProperties
+     * (if any) set it to properties().
+     */
+    private void setFrom(Properties originalProperties,QueryProfileType arguments,Map<String,String> context) {
+        String prefix=getPrefix(arguments);
+        for (FieldDescription field : arguments.fields().values()) {
+            String fullName=prefix + field.getName();
+            if (field.getType() == FieldType.genericQueryProfileType) {
+                for (Map.Entry<String, Object> entry : originalProperties.listProperties(fullName,context).entrySet()) {
+                    try {
+                        properties().set(fullName + "." + entry.getKey(), entry.getValue(), context);
+                    } catch (IllegalArgumentException e) {
+                        throw new QueryException("Invalid request parameter", e);
+                    }
+                }
+            } else {
+                Object value=originalProperties.get(fullName,context);
+                if (value!=null) {
+                    try {
+                        properties().set(fullName,value,context);
+                    } catch (IllegalArgumentException e) {
+                        throw new QueryException("Invalid request parameter", e);
+                    }
+                }
+            }
+        }
+    }
+
+    /** Calls properties.set on all entries in requestMap */
+    private void setPropertiesFromRequestMap(Map<String, String> requestMap, Properties properties) {
+        for (Map.Entry<String, String> entry : requestMap.entrySet()) {
+            try {
+                if (entry.getKey().equals("queryProfile")) continue;
+                properties.set(entry.getKey(), entry.getValue(), requestMap);
+            }
+            catch (IllegalArgumentException e) {
+                throw new QueryException("Invalid request parameter", e);
+            }
+        }
+    }
+
+    /** Returns the properties of this query. The properties are modifiable */
+    @Override
+    public Properties properties() { return (Properties)super.properties(); }
+
+    /**
+     * Traces how properties was resolved and from where. Done after the fact to avoid special handling
+     * of tracelevel, which is the property deciding whether this needs to be done
+     */
+    private void traceProperties() {
+        if (traceLevel==0) return;
+        CompiledQueryProfile profile=null;
+        QueryProfileProperties profileProperties=properties().getInstance(QueryProfileProperties.class);
+        if (profileProperties!=null)
+            profile=profileProperties.getQueryProfile();
+
+        if (profile==null)
+            trace("No query profile is used", false, 1);
+        else
+            trace("Using " + profile.toString(), false, 1);
+        if (traceLevel<4) return;
+
+        StringBuilder b=new StringBuilder("Resolved properties:\n");
+        Set<String> mentioned=new HashSet<>();
+        for (Map.Entry<String,String> requestProperty : requestProperties().entrySet() ) {
+            Object resolvedValue = properties().get(requestProperty.getKey(), requestProperties());
+            if (resolvedValue == null && requestProperty.getKey().equals("queryProfile"))
+                resolvedValue = requestProperty.getValue();
+
+            b.append(requestProperty.getKey());
+            b.append("=");
+            b.append(String.valueOf(resolvedValue)); // (may be null)
+            b.append(" (");
+
+            if (profile != null && ! profile.isOverridable(new CompoundName(requestProperty.getKey()), requestProperties()))
+                b.append("value from query profile - unoverridable, ignoring request value");
+            else
+                b.append("value from request");
+            b.append(")\n");
+            mentioned.add(requestProperty.getKey());
+        }
+        if (profile!=null) {
+            appendQueryProfileProperties(profile,mentioned,b);
+        }
+        trace(b.toString(),false,4);
+    }
+
+    private Map<String, String> requestProperties() {
+        return httpRequest.propertyMap();
+    }
+
+    private void appendQueryProfileProperties(CompiledQueryProfile profile,Set<String> mentioned,StringBuilder b) {
+        for (Map.Entry<String,Object> property : profile.listValues("",requestProperties()).entrySet()) {
+            if ( ! mentioned.contains(property.getKey()))
+                b.append(property.getKey() + "=" + property.getValue() + " (value from query profile)<br/>\n");
+        }
+    }
+
+    /**
+     * Validates this query
+     *
+     * @return the reason if it is invalid, null if it is valid
+     */
+    public String validate() {
+        // Validate the query profile
+        QueryProfileProperties queryProfileProperties = properties().getInstance(QueryProfileProperties.class);
+        if (queryProfileProperties == null) return null; // Valid
+        StringBuilder missingName = new StringBuilder();
+        if (! queryProfileProperties.isComplete(missingName, httpRequest.propertyMap()))
+            return "Incomplete query: Parameter '" + missingName + "' is mandatory in " +
+                   queryProfileProperties.getQueryProfile() + " but is not set";
+        else
+            return null; // is valid
+    }
+
+    /** Returns the time (in milliseconds since epoch) when this query was started */
+    public long getStartTime() { return startTime; }
+
+    /** Returns the time (in milliseconds) since the query was started/created */
+    public long getDurationTime() {
+        return System.currentTimeMillis() - startTime;
+    }
+
+    /**
+     * Get the appropriate timeout for the query.
+     *
+     * @return timeout in milliseconds
+     **/
+    public long getTimeLeft() {
+        return getTimeout() - getDurationTime();
+    }
+
+    public boolean requestHasProperty(String name) {
+        return httpRequest.hasProperty(name);
+    }
+
+    /**
+     * Returns the number of milliseconds to wait for a response from a search backend
+     * before timing it out. Default is 5000.
+     * <p>
+     * Note: If Ranking.RANKFEATURES is turned on, this is hardcoded to 6 minutes.
+     *
+     * @return timeout in milliseconds.
+     */
+    public long getTimeout() {
+        return properties().getBoolean(Ranking.RANKFEATURES, false) ? dumpTimeout : timeout;
+    }
+
+    /**
+     * Sets the number of milliseconds to wait for a response from a search backend
+     * before time out. Default is 5000.
+     */
+    public void setTimeout(long timeout) {
+        if (timeout > 1000000000 || timeout < 0)
+            throw new IllegalArgumentException("'timeout' must be positive and smaller than 1000000000 ms but was " + timeout);
+        this.timeout = timeout;
+    }
+
+    /**
+     * Sets timeout from a string which will be parsed as a
+     */
+    public void setTimeout(String timeoutString) {
+        setTimeout(ParameterParser.asMilliSeconds(timeoutString, timeout));
+    }
+
+    /**
+     * Resets the start time of the query. This will ensure that the query will run
+     * for the same amount of time as a newly created query.
+     */
+    public void resetTimeout() { this.startTime = System.currentTimeMillis(); }
+
+    /**
+     * Sets the context level of this query, 0 means no tracing
+     * Higher numbers means increasingly more tracing
+     */
+    public void setTraceLevel(int traceLevel) { this.traceLevel = traceLevel; }
+
+    /**
+     * Returns the context level of this query, 0 means no tracing
+     * Higher numbers means increasingly more tracing
+     */
+    public int getTraceLevel() { return traceLevel; }
+
+    /**
+     * Returns the context level of this query, 0 means no tracing
+     * Higher numbers means increasingly more tracing
+     */
+    public final boolean isTraceable(int level) { return traceLevel >= level; }
+
+
+    /** Returns whether this query should never be served from a cache. Default is false */
+    public boolean getNoCache() { return noCache; }
+
+    /** Sets whether this query should never be server from a cache. Default is false */
+    public void setNoCache(boolean noCache) { this.noCache = noCache; }
+
+    /** Returns whether this query should use the grouping session cache. Default is false */
+    public boolean getGroupingSessionCache() { return groupingSessionCache; }
+
+    /** Sets whether this query should use the grouping session cache. Default is false */
+    public void setGroupingSessionCache(boolean groupingSessionCache) { this.groupingSessionCache = groupingSessionCache; }
+
+    /**
+     * Returns the offset from the most relevant hits requested by the submitter
+     * of this query.
+     * Default is 0 - to return the most relevant hits
+     */
+    public int getOffset() { return offset; }
+
+    /**
+     * Returns the number of hits requested by the submitter of this query.
+     * The default is 10.
+     */
+    public int getHits() { return hits; }
+
+    /**
+     * Sets the number of hits requested. If hits is less than 0, an
+     * IllegalArgumentException is thrown. Default number of hits is 10.
+     */
+    public void setHits(int hits) {
+        if (hits < 0)
+            throw new IllegalArgumentException("Must be a positive number");
+        this.hits = hits;
+    }
+
+    /**
+     * Set the hit offset. Can not be less than 0. Default is 0.
+     */
+    public void setOffset(int offset) {
+        if (offset < 0)
+            throw new IllegalArgumentException("Must be a positive number");
+        this.offset = offset;
+    }
+
+    /** Convenience method to set both the offset and the number of hits to return */
+    public void setWindow(int offset,int hits) {
+        setOffset(offset);
+        setHits(hits);
+    }
+
+    /**
+     * This is ignored - compression is controlled at the network level.
+     *
+     * @deprecated this is ignored
+     */
+    @Deprecated
+    public void setCompress(boolean ignored) { }
+
+    /**
+     * Returns false.
+     *
+     * @deprecated this always returns false
+     */
+    @Deprecated
+    public boolean getCompress() { return false; }
+
+    /** Returns a string describing this query */
+    @Override
+    public String toString() {
+        String queryTree;
+        // getQueryTree isn't exception safe
+        try {
+            queryTree = model.getQueryTree().toString();
+        } catch (Exception e) {
+            queryTree = "[Could not parse user input: " + model.getQueryString() + "]";
+        }
+        return "query '" + queryTree + "'";
+    }
+
+    /** Returns a string describing this query in more detail */
+    public String toDetailString() {
+        String queryTree;
+        // getQueryTree isn't exception safe
+        try {
+            queryTree = model.getQueryTree().toString();
+        } catch (Exception e) {
+            queryTree = "Could not parse user input: " + model.getQueryString();
+        }
+        return "query=[" + queryTree + "]" + " offset=" + getOffset() + " hits=" + getHits() + "]";
+    }
+
+    /**
+     * Encodes this query onto the given buffer
+     *
+     * @param buffer The buffer to encode the query to
+     * @return the number of encoded items
+     */
+    public int encode(ByteBuffer buffer) {
+        return model.getQueryTree().encode(buffer);
+    }
+
+    /**
+     * Adds a context message to this query and to the info log,
+     * if the context level of the query is sufficiently high.
+     * The context information will be carried over to the result at creation.
+     * The message parameter will be included <i>with</i> XML escaping.
+     *
+     * @param message      the message to add
+     * @param traceLevel   the context level of the message, this method will do nothing
+     *                     if the traceLevel of the query is lower than this value
+     */
+    public void trace(String message, int traceLevel) {
+        trace(message, false, traceLevel);
+    }
+
+    /**
+     * Adds a trace message to this query
+     * if the trace level of the query is sufficiently high.
+     *
+     * @param message      the message to add
+     * @param includeQuery true to append the query root stringValue
+     *        at the end of the message
+     * @param traceLevel   the context level of the message, this method will do nothing
+     *                     if the traceLevel of the query is lower than this value
+     */
+    public void trace(String message, boolean includeQuery, int traceLevel) {
+        if ( ! isTraceable(traceLevel)) return;
+
+        if (includeQuery)
+            message += ": [" + queryTreeText() + "]";
+
+        log.log(LogLevel.DEBUG,message);
+
+        // Pass 0 as traceLevel as the trace level check is already done above,
+        // and it is not propagated to trace until execution has started
+        // (it is done in the execution.search method)
+        getContext(true).trace(message, 0);
+    }
+
+    /**
+     * Adds a trace message to this query
+     * if the trace level of the query is sufficiently high.
+     *
+     * @param includeQuery true to append the query root stringValue at the end of the message
+     * @param traceLevel   the context level of the message, this method will do nothing
+     *                     if the traceLevel of the query is lower than this value
+     * @param messages     the messages whose toStrings will be concatenated into the trace message.
+     *                     Concatenation will only happen if the trace level is sufficiently high.
+     */
+    public void trace(boolean includeQuery, int traceLevel, Object... messages) {
+        if ( ! isTraceable(traceLevel)) return;
+
+        StringBuilder concatenated = new StringBuilder();
+        for (Object message : messages)
+            concatenated.append(String.valueOf(message));
+        trace(concatenated.toString(), includeQuery, traceLevel);
+    }
+
+    /**
+     * Set the context information for another query to be part of this query's
+     * context information. This is to be used if creating fresh query objects as
+     * part of a plug-in's execution. The query should be attached before it is
+     * used, in case an exception causes premature termination. This is enforced
+     * by an IllegalStateException. In other words, intended use is create the
+     * new query, and attach the context to the invoking query as soon as the new
+     * query is properly initialized.
+     *
+     * <p>
+     * This method will always set the argument query's context level to the context
+     * level of this query.
+     *
+     * @param query
+     *                The query which should be traced as a part of this query.
+     * @throws IllegalStateException
+     *                 If the query given as argument already has context
+     *                 information.
+     */
+    public void attachContext(Query query) throws IllegalStateException {
+        query.setTraceLevel(getTraceLevel());
+        if (context == null) {
+            // Nothing to attach to. This is about the same as
+            // getTraceLevel() == 0,
+            // but is a direct test of what will make the function superfluous.
+            return;
+        }
+        if (query.getContext(false) != null) {
+            // If we added the other query's context info as a subnode in this
+            // query's context tree, we would have to check for loops in the
+            // context graph. If we simply created a new node without checking,
+            // we might silently overwrite useful information.
+            throw new IllegalStateException("Query to attach already has context information stored.");
+        }
+        query.context = context;
+    }
+
+    private String queryTreeText() {
+        QueryTree root = getModel().getQueryTree();
+
+        if (getTraceLevel() < 2)
+            return root.toString();
+        if (getTraceLevel() < 6)
+            return yqlRepresentation();
+        else
+            return "\n" + yqlRepresentation() + "\n" + new TextualQueryRepresentation(root.getRoot()) + "\n";
+    }
+
+    /**
+     * Serialize this query as YQL+. This method will never throw exceptions,
+     * but instead return a human readable error message if a problem occured
+     * serializing the query. Hits and offset information will be included if
+     * different from default, while linguistics metadata are not added.
+     *
+     * @return a valid YQL+ query string or a human readable error message
+     * @see Query#yqlRepresentation(Tuple2, boolean)
+     */
+    public String yqlRepresentation() {
+        try {
+            return yqlRepresentation(null, true);
+        } catch (NullItemException e) {
+            return "Query currently a placeholder, NullItem encountered.";
+        } catch (RuntimeException e) {
+            return "Failed serializing query as YQL+, please file a ticket including the query causing this: "
+                    + Exceptions.toMessageString(e);
+        }
+    }
+
+    private void commaSeparated(StringBuilder yql, Set<String> fields) {
+        int initLen = yql.length();
+        for (String field : fields) {
+            if (yql.length() > initLen) {
+                yql.append(", ");
+            }
+            yql.append(field);
+        }
+    }
+
+    /**
+     * Serialize this query as YQL+. This will create a string representation
+     * which should always be legal YQL+. If a problem occurs, a
+     * RuntimeException is thrown.
+     *
+     * @param segmenterVersion
+     *            linguistics metadata used in federation, set to null if the
+     *            annotation is not necessary
+     * @param includeHitsAndOffset
+     *            whether to include hits and offset parameters converted to a
+     *            offset/limit slice
+     * @return a valid YQL+ query string
+     * @throws RuntimeException if there is a problem serializing the query tree
+     */
+    public String yqlRepresentation(@Nullable Tuple2<String, Version> segmenterVersion, boolean includeHitsAndOffset) {
+        String q = VespaSerializer.serialize(this);
+
+        Set<String> sources = getModel().getSources();
+        Set<String> fields = getPresentation().getSummaryFields();
+        StringBuilder yql = new StringBuilder("select ");
+        if (fields.isEmpty()) {
+            yql.append('*');
+        } else {
+            commaSeparated(yql, fields);
+        }
+        yql.append(" from ");
+        if (sources.isEmpty()) {
+            yql.append("sources *");
+        } else {
+            if (sources.size() > 1) {
+                yql.append("sources ");
+            }
+            commaSeparated(yql, sources);
+        }
+        yql.append(" where ");
+        if (segmenterVersion != null) {
+            yql.append("[{\"segmenter\": {\"version\": \"")
+                    .append(segmenterVersion.second.toString())
+                    .append("\", \"backend\": \"")
+                    .append(segmenterVersion.first).append("\"}}](");
+        }
+        yql.append(q);
+        if (segmenterVersion != null) {
+            yql.append(')');
+        }
+        if (getRanking().getSorting() != null && getRanking().getSorting().fieldOrders().size() > 0) {
+            serializeSorting(yql);
+        }
+        if (includeHitsAndOffset) {
+            if (getOffset() != 0) {
+                yql.append(" limit ")
+                        .append(Integer.toString(getHits() + getOffset()))
+                        .append(" offset ")
+                        .append(Integer.toString(getOffset()));
+            } else if (getHits() != 10) {
+                yql.append(" limit ").append(Integer.toString(getHits()));
+            }
+        }
+        if (getTimeout() != 5000L) {
+            yql.append(" timeout ").append(Long.toString(getTimeout()));
+        }
+        yql.append(';');
+        return yql.toString();
+    }
+
+    private void serializeSorting(StringBuilder yql) {
+        yql.append(" order by ");
+        int initLen = yql.length();
+        for (FieldOrder f : getRanking().getSorting().fieldOrders()) {
+            if (yql.length() > initLen) {
+                yql.append(", ");
+            }
+            final Class<? extends AttributeSorter> sorterType = f.getSorter()
+                    .getClass();
+            if (sorterType == Sorting.RawSorter.class) {
+                yql.append("[{\"").append(YqlParser.SORTING_FUNCTION)
+                        .append("\": \"").append(Sorting.RAW).append("\"}]");
+            } else if (sorterType == Sorting.LowerCaseSorter.class) {
+                yql.append("[{\"").append(YqlParser.SORTING_FUNCTION)
+                        .append("\": \"").append(Sorting.LOWERCASE)
+                        .append("\"}]");
+            } else if (sorterType == Sorting.UcaSorter.class) {
+                Sorting.UcaSorter uca = (Sorting.UcaSorter) f.getSorter();
+                String ucaLocale = uca.getLocale();
+                Sorting.UcaSorter.Strength ucaStrength = uca.getStrength();
+                yql.append("[{\"").append(YqlParser.SORTING_FUNCTION)
+                        .append("\": \"").append(Sorting.UCA).append("\"");
+                if (ucaLocale != null) {
+                    yql.append(", \"").append(YqlParser.SORTING_LOCALE)
+                            .append("\": \"").append(ucaLocale).append('"');
+                }
+                if (ucaStrength != Sorting.UcaSorter.Strength.UNDEFINED) {
+                    yql.append(", \"").append(YqlParser.SORTING_STRENGTH)
+                            .append("\": \"").append(ucaStrength.name())
+                            .append('"');
+                }
+                yql.append("}]");
+            }
+            yql.append(f.getFieldName());
+            if (f.getSortOrder() == Order.DESCENDING) {
+                yql.append(" desc");
+            }
+        }
+    }
+
+    /** Returns the context of this query, possibly creating it if missing. Returns the context, or null */
+    public QueryContext getContext(boolean create) {
+        if (context==null && create)
+            context=new QueryContext(getTraceLevel(),this);
+        return context;
+    }
+
+    /** Returns a hash of this query based on (some of) its content. */
+    @Override
+    public int hashCode() {
+        return ranking.hashCode()+3*presentation.hashCode()+5* model.hashCode()+ 11*offset+ 13*hits;
+    }
+
+    /** Returns whether the given query is equal to this */
+    @Override
+    public boolean equals(Object other) {
+        if (this==other) return true;
+
+        if ( ! (other instanceof Query)) return false;
+        Query q = (Query) other;
+
+        if (getOffset() != q.getOffset()) return false;
+        if (getHits() != q.getHits()) return false;
+        if ( ! getPresentation().equals(q.getPresentation())) return false;
+        if ( ! getRanking().equals(q.getRanking())) return false;
+        if ( ! getModel().equals(q.getModel())) return false;
+
+        // TODO: Compare property settings
+
+        return true;
+    }
+
+    /** Returns a clone of this query */
+    @Override
+    public Query clone() {
+        Query clone = (Query) super.clone();
+        copyPropertiesTo(clone);
+        return clone;
+    }
+
+    private void copyPropertiesTo(Query clone) {
+        clone.model = model.cloneFor(clone);
+        clone.ranking = (Ranking) ranking.clone();
+        clone.presentation = (Presentation) presentation.clone();
+        clone.context = getContext(true).cloneFor(clone);
+
+        if (errors != null)
+            clone.errors = new ArrayList<>(errors);
+
+        // Correct the Query instance in properties
+        clone.properties().setParentQuery(clone);
+        assert (clone.properties().getParentQuery() == clone);
+
+        clone.setTraceLevel(getTraceLevel());
+        clone.setHits(getHits());
+        clone.setOffset(getOffset());
+        clone.setNoCache(getNoCache());
+        clone.setGroupingSessionCache(getGroupingSessionCache());
+    }
+
+    /** Returns the presentation to be used for this query, never null */
+    public Presentation getPresentation() { return presentation; }
+
+    /** Returns the ranking to be used for this query, never null */
+    public Ranking getRanking() { return ranking; }
+
+    /** Returns the query representation model to be used for this query, never null */
+    public Model getModel() { return model; }
+
+    /**
+     * Return the HTTP request which caused this query. This will never be null
+     * when running with queries from the network.
+     * (Except when running with deprecated code paths, in which case this will
+     * return null but getRequest() will not.)
+     */
+     public HttpRequest getHttpRequest() { return httpRequest; }
+
+    /**
+     * Returns the unique and stable session id of this query.
+     *
+     * @param create if true this is created if not already set
+     * @return the session id of this query, or null if not set and create is false
+     */
+    public SessionId getSessionId(boolean create) {
+        if (sessionId == null && create)
+            this.sessionId = SessionId.next();
+        return sessionId;
+    }
+
+    public boolean hasEncodableProperties() {
+        if ( ! ranking.getProperties().isEmpty()) return true;
+        if ( ! ranking.getFeatures().isEmpty()) return true;
+        if ( ranking.getFreshness() != null) return true;
+        if ( model.getSearchPath() != null) return true;
+        if ( model.getDocumentDb() != null) return true;
+        if ( presentation.getHighlight() != null && ! presentation.getHighlight().getHighlightItems().isEmpty()) return true;
+        return false;
+    }
+
+    /**
+     * Encodes properties of this query.
+     *
+     * @param buffer the buffer to encode to
+     * @param encodeQueryData true to encode all properties, false to only include session information, not actual query data
+     * @return the encoded length
+     */
+    public int encodeAsProperties(ByteBuffer buffer, boolean encodeQueryData) {
+        // Make sure we don't encode anything here if we have turned the property feature off
+        // Due to sendQuery we sometimes end up turning this feature on and then encoding a 0 int as the number of
+        // property maps - that's ok (probably we should simplify by just always turning the feature on)
+        if (! hasEncodableProperties()) return 0;
+
+        int start = buffer.position();
+
+        int mapCountPosition = buffer.position();
+        buffer.putInt(0); // map count will go here
+
+        int mapCount = 0;
+
+        // TODO: Push down
+        mapCount += ranking.getProperties().encode(buffer, encodeQueryData);
+        if (encodeQueryData) mapCount += ranking.getFeatures().encode(buffer);
+
+        // TODO: Push down
+        if (encodeQueryData && presentation.getHighlight() != null) mapCount += MapEncoder.encodeStringMultiMap(Highlight.HIGHLIGHTTERMS, presentation.getHighlight().getHighlightTerms(), buffer);
+
+        // TODO: Push down
+        if (encodeQueryData) mapCount += MapEncoder.encodeSingleValue("model", "searchpath", model.getSearchPath(), buffer);
+        mapCount += MapEncoder.encodeSingleValue(DocumentDatabase.MATCH_PROPERTY, DocumentDatabase.SEARCH_DOC_TYPE_KEY, model.getDocumentDb(), buffer);
+
+        mapCount += MapEncoder.encodeMap("caches", createCacheSettingMap(), buffer);
+
+        buffer.putInt(mapCountPosition, mapCount);
+
+        return buffer.position() - start;
+    }
+
+    private Map<String, Boolean> createCacheSettingMap() {
+        if (getGroupingSessionCache() && ranking.getQueryCache()) {
+            Map<String, Boolean> cacheSettingMap = new HashMap<>();
+            cacheSettingMap.put("grouping", true);
+            cacheSettingMap.put("query", true);
+            return cacheSettingMap;
+        }
+        if (getGroupingSessionCache())
+            return Collections.singletonMap("grouping", true);
+        if (ranking.getQueryCache())
+            return Collections.singletonMap("query", true);
+        return Collections.<String,Boolean>emptyMap();
+    }
+
+    /**
+     * Prepares this for binary serialization.
+     * <p>
+     * This must be invoked after all changes have been made to this query before it is passed
+     * on to a receiving backend. Calling it is somewhat expensive, so it should only happen once.
+     * If a prepared query is cloned, it stays prepared.
+     */
+    public void prepare() {
+        getModel().prepare(getRanking());
+        getPresentation().prepare();
+        getRanking().prepare();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java
new file mode 100644
index 00000000000..b6a88200084
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/Result.java
@@ -0,0 +1,365 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search;
+
+import com.yahoo.collections.ListMap;
+import com.yahoo.net.URI;
+import com.yahoo.protect.Validator;
+import com.yahoo.search.query.context.QueryContext;
+import com.yahoo.search.result.*;
+import com.yahoo.search.statistics.ElapsedTime;
+
+import java.util.Iterator;
+
+/**
+ * The Result contains all the data produced by executing a Query: Some very limited global information, and
+ * a single HitGroup containing hits of the result. The HitGroup may contain Hits, which are the individual
+ * result items, as well as further HitGroups, making up a <i>composite</i> structure. This allows the hits of a result
+ * to be hierarchically organized. A Hit is polymorphic and may contain any kind of information deemed
+ * an approriate partial answer to the Query.
+ *
+ * @author bratseth
+ */
+public final class Result extends com.yahoo.processing.Response implements Cloneable {
+
+    // Note to developers: If you think you should add something here you are probably wrong
+    // To add some new kind of data, create a Hit subclass carrying the data and add that instead
+
+    /** The top level hit group of this result */
+    private HitGroup hits;
+
+    /** The estimated total number of hits which would in theory be displayed this result is a part of */
+    private long totalHitCount;
+
+    /**
+     * The estimated total number of <i>deep</i> hits, which includes every object which matches the query.
+     * This is always at least the same as totalHitCount. A lower value will cause hitCount to be returned.
+     */
+    private long deepHitCount;
+
+    /** The time spent producing this result */
+    private ElapsedTime timeAccountant = new ElapsedTime();
+
+    /** Coverage information for this result. */
+    private Coverage coverage = null;
+
+    /**
+     * Headers containing "envelope" meta information to be returned with this result.
+     * Used for HTTP getHeaders when the return protocol is HTTP.
+     */
+    private ListMap<String,String> headers=null;
+
+    /**
+     * Result rendering infrastructure.
+     */
+    private final Templating templating;
+
+    /** Creates a new Result where the top level hit group has id "toplevel" */
+    public Result(Query query) {
+        this(query, new HitGroup("toplevel"));
+    }
+
+    /**
+     * Create an empty result.
+     * A source creating a result is <b>required</b> to call
+     * {@link #setTotalHitCount} before releasing this result.
+     *
+     * @param query the query which produced this result
+     * @param hits the hit container which this will return from {@link #hits()}
+     */
+    @SuppressWarnings("deprecation")
+    public Result(Query query, HitGroup hits) {
+        super(query);
+        if (query==null) throw new NullPointerException("The query reference in a result cannot be null");
+        this.hits=hits;
+        hits.setQuery(query);
+        if (query.getRanking().getSorting() != null) {
+            setHitOrderer(new HitSortOrderer(query.getRanking().getSorting()));
+        }
+        templating = new Templating(this);
+    }
+
+    /** Create a result containing an error */
+    public Result(Query query, ErrorMessage errorMessage) {
+        this(query);
+        hits.setError(errorMessage);
+    }
+
+    /**
+     * Merges <b>meta information</b> from a result into this.
+     * This does not merge hits, but the other information associated
+     * with a result. It should <b>always</b> be called when adding
+     * hits from a result, but there is no constraints on the order of the calls.
+     */
+    @SuppressWarnings("deprecation")
+    public void mergeWith(Result result) {
+        if (templating.usesDefaultTemplate())
+            templating.setRenderer(result.templating.getRenderer());
+        totalHitCount += result.getTotalHitCount();
+        deepHitCount += result.getDeepHitCount();
+        timeAccountant.merge(result.getElapsedTime());
+        boolean create=true;
+        if (result.getCoverage(!create) != null || getCoverage(!create) != null)
+            getCoverage(create).merge(result.getCoverage(create));
+    }
+
+    /**
+     * Merges meta information produced when a Hit already
+     * contained in this result has been filled using another
+     * result as an intermediary. @see mergeWith(Result) mergeWith.
+     */
+    public void mergeWithAfterFill(Result result) {
+        timeAccountant.merge(result.getElapsedTime());
+    }
+
+    /**
+     * Returns the number of hit objects available in the top level group of this result.
+     * Note that this number is allowed to be higher than the requested number
+     * of hits, because a searcher is allowed to add <i>meta</i> hits as well
+     * as the requested number of concrete hits.
+     */
+    public int getHitCount() {
+        return hits.size();
+    }
+
+    /**
+     * <p>Returns the total number of concrete hits contained (directly or in subgroups) in this result.
+     * This should equal the requested hits count if the query has that many matches.</p>
+     */
+    public int getConcreteHitCount() {
+        return hits.getConcreteSize();
+    }
+
+    /**
+     * Returns the <b>estimated</b> total number of concrete hits which would be returned for this query.
+     */
+    public long getTotalHitCount() {
+        return totalHitCount;
+    }
+
+    /**
+     * Returns the estimated total number of <i>deep</i> hits, which includes every object which matches the query.
+     * This is always at least the same as totalHitCount. A lower value will cause hitCount to be returned.
+     */
+    public long getDeepHitCount() {
+        if (deepHitCount<totalHitCount) return totalHitCount;
+        return deepHitCount;
+    }
+
+
+    /** Sets the estimated total number of hits this result is a subset of */
+    public void setTotalHitCount(long totalHitCount) {
+        this.totalHitCount = totalHitCount;
+    }
+
+    /** Sets the estimated total number of deep hits this result is a subset of */
+    public void setDeepHitCount(long deepHitCount) {
+        this.deepHitCount = deepHitCount;
+    }
+
+    public ElapsedTime getElapsedTime() {
+        return timeAccountant;
+    }
+
+    public void setElapsedTime(ElapsedTime t) {
+        timeAccountant = t;
+    }
+
+    /**
+     * Returns true only if _all_ hits in this result originates from a cache.
+     */
+    public boolean isCached() {
+        return hits.isCached();
+    }
+
+    /**
+     * Returns whether all hits in this result have been filled with
+     * the properties contained in the given summary class. Note that
+     * this method will also return true if no hits in this result are
+     * fillable.
+     */
+    public boolean isFilled(String summaryClass) {
+        return hits.isFilled(summaryClass);
+    }
+
+    /** Returns the query which produced this result */
+    public Query getQuery() { return hits.getQuery(); }
+
+    /** Sets a query for this result */
+    public void setQuery(Query query) { hits.setQuery(query); }
+
+    /**
+     * <p>Sets the hit orderer to be used for the top level hit group.</p>
+     *
+     * @param hitOrderer the new hit orderer, or null to use default relevancy ordering
+     */
+    public void setHitOrderer(HitOrderer hitOrderer) { hits.setOrderer(hitOrderer); }
+
+    /** Returns the orderer used by the top level group, or null if the default relevancy order is used */
+    public HitOrderer getHitOrderer() { return hits.getOrderer(); }
+
+    public void setDeletionBreaksOrdering(boolean flag) { hits.setDeletionBreaksOrdering(flag); }
+
+    public boolean getDeletionBreaksOrdering() { return hits.getDeletionBreaksOrdering(); }
+
+    /** Update cached and filled by iterating through the hits of this result */
+    public void analyzeHits() { hits.analyze(); }
+
+    /** Returns the top level hit group containing all the hits of this result */
+    public HitGroup hits() { return hits; }
+
+    @Override
+    public com.yahoo.processing.response.DataList<?> data() {
+        return hits;
+    }
+
+
+    /** Sets the top level hit group containing all the hits of this result */
+    public void setHits(HitGroup hits) {
+        Validator.ensureNotNull("The top-level hit group of " + this,hits);
+        this.hits=hits;
+    }
+
+    /**
+     * Deep clones this result - copies are made of all hits and subgroups of hits,
+     * <i>but not of the query referenced by this</i>.
+     */
+    public Result clone() {
+        Result resultClone = (Result) super.clone();
+
+        resultClone.hits = hits.clone();
+
+        resultClone.getTemplating().setRenderer(null); // TODO: Kind of wrong
+        resultClone.setElapsedTime(new ElapsedTime());
+        return resultClone;
+    }
+
+
+    public String toString() {
+        if (hits.getError() != null) {
+            return "Result: " + hits.getErrorHit().errors().iterator().next();
+        } else {
+            return "Result (" + getConcreteHitCount() + " of total " + getTotalHitCount() + " hits)";
+        }
+    }
+
+    /**
+     * Adds a context message to this query containing the entire content of this result,
+     * if tracelevel is 5 or more.
+     *
+     * @param name the name of the searcher instance returning this result
+     */
+    public void trace(String name) {
+        if (hits().getQuery().getTraceLevel() < 5) {
+            return;
+        }
+        StringBuilder hitBuffer = new StringBuilder(name);
+
+        hitBuffer.append(" returns:\n");
+        int counter = 0;
+
+        for (Iterator<Hit> i = hits.unorderedIterator(); i.hasNext();) {
+            Hit hit = i.next();
+
+            if (hit.isMeta()) continue;
+
+            hitBuffer.append("  #: ");
+            hitBuffer.append(counter);
+
+            traceExtraHitProperties(hitBuffer, hit);
+
+            hitBuffer.append(", relevancy: ");
+            hitBuffer.append(hit.getRelevance());
+
+            hitBuffer.append(", addno: ");
+            hitBuffer.append(hit.getAddNumber());
+
+            hitBuffer.append(", source: ");
+            hitBuffer.append(hit.getSource());
+
+            hitBuffer.append(", uri: ");
+            URI uri = hit.getId();
+
+            if (uri != null) {
+                hitBuffer.append(uri.getHost());
+            } else {
+                hitBuffer.append("(no uri)");
+            }
+            hitBuffer.append("\n");
+            counter++;
+        }
+        if (counter == 0) {
+            hitBuffer.append("(no hits)\n");
+        }
+        hits.getQuery().trace(hitBuffer.toString(), false, 5);
+    }
+
+    /**
+     * For tracing custom properties of a hit, see trace(String). An example of
+     * using this is in com.yahoo.prelude.Result.
+     *
+     * @param hitBuffer
+     *                the render target
+     * @param hit
+     *                the hit to be analyzed
+     */
+    protected void traceExtraHitProperties(StringBuilder hitBuffer, Hit hit) {
+        return;
+    }
+
+    /** Returns the context of this result - this is equal to getQuery().getContext(create) */
+    public QueryContext getContext(boolean create) { return getQuery().getContext(create); }
+
+    public void setCoverage(Coverage coverage) { this.coverage = coverage; }
+
+    // Coverage a part of tracing?
+    // Coverage logic might me moved around, but it should not be a part of tracing.
+    // Coverage is status information about access to a corpus, tracing is voluntary,
+    // diagnostic search status.
+    /**
+     * Returns coverage information
+     *
+     * @param create if true the coverage information of this result is created if missing
+     * @return the coverage information of this, or null if none and create is false
+     */
+    public Coverage getCoverage(boolean create) {
+        if (coverage == null && create) {
+            if (hits.getError() == null) {
+                // No error here implies full coverage.
+                // Don't count this as a result set if there's no data - avoid counting empty results made
+                // to simplify code paths
+                coverage = new Coverage(0L, 0, true, (hits().size()==0 ? 0 : 1));
+            } else {
+                coverage = new Coverage(0L, 0, false);
+            }
+        }
+        return coverage;
+    }
+
+    /**
+     * Returns the set of "envelope" headers to be returned with this result.
+     * This returns the live map in modifiable form - modify this to change the
+     * headers. Or null if none, and it should not be created.
+     * <p>
+     * Used for HTTP headers when the return protocol is HTTP, e.g
+     * <pre>result.getHeaders(true).put("Cache-Control","max-age=120")</pre>
+     *
+     * @param create if true, create the header ListMap if it does not exist
+     * @return returns the ListMap of current headers, or null if no headers are set and <pre>create</pre> is false
+     */
+    public ListMap<String, String> getHeaders(boolean create) {
+        if (headers == null && create)
+            headers = new ListMap<>();
+        return headers;
+    }
+
+    /**
+     * The Templating object contains helper methods and data containers for
+     * result rendering.
+     *
+     * @return helper object for result rendering
+     */
+    public Templating getTemplating() {
+        return templating;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/Searcher.java b/container-search/src/main/java/com/yahoo/search/Searcher.java
new file mode 100644
index 00000000000..95b4f92ca56
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/Searcher.java
@@ -0,0 +1,175 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.processing.Processor;
+import com.yahoo.processing.Response;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.logging.Logger;
+
+/**
+ * Superclass of all {@link com.yahoo.component.Component Components} which produces Results in response to
+ * Queries by calling the {@link #search search} method.
+ * <p>
+ * Searchers are participants in <i>chain of responsibility</i> {@link com.yahoo.search.searchchain.SearchChain search chains}
+ * where they passes the Queries downwards by synchroneously calling the next Searcher in the chain, and returns the
+ * Results back up as the response.
+ * <p>
+ * Any Searcher may
+ * <ul>
+ * <li>Do modifications to the Query before passing it on (a <i>query rerwiter</i>)
+ * <li>Do modifications to the Result before passing it on up, e.g removing altering, reorganizing or adding Hits
+ * (a <i>result processor</i>)
+ * <li>Pass the Query on to multiple other search chains, either in series
+ * (by creating a new {@link com.yahoo.search.searchchain.Execution} for each chain), or in parallel (by creating a
+ * {@link com.yahoo.search.searchchain.AsyncExecution}) (a <i>federator</i>)
+ * <li>Create a Result and pass it back up, either by calling some other node(s) to get the data, or by creating the
+ * Result from internal data (a <i>source</i>)
+ * <li>Pass some query on downwards multiple times, or in different ways, typically each time depending of the Result
+ * returned the last time (a <i>workflow</i>)
+ * </ul>
+ *
+ * <p>...or some combination of the above of course. Note that as Searchers work synchronously, any information can be
+ * retained on the stack in the Searcher from the Query is received until the Result is returned simply by declaring
+ * variables for the data in the search method (or whatever it calls), and for the same reason workflows are
+ * implemented as Java code. However, searchers are executed by many threads, for different Queries, in parallell, so
+ * any mutable data shared between queries (and hence stored as instance members must be accessed multithread safely.
+ * In many cases, shared data can simply be instantiated in the constructor and used in read-only mode afterwards
+ * <p>
+ * <b>Searcher lifecycle:</b> A searcher has a simple life-cycle:
+ *
+ * <ul>
+ * <li><b>Construction: While a constructor is running.</b> A searcher is handed its id and configuration
+ * (if any) in the constructor. During construction, the searcher should build any in-memory structures needed.
+ * A new instance of the searcher will be created when the configuration is changed.
+ * Constructors are called with this priority:
+ *
+ *   <ul>
+ *     <li>The constructor taking a ComponentId, followed by the highest number of config classes (subclasses of
+ *         {@link com.yahoo.config.ConfigInstance}) as arguments.
+ *     <li>The constructor taking a string id followed by the highest number of config classes as arguments.
+ *     <li>The constructor taking only the highest number of config classes as arguments.
+ *     <li>The constructor taking a ComponentId as the only argument
+ *     <li>The constructor taking a string id as the only argument
+ *     <li>The default (no-argument) constructor.
+ *   </ul>
+ *
+ * If none of these constructors are declared, searcher construction will fail.
+ *
+ * <li><b>In service: After the constructor has returned.</b> In this phase, searcher service methods are
+ * called at any time by multiple threads in parallel.
+ * Implementations should avoid synchronization and access to volatiles as much as possible by keeping
+ * data structures build in construction read-only.
+ *
+ * <li><b>Deconstruction: While deconstruct is running.</b> All Searcher service method calls have completed when
+ * this method is called. When it returns, the searcher will be eligible for garbage collection.
+ *
+ * </ul>
+ *
+ * @author bratseth
+ */
+public abstract class Searcher extends Processor {
+
+    // Note to developers: If you think you should add something here you are probably wrong
+    //                     Create a subclass containing the new method instead.
+
+    private final Logger logger = Logger.getLogger(getClass().getName());
+
+    public Searcher() {}
+
+    /** Creates a searcher from an id */
+    public Searcher(ComponentId id) {
+        super();
+        initId(id);
+    }
+
+    /**
+     * Override this to implement your searcher.
+     * <p>
+     * Searcher implementation subclasses will, depending on their type of logic, do one of the following:
+     * <ul>
+     * <li><b>Query processors:</b> Access the query, then call execution.search and return the result
+     * <li><b>Result processors:</b> Call execution.search to get the result, access it and return
+     * <li><b>Sources</b> (which produces results): Create a result, add the desired hits and return it.
+     * <li><b>Federators</b> (which forwards the search to multiple subchains): Call search on the
+     * desired subchains in parallel and get the results. Combine the results to one and return it.
+     * <li><b>Workflows:</b> Call execution.search as many times as desired, using different queries.
+     * Eventually return a result.
+     * </ul>
+     * <p>
+     * Hits come in two kinds - <i>concrete hits</i> are actual
+     * content of the kind requested by the user, <i>meta hits</i> are
+     * hits which provides information about the collection of hits,
+     * on the query, the service and so on.
+     * <p>
+     * The query specifies a window into a larger result list that must be returned from the searcher
+     * through <i>hits</i> and <i>offset</i>;
+     * Searchers which returns list of hits in the top level in the result
+     * must return at least <i>hits</i> number of hits (or if impossible; all that are available),
+     * starting at the given offset.
+     * In addition, searchers are allowed to return
+     * any number of meta hits (although this number is expected to be low).
+     * For hits contained in nested hit groups, the concept of a window defined by hits and offset
+     * is not well defined and does not apply.
+     * <p>
+     * Error handling in searchers:
+     * <ul>
+     * <li>Unexpected events: Throw any RuntimeException. This query will fail
+     * with the exception message, and the error will be logged
+     * <li>Expected events: Create (new Result(Query, ErrorMessage) or add
+     * result.setErrorIfNoOtherErrors(ErrorMessage) an error message to the Result.
+     * <li>Recoverable user errors: Add a FeedbackHit explaining the condition
+     * and how to correct it.
+     * </ul>
+     *
+     * @param query the query
+     * @return the result of making this query
+     */
+    public abstract Result search(Query query,Execution execution);
+
+    /** Use the search method in Searcher processors. This forwards to it. */
+    @Override
+    public final Response process(com.yahoo.processing.Request request, com.yahoo.processing.execution.Execution execution) {
+        return search((Query)request,(Execution)execution);
+    }
+
+    /**
+     * Fill hit properties with data using the given summary class.
+     * Calling this on already filled results has no cost.
+     * <p>
+     * This needs to be overridden by <i>federating</i> searchers to contact search sources again by
+     * propagating the fill call down through the search chain, and by <i>source</i> searchers
+     * which talks to fill capable backends to request the data to be filled. Other searchers do
+     * not need to override this.
+     *
+     * @param result the result to fill
+     * @param summaryClass the name of the collection of fields to fetch the values of, or null to use the default
+     */
+    public void fill(Result result, String summaryClass, Execution execution) {
+        execution.fill(result,summaryClass);
+    }
+
+    /**
+     * Fills the result if it is not already filled for the given summary class.
+     * See the fill method.
+     **/
+    public final void ensureFilled(Result result, String summaryClass, Execution execution) {
+        if (summaryClass == null)
+            summaryClass = result.getQuery().getPresentation().getSummary();
+
+        if (!result.isFilled(summaryClass)) {
+            fill(result, summaryClass, execution);
+        }
+    }
+
+    /** Returns a logger unique for the instance subclass */
+    protected Logger getLogger() { return logger; }
+
+    /** Returns "searcher 'getId()'" */
+    public @Override String toString() {
+        return "searcher '" + getIdString() + "'";
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/app/.gitignore b/container-search/src/main/java/com/yahoo/search/app/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/app/.gitignore
diff --git a/container-search/src/main/java/com/yahoo/search/cache/package-info.java b/container-search/src/main/java/com/yahoo/search/cache/package-info.java
new file mode 100644
index 00000000000..292b491c52b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cache/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Cache package, exported to keep the ignored legacy cache config around until Vespa 7.
+ *
+ * @author bratseth
+ */
+@ExportPackage
+package com.yahoo.search.cache;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java
new file mode 100644
index 00000000000..de67369a231
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+import java.util.logging.Logger;
+
+import com.yahoo.search.result.ErrorMessage;
+
+
+/**
+ * A node monitor is responsible for maintaining the state of a monitored node.
+ * It has the following properties:
+ * <ul>
+ * <li>A node is taken out of operation if it fails</li>
+ * <li>A node is put back in operation when it responds correctly again
+ *     <i>responseAfterFailLimit</i> times <b>unless</b>
+ *     it has failed <i>failQuarantineLimit</i>. In the latter case it won't
+ *     be put into operation again before that time period has expired</li>
+ * </ul>
+ *
+ * @author bratseth
+ */
+public abstract class BaseNodeMonitor<T> {
+
+    protected static Logger log=Logger.getLogger(BaseNodeMonitor.class.getName());
+
+    /** The object representing the monitored node */
+    protected T node;
+
+    protected boolean isWorking=true;
+
+    /** Whether this node is quarantined for unstability */
+    protected boolean isQuarantined=false;
+
+    /** The last time this node failed, in ms */
+    protected long failedAt=0;
+
+    /** The last time this node responded (failed or succeeded), in ms */
+    protected long respondedAt=0;
+
+    /** The last time this node responded successfully */
+    protected long succeededAt=0;
+
+    /** The configuration of this monitor */
+    protected MonitorConfiguration configuration;
+
+    /** Is the node we monitor part of an internal Vespa cluster or not */
+    private boolean internal=false;
+
+    public BaseNodeMonitor(boolean internal) {
+        this.internal=internal;
+    }
+
+    public T getNode() { return node; }
+
+    /**
+     * Returns whether this node is currently in a state suitable
+     * for receiving traffic. As far as we know, that is
+     */
+    public boolean isWorking() { return isWorking; }
+
+    public boolean isQuarantined() { return isQuarantined; }
+
+    /**
+     * Called when this node fails.
+     *
+     * @param error a description of the error
+     */
+    public abstract void failed(ErrorMessage error);
+
+    /**
+     * Called when a response is received from this node. If the node was
+     * quarantined and it has been in that state for more than QuarantineTime
+     * milliseconds, it is taken out of quarantine.
+     *
+     * if it is not in quarantine but is not working, it may be set to working
+     * if this method is called at least responseAfterFailLimit times
+     */
+    public abstract void responded();
+
+    public boolean isIdle() {
+        return (now()-respondedAt) >= configuration.getIdleLimit();
+    }
+
+    protected long now() {
+        return System.currentTimeMillis();
+    }
+
+    /** Thread-safely changes the state of this node if required */
+    protected abstract void setWorking(boolean working,String explanation);
+
+    /** Returns whether or not this is monitoring an internal node. Default is false. */
+    public boolean isInternal() { return internal; }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
new file mode 100644
index 00000000000..1c50ea5d904
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
@@ -0,0 +1,157 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+
+import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.search.result.ErrorMessage;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Monitors of a cluster of remote nodes.
+ * The monitor uses an internal thread for node monitoring.
+ * All <i>public</i> methods of this class are multithread safe.
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class ClusterMonitor<T> {
+
+    private MonitorConfiguration configuration=new MonitorConfiguration();
+
+    private static Logger log=Logger.getLogger(ClusterMonitor.class.getName());
+
+    private NodeManager<T> nodeManager;
+
+    private MonitorThread monitorThread;
+
+    private volatile boolean shutdown = false;
+
+    /** A map from Node to corresponding MonitoredNode */
+    private Map<T,BaseNodeMonitor<T>> nodeMonitors=
+        Collections.synchronizedMap(new java.util.LinkedHashMap<T, BaseNodeMonitor<T>>());
+
+    public ClusterMonitor(NodeManager<T> manager, String monitorConfigID) {
+        nodeManager=manager;
+        monitorThread=new MonitorThread("search.clustermonitor");
+        monitorThread.start();
+        log.fine("checkInterval is " + configuration.getCheckInterval()+" ms");
+    }
+
+    /** Returns the configuration of this cluster monitor */
+    public MonitorConfiguration getConfiguration() { return configuration; }
+
+    /**
+     * Adds a new node for monitoring.
+     * The object representing the node must
+     * <ul>
+     * <li>Have a sensible toString</li>
+     * <li>Have a sensible identity (equals and hashCode)</li>
+     * </ul>
+     *
+     * @param node the object representing the node
+     * @param internal whether or not this node is internal to this cluster
+     */
+    public void add(T node,boolean internal) {
+        BaseNodeMonitor<T> monitor=new TrafficNodeMonitor<>(node,configuration,internal);
+        // BaseNodeMonitor monitor=new NodeMonitor(node,configuration);
+        nodeMonitors.put(node,monitor);
+    }
+
+    /**
+     * Returns the monitor of the given node, or null if this node has not been added
+     */
+    public BaseNodeMonitor<T> getNodeMonitor(T node) {
+        return nodeMonitors.get(node);
+    }
+
+    /** Called from ClusterSearcher/NodeManager when a node failed */
+    public synchronized void failed(T node, ErrorMessage error) {
+        BaseNodeMonitor<T> monitor=nodeMonitors.get(node);
+        boolean wasWorking=monitor.isWorking();
+        monitor.failed(error);
+        if (wasWorking && !monitor.isWorking()) {
+            nodeManager.failed(node);
+        }
+    }
+
+    /** Called when a node responded */
+    public synchronized void responded(T node) {
+        BaseNodeMonitor<T> monitor = nodeMonitors.get(node);
+        boolean wasFailing=!monitor.isWorking();
+        monitor.responded();
+        if (wasFailing && monitor.isWorking()) {
+            nodeManager.working(monitor.getNode());
+        }
+    }
+
+    /**
+     * Ping all nodes which needs pinging to discover state changes
+     */
+    public void ping(Executor executor) {
+        for (Iterator<BaseNodeMonitor<T>> i=nodeMonitorIterator(); i.hasNext(); ) {
+            BaseNodeMonitor<T> monitor= i.next();
+            // always ping
+            // if (monitor.isIdle())
+            nodeManager.ping(monitor.getNode(),executor); // Cause call to failed or responded
+        }
+    }
+
+    /** Returns a thread-safe snapshot of the NodeMonitors of all added nodes */
+    public Iterator<BaseNodeMonitor<T>> nodeMonitorIterator() {
+        return nodeMonitors().iterator();
+    }
+
+    /** Returns a thread-safe snapshot of the NodeMonitors of all added nodes */
+    public List<BaseNodeMonitor<T>> nodeMonitors() {
+        synchronized (nodeMonitors) {
+            return new java.util.ArrayList<>(nodeMonitors.values());
+        }
+    }
+
+    /** Must be called when this goes out of use */
+    public void shutdown() {
+        shutdown = true;
+        monitorThread.interrupt();
+    }
+
+    private class MonitorThread extends Thread {
+        MonitorThread(String name) {
+            super(name);
+        }
+
+        public void run() {
+            log.fine("Starting cluster monitor thread");
+            // Pings must happen in a separate thread from this to handle timeouts
+            // By using a cached thread pool we ensured that 1) a single thread will be used
+            // for all pings when there are no problems (important because it ensures that
+            // any thread local connections are reused) 2) a new thread will be started to execute
+            // new pings when a ping is not responding
+            Executor pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping"));
+            while (!isInterrupted()) {
+                try {
+                    Thread.sleep(configuration.getCheckInterval());
+                    log.finest("Activating ping");
+                    ping(pingExecutor);
+                }
+                catch (Exception e) {
+                    if (shutdown && e instanceof InterruptedException) {
+                        break;
+                    } else {
+                        log.log(Level.WARNING,"Error in monitor thread",e);
+                    }
+                }
+            }
+            log.fine("Stopped cluster monitor thread");
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
new file mode 100644
index 00000000000..da3d0d8e20b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
@@ -0,0 +1,374 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.container.protect.Error;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.Ping;
+import com.yahoo.prelude.Pong;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.cluster.Hasher.NodeList;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.List;
+import java.util.concurrent.*;
+
+/**
+ * Implements clustering (failover and load balancing) over a set of client
+ * connections to a homogenous cluster of nodes. Searchers which wants to make
+ * clustered connections to some service should use this.
+ * <p>
+ * This replaces the usual searcher methods by ones which have the same contract
+ * and semantics but which takes an additional parameter which is the Connection
+ * selected by the cluster searcher which the method should use. Overrides of
+ * these connection methods <i>must not</i> call the super methods to pass on
+ * but must use the methods on execution.
+ * <p>
+ * The type argument is the class (of any type) representing the connections.
+ * The connection objects should implement a good toString to ease diagnostics.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ */
+public abstract class ClusterSearcher<T> extends PingableSearcher implements NodeManager<T> {
+
+    private Hasher<T> hasher = new Hasher<>();
+    private ClusterMonitor<T> monitor = new ClusterMonitor<>(this, "dummy");
+
+    /**
+     * Creates a new cluster searcher
+     *
+     * @param id
+     *                the id of this searcher
+     * @param connections
+     *                the connections of the cluster
+     * @param internal
+     *                whether or not this cluster is internal (part of the same
+     *                installation)
+     */
+    public ClusterSearcher(ComponentId id, List<T> connections, boolean internal) {
+        this(id, connections, new Hasher<T>(), internal);
+    }
+
+    public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal) {
+        super(id);
+        this.hasher = hasher;
+        for (T connection : connections) {
+            monitor.add(connection, internal);
+            hasher.add(connection);
+        }
+    }
+
+    /**
+     * Pinging a node by sending a query NodeManager method, called from
+     * ClusterMonitor
+     */
+    public final @Override void ping(T p, Executor executor) {
+        log(LogLevel.FINE, "Sending ping to: ", p);
+        Pinger pinger = new Pinger(p);
+        FutureTask<Pong> future = new FutureTask<>(pinger);
+
+        executor.execute(future);
+        Pong pong;
+        Throwable logThrowable = null;
+
+        try {
+            pong = future.get(monitor.getConfiguration().getFailLimit(),
+                    TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            pong = new Pong();
+            pong.addError(ErrorMessage
+                    .createUnspecifiedError("Ping was interrupted: " + p));
+            logThrowable = e;
+        } catch (ExecutionException e) {
+            pong = new Pong();
+            pong.addError(ErrorMessage
+                    .createUnspecifiedError("Execution was interrupted: " + p));
+            logThrowable = e;
+        } catch (LinkageError e) { // Typically Osgi woes
+            pong = new Pong();
+            pong.addError(ErrorMessage.createErrorInPluginSearcher("Class loading problem",e));
+            logThrowable=e;
+        } catch (TimeoutException e) {
+            pong = new Pong();
+            pong.addError(ErrorMessage
+                    .createNoAnswerWhenPingingNode("Ping thread timed out."));
+        }
+        future.cancel(true);
+
+        if (pong.badResponse()) {
+            monitor.failed(p, pong.getError(0));
+            log(LogLevel.FINE, "Failed ping - ", pong);
+        } else {
+            monitor.responded(p);
+            log(LogLevel.FINE, "Answered ping - ", p);
+        }
+
+        if (logThrowable != null) { // This looks strange, but yes - it is
+                                    // needed
+            String logMsg;
+            if (logThrowable instanceof TimeoutException) {
+                logMsg = "Ping timed out for " + getId().getName() + ".";
+            } else {
+                StackTraceElement[] trace = logThrowable.getStackTrace();
+                String traceAsString = null;
+                if (trace != null) {
+                    StringBuilder b = new StringBuilder(": ");
+                    for (StackTraceElement k : trace) {
+                        if (k == null) {
+                            b.append("null\n");
+                        } else {
+                            b.append(k.toString()).append('\n');
+                        }
+                    }
+                    traceAsString = b.toString();
+                }
+                logMsg = "Caught " + logThrowable.getClass().getName()
+                        + " exception in " + getId().getName() + " ping"
+                        + (trace == null ? ", no stack trace available." : traceAsString);
+            }
+            getLogger().warning(logMsg);
+        }
+
+    }
+
+    /**
+     * Pings this connection. Pings may be sent "out of band" at any time by the
+     * monitoring subsystem to determine the status of this connection. If the
+     * ping fails, it is ok both to set an error in the pong or to throw an
+     * exception.
+     */
+    protected abstract Pong ping(Ping ping, T connection);
+
+    protected T getFirstConnection(NodeList<T> nodes, int code, int trynum, Query query) {
+        return nodes.select(code, trynum);
+    }
+
+    @Override
+    public final Result search(Query query, Execution execution) {
+        int tries = 0;
+
+        Hasher.NodeList<T> nodes = getHasher().getNodes();
+
+        if (nodes.getNodeCount() == 0)
+            return search(query, execution, ErrorMessage
+                    .createNoBackendsInService("No nodes in service in " + this + " (" + monitor.nodeMonitors().size()
+                                               + " was configured, none is responding)"));
+
+        int code = query.hashCode();
+        Result result;
+        T connection = getFirstConnection(nodes, code, tries, query);
+        do {
+            // The loop is in case there are other searchers available
+            // able to produce results
+            if (connection == null)
+                return search(query, execution, ErrorMessage
+                        .createNoBackendsInService("No in node could handle " + query + " according to " +
+                                                   hasher + " in " + this));
+            if (timedOut(query))
+                return new Result(query, ErrorMessage.createTimeout("No time left for searching"));
+
+            if (query.getTraceLevel() >= 8)
+                query.trace("Trying " + connection, false, 8);
+
+            result = robustSearch(query, execution, connection);
+
+            if (!shouldRetry(query, result))
+                return result;
+
+            if (query.getTraceLevel() >= 6)
+                query.trace("Error from connection " + connection + " : " + result.hits().getError(), false, 6);
+
+            if (result.hits().getError().getCode() == Error.TIMEOUT.code)
+                return result; // Retry is unlikely to help
+
+            log(LogLevel.FINER, "No result, checking for timeout.");
+            tries++;
+            connection = nodes.select(code, tries);
+        } while (tries < nodes.getNodeCount());
+
+        // only error result gets returned here.
+        return result;
+
+    }
+
+    /**
+     * Returns whether this query and result should be retried against another
+     * connection if possible. This default implementation returns true if the
+     * result contains some error.
+     */
+    protected boolean shouldRetry(Query query, Result result) {
+        return result.hits().getError() != null;
+    }
+
+    /**
+     * This is called (instead of search(quer,execution,connextion) to handle
+     * searches where no (suitable) backend was available. The default
+     * implementation returns an error result.
+     */
+    protected Result search(Query query, Execution execution, ErrorMessage message) {
+        return new Result(query, message);
+    }
+
+    /**
+     * Call search(Query,Execution,T) and handle any exceptions returned which
+     * we do not want to propagate upwards By default this catches all runtime
+     * exceptions and puts them into the result
+     */
+    protected Result robustSearch(Query query, Execution execution, T connection) {
+        Result result;
+        try {
+            result = search(query, execution, connection);
+        } catch (RuntimeException e) { //TODO: Exceptions should not be used to signal backend communication errors
+            log(LogLevel.WARNING, "An exception occurred while invoking backend searcher.", e);
+            result = new Result(query, ErrorMessage
+                    .createBackendCommunicationError("Failed calling "
+                            + connection + " in " + this + " for " + query
+                            + ": " + Exceptions.toMessageString(e)));
+        }
+
+        if (result == null)
+            result = new Result(query, ErrorMessage
+                    .createBackendCommunicationError("No result returned in "
+                            + this + " from " + connection + " for " + query));
+
+        if (result.hits().getError() != null) {
+            log(LogLevel.FINE, "FAILED: ", query);
+        } else if (!result.isCached()) {
+            log(LogLevel.FINE, "WORKING: ", query);
+        } else {
+            log(LogLevel.FINE, "CACHE HIT: ", query);
+        }
+        return result;
+    }
+
+    /**
+     * Perform the search against the given connection. Return a result
+     * containing an error or throw an exception on failures.
+     */
+    protected abstract Result search(Query query, Execution execution, T connection);
+
+    public @Override
+    final void fill(Result result, String summaryClass, Execution execution) {
+        Query query = result.getQuery();
+        Hasher.NodeList<T> nodes = getHasher().getNodes();
+        int code = query.hashCode();
+
+        T connection = nodes.select(code, 0);
+        if (connection != null) {
+            if (timedOut(query)) {
+                result.hits().addError(
+                        ErrorMessage.createTimeout(
+                                "No time left to get summaries for "
+                                + result));
+            } else {
+                // query.setTimeout(getNodeTimeout(query));
+                doFill(connection, result, summaryClass, execution);
+            }
+        } else {
+            result.hits().addError(
+                    ErrorMessage.createNoBackendsInService("Could not fill '"
+                            + result + "' in '" + this + "'"));
+        }
+    }
+
+    private void doFill(T connection, Result result, String summaryClass, Execution execution) {
+        try {
+            fill(result, summaryClass, execution, connection);
+        } catch (RuntimeException e) {
+            result.hits().addError(
+                    ErrorMessage
+                            .createBackendCommunicationError("Error filling "
+                                    + result + " from " + connection + ": "
+                                    + Exceptions.toMessageString(e)));
+        }
+        if (result.hits().getError() != null) {
+            log(LogLevel.FINE, "FAILED: ", result.getQuery());
+        } else if (!result.isCached()) {
+            log(LogLevel.FINE, "WORKING: ", result.getQuery());
+        } else {
+            log(LogLevel.FINE, "CACHE HIT: " + result.getQuery());
+        }
+    }
+
+    /**
+     * Perform the fill against the given connection. Add an error to the result
+     * or throw an exception on failures.
+     */
+    protected abstract void fill(Result result, String summaryClass,
+            Execution execution, T connection);
+
+    /** NodeManager method, called from ClusterMonitor */
+    public @Override
+    void working(T node) {
+        getHasher().add(node);
+    }
+
+    /** NodeManager method, called from ClusterMonitor */
+    public @Override
+    void failed(T node) {
+        getHasher().remove(node);
+    }
+
+    /**
+     * Returns the hasher used internally in this. Do not mutate this hasher
+     * while in use.
+     */
+    public Hasher<T> getHasher() {
+        return hasher;
+    }
+
+    /** Returns the monitor of these nodes */
+    public ClusterMonitor<T> getMonitor() {
+        return monitor;
+    }
+
+    /** Returns true if this query has timed out now */
+    protected boolean timedOut(Query query) {
+        long duration = query.getDurationTime();
+        return duration >= query.getTimeout();
+    }
+
+    protected void log(java.util.logging.Level level, Object... objects) {
+        if (!getLogger().isLoggable(level))
+            return;
+        StringBuilder sb = new StringBuilder();
+        for (Object object : objects) {
+            sb.append(object);
+        }
+        getLogger().log(level, sb.toString());
+    }
+
+    public @Override void deconstruct() {
+        super.deconstruct();
+        monitor.shutdown();
+    }
+
+    private class Pinger implements Callable<Pong> {
+
+        private T connection;
+
+        public Pinger(T connection) {
+            this.connection = connection;
+        }
+
+        public Pong call() {
+            Pong pong;
+            try {
+                pong = ping(new Ping(monitor.getConfiguration().getRequestTimeout()), connection);
+            } catch (RuntimeException e) {
+                pong = new Pong();
+                pong.addError(
+                        ErrorMessage.createBackendCommunicationError(
+                                "Exception when pinging "
+                                + connection + ": "
+                                + Exceptions.toMessageString(e)));
+            }
+            return pong;
+        }
+
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java
new file mode 100644
index 00000000000..7ef71a7968d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+/**
+ * A hasher load balances between a set of nodes, represented by object ids.
+ *
+ * @author Arne B Fossaa
+ * @author bratseth
+ * @author Prashanth B. Bhat
+ */
+public class Hasher<T> {
+
+    public static class NodeFactor<T> {
+        private final T node;
+        /**
+         * The relative weight of the different nodes.
+         * Hashing are based on the proportions of the weights.
+         */
+        private final int load;
+        public NodeFactor(T node, int load) {
+            this.node = node;
+            this.load = load;
+        }
+        public final T getNode() { return node; }
+        public final int    getLoad() { return load; }
+    }
+
+    public static class NodeList<T> {
+        private final NodeFactor<T>[] nodes;
+
+        private int totalLoadFactor;
+
+        public NodeList(NodeFactor<T>[] nodes) {
+            this.nodes = nodes;
+            totalLoadFactor = 0;
+            if(nodes != null) {
+                for(NodeFactor<T> node:nodes) {
+                    totalLoadFactor += node.getLoad();
+                }
+            }
+        }
+
+        public int getNodeCount() {
+            return nodes.length;
+        }
+
+        public  T select(int code, int trynum) {
+            if (totalLoadFactor <= 0) return null;
+
+            // Multiply by a prime number much bigger than the likely number of hosts
+            int hashValue=(Math.abs(code*76103)) % totalLoadFactor;
+            int sumLoad=0;
+            int targetNode=0;
+            for (targetNode=0; targetNode<nodes.length; targetNode++) {
+                sumLoad +=nodes[targetNode].getLoad();
+                if (sumLoad > hashValue)
+                    break;
+            }
+            // Skip the ones we have tried before.
+            targetNode += trynum;
+            targetNode %= nodes.length;
+            return nodes[targetNode].getNode();
+        }
+
+        public boolean hasNode(T node) {
+            for(int i = 0;i<nodes.length;i++) {
+                if(node == nodes[i].getNode()) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+    }
+
+    private volatile NodeList<T> nodes;
+
+    @SuppressWarnings("unchecked")
+    public Hasher() {
+        this.nodes = new NodeList<T>(new NodeFactor[0]);
+    }
+
+    /** Adds a node with load factor 100 */
+    public void add(T node) {
+        add(node,100);
+    }
+
+    /**
+     * Adds a code with a load factor.
+     * The load factor is relative to the load of the other added nodes
+     * and determines how often this node will be selected compared
+     * to the other nodes
+     */
+    public synchronized void add(T node,int load) {
+        assert(nodes != null);
+        if(!nodes.hasNode(node)) {
+            NodeFactor<T>[] oldNodes = nodes.nodes;
+            @SuppressWarnings("unchecked")
+            NodeFactor<T>[] newNodes = (NodeFactor<T>[]) new NodeFactor[oldNodes.length+ 1];
+            System.arraycopy(oldNodes,0,newNodes,0,oldNodes.length);
+            newNodes[newNodes.length-1] = new NodeFactor<>(node, load);
+
+            //Atomic switch due to volatile
+            nodes = new NodeList<>(newNodes);
+        }
+    }
+
+    /** Removes a node */
+    public synchronized void remove(T node) {
+        if( nodes.hasNode(node)) {
+            NodeFactor<T>[] oldNodes = nodes.nodes;
+            @SuppressWarnings("unchecked")
+            NodeFactor<T>[] newNodes = (NodeFactor<T>[]) new NodeFactor[oldNodes.length - 1];
+            for (int i = 0, j = 0; i < oldNodes.length; i++) {
+                if (oldNodes[i].getNode() != node) {
+                    newNodes[j++] = oldNodes[i];
+                }
+            }
+            // An atomic switch due to volatile.
+            nodes = new NodeList<>(newNodes);
+        }
+    }
+
+    /**
+     * Returns a list of nodes that are up.
+     */
+    public NodeList<T> getNodes() {
+        return nodes;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
new file mode 100644
index 00000000000..c68b60a743b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+/**
+ * The configuration of a cluster monitor instance
+ *
+ * @author bratseth
+ */
+public class MonitorConfiguration  {
+
+    /**
+     * The interval in ms between consecutive checks of the monitored
+     * nodes
+     */
+    private long checkInterval=1000;
+
+    /**
+     * The number of times a failed node must respond before getting
+     * traffic again
+     */
+    private int responseAfterFailLimit=3;
+
+    /**
+     * The number of ms a node is allowed to stay idle before it is
+     * pinged
+     */
+    private long idleLimit=3000;
+
+    /**
+     * The number of milliseconds to attempt to complete a request
+     * before giving up
+     */
+    private long requestTimeout = 5000;
+
+    /**
+     * The number of milliseconds a node is allowed to fail before we
+     * mark it as not working
+     */
+    private long failLimit=5000;
+
+    /**
+     * The number of times a node is allowed to fail in one hour
+     * before it is quarantined for an hour
+     */
+    private int failQuarantineLimit=3;
+
+    /**
+     * The number of ms to quarantine an unstable node
+     */
+    private long quarantineTime=1000*60*60;
+
+    /**
+     * Sets the interval between each ping of idle or failing nodes
+     * Default is 1000ms
+     */
+    public void setCheckInterval(long intervalMs) {
+        this.checkInterval=intervalMs;
+    }
+
+    /**
+     * Returns the interval between each ping of idle or failing nodes
+     * Default is 1000ms
+     */
+    public long getCheckInterval() {
+        return checkInterval;
+    }
+
+    /**
+     * Sets the number of times a failed node must respond before it is put
+     * back in service. Default is 3.
+     */
+    public void setResponseAfterFailLimit(int responseAfterFailLimit) {
+        this.responseAfterFailLimit=responseAfterFailLimit;
+    }
+
+    /**
+     * Sets the number of ms a node (failing or working) is allowed to
+     * stay idle before it is pinged. Default is 3000
+     */
+    public void setIdleLimit(int idleLimit) {
+        this.idleLimit=idleLimit;
+    }
+
+    /**
+     * Gets the number of ms a node (failing or working)
+     * is allowed to stay idle before it is pinged. Default is 3000
+     */
+    public long getIdleLimit() {
+        return idleLimit;
+    }
+
+    /**
+     * Returns the number of milliseconds to attempt to service a request
+     * (at different nodes) before giving up. Default is 5000 ms.
+     */
+    public long getRequestTimeout() { return requestTimeout; }
+
+    /**
+     * Sets the number of milliseconds a node is allowed to fail before we
+     * mark it as not working
+     */
+    public void setFailLimit(long failLimit) { this.failLimit=failLimit; }
+
+    /**
+     * Returns the number of milliseconds a node is allowed to fail before we
+     * mark it as not working
+     */
+    public long getFailLimit() { return failLimit; }
+
+    /**
+     * The number of times a node must fail in one hour to be placed
+     * in quarantine.  Once in quarantine it won't be put back in
+     * productuion before quarantineTime has expired even if it is
+     * working. Default is 3
+     */
+    public void setFailQuarantineLimit(int failQuarantineLimit) {
+        this.failQuarantineLimit=failQuarantineLimit;
+    }
+
+    /**
+     * The number of ms an unstable node is quarantined. Default is
+     * 100*60*60
+     */
+    public void setQuarantineTime(long quarantineTime) {
+        this.quarantineTime=quarantineTime;
+    }
+
+    public String toString() {
+        return "monitor configuration [" +
+            "checkInterval: " + checkInterval +
+            " responseAfterFailLimit: " + responseAfterFailLimit +
+            " idleLimit: " + idleLimit +
+            " requestTimeout " + requestTimeout +
+            " feilLimit " + failLimit +
+            " failQuerantineLimit " + failQuarantineLimit +
+            " quarantineTime " + quarantineTime +
+            "]";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
new file mode 100644
index 00000000000..7071867c8c7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+import java.util.concurrent.Executor;
+
+/**
+ * Must be implemented by a node collection which wants
+ * it's node state monitored by a ClusterMonitor
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+public interface NodeManager<T> {
+
+    /** Called when a failed node is working (ready for production) again */
+    public void working(T node);
+
+    /** Called when a working node fails */
+    public void failed(T node);
+
+    /** Called when a node should be pinged */
+    public void ping(T node, Executor executor);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/PingableSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/PingableSearcher.java
new file mode 100644
index 00000000000..486473eba8d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/PingableSearcher.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.prelude.Ping;
+import com.yahoo.prelude.Pong;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * A searcher to which we can send a ping to probe if it is alive
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public abstract class PingableSearcher extends Searcher {
+
+    public PingableSearcher() {
+    }
+
+    public PingableSearcher(ComponentId id) {
+        super(id);
+    }
+
+    /** Send a ping request downwards to probe if this searcher chain is in functioning order */
+    public Pong ping(Ping ping, Execution execution) {
+        return execution.ping(ping);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java
new file mode 100644
index 00000000000..6464f0101be
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java
@@ -0,0 +1,93 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.cluster;
+
+import com.yahoo.search.result.ErrorMessage;
+
+
+/**
+ * This node monitor is responsible for maintaining the state of a monitored node.
+ * It has the following properties:
+ * <ul>
+ * <li>A node is taken out of operation if it gives no response in 10 s</li>
+ * <li>A node is put back in operation when it responds correctly again
+ * </ul>
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> {
+    /**
+     * Creates a new node monitor for a node
+     */
+    public TrafficNodeMonitor(T node,MonitorConfiguration configuration,boolean internal) {
+        super(internal);
+        this.node=node;
+        this.configuration=configuration;
+    }
+
+    /** Whether or not this has ever responded successfully */
+    private boolean atStartUp = true;
+
+    public T getNode() { return node; }
+
+    /**
+     * Called when this node fails.
+     *
+     * @param error A container which should contain a short description
+     */
+    @Override
+    public void failed(ErrorMessage error) {
+        respondedAt=now();
+
+        switch (error.getCode()) {
+            // TODO: Remove hard coded error messages.
+            // Refer to docs/errormessages
+            case 10:
+            case 11:
+                // Only count not being able to talk to backend at all
+                // as errors we care about
+                if ((respondedAt-succeededAt) > 10000) {
+                    setWorking(false,"Not working for 10 s: " + error.toString());
+                }
+                break;
+            default:
+                succeededAt = respondedAt;
+                break;
+        }
+    }
+
+    /**
+     * Called when a response is received from this node.
+     */
+    public void responded() {
+        respondedAt=now();
+        succeededAt=respondedAt;
+        atStartUp = false;
+
+        if (!isWorking) {
+            setWorking(true,"Responds correctly");
+        }
+    }
+
+    /** Thread-safely changes the state of this node if required */
+    protected synchronized void setWorking(boolean working,String explanation) {
+        if (this.isWorking==working) return; // Old news
+
+        if (explanation==null) {
+            explanation="";
+        } else {
+            explanation=": " + explanation;
+        }
+
+        if (working) {
+            log.info("Putting " + node + " in service" + explanation);
+        }
+        else {
+            if (!atStartUp || !isInternal())
+                log.warning("Taking " + node + " out of service" + explanation);
+            failedAt=now();
+        }
+
+        this.isWorking=working;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/package-info.java b/container-search/src/main/java/com/yahoo/search/cluster/package-info.java
new file mode 100644
index 00000000000..b470d8c8150
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/cluster/package-info.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Standard searchers to compose in <i>source</i> search chains (those containing searchers specific for one source and
+ * which ends with a call to some provider) which calls a cluster of provider nodes. These searchers provides hashing
+ * and failover of the provider nodes.
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.cluster;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/config/dispatchprototype/package-info.java b/container-search/src/main/java/com/yahoo/search/config/dispatchprototype/package-info.java
new file mode 100644
index 00000000000..2a7b4f96aa8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/config/dispatchprototype/package-info.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Package for dispatchprototype config.
+ * @author tonytv
+ */
+@ExportPackage
+package com.yahoo.search.config.dispatchprototype;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/config/package-info.java b/container-search/src/main/java/com/yahoo/search/config/package-info.java
new file mode 100644
index 00000000000..84eb92be0ea
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/config/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.search.config;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/debug/BackendStatistics.java b/container-search/src/main/java/com/yahoo/search/debug/BackendStatistics.java
new file mode 100644
index 00000000000..8086048890f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/BackendStatistics.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import static com.yahoo.search.debug.SearcherUtils.clusterSearchers;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.ArrayUtils;
+
+import com.yahoo.fs4.mplex.Backend;
+import com.yahoo.jrt.Int32Array;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.StringArray;
+import com.yahoo.jrt.Value;
+import com.yahoo.jrt.Values;
+import com.yahoo.prelude.cluster.ClusterSearcher;
+import com.yahoo.yolean.Exceptions;
+
+/**
+ * @author tonytv
+ */
+public class BackendStatistics implements DebugMethodHandler {
+    public JrtMethodSignature getSignature() {
+        String returnTypes = "" + (char)Value.STRING_ARRAY + (char)Value.INT32_ARRAY + (char)Value.INT32_ARRAY;
+        String parametersTypes = "" + (char)Value.STRING;
+
+        return new JrtMethodSignature(returnTypes, parametersTypes);
+    }
+
+    public void invoke(Request request) {
+        try {
+            Collection<ClusterSearcher> searchers = clusterSearchers(request);
+            List<String> backendIdentificators = new ArrayList<>();
+            List<Integer> activeConnections = new ArrayList<>();
+            List<Integer> totalConnections = new ArrayList<>();
+
+            for (ClusterSearcher searcher : searchers) {
+                for (Map.Entry<String,Backend.BackendStatistics> statistics : searcher.getBackendStatistics().entrySet()) {
+                    backendIdentificators.add(statistics.getKey());
+                    activeConnections.add(statistics.getValue().activeConnections);
+                    totalConnections.add(statistics.getValue().totalConnections());
+                }
+            }
+            Values returnValues = request.returnValues();
+            returnValues.add(new StringArray(backendIdentificators.toArray(new String[0])));
+            addInt32Array(returnValues, activeConnections);
+            addInt32Array(returnValues, totalConnections);
+
+        } catch (Exception e) {
+            request.setError(1000, Exceptions.toMessageString(e));
+        }
+    }
+
+    private void addInt32Array(Values returnValues, List<Integer> ints) {
+        returnValues.add(new Int32Array(ArrayUtils.toPrimitive(ints.toArray(new Integer[0]))));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/debug/DebugMethodHandler.java b/container-search/src/main/java/com/yahoo/search/debug/DebugMethodHandler.java
new file mode 100644
index 00000000000..55f36b9670e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/DebugMethodHandler.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import com.yahoo.jrt.MethodHandler;
+
+/**
+ * A method handler that can describe its signature.
+ *
+ * @author tonytv
+ */
+interface DebugMethodHandler extends MethodHandler {
+    JrtMethodSignature getSignature();
+}
diff --git a/container-search/src/main/java/com/yahoo/search/debug/DebugRpcAdaptor.java b/container-search/src/main/java/com/yahoo/search/debug/DebugRpcAdaptor.java
new file mode 100644
index 00000000000..2309f23985c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/DebugRpcAdaptor.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import com.yahoo.container.osgi.AbstractRpcAdaptor;
+import com.yahoo.jrt.Method;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.fs4.PacketDumper.PacketType;
+
+/**
+ * Handles rpc calls for retrieving debug information.
+ *
+ * @author tonytv
+ */
+public final class DebugRpcAdaptor extends AbstractRpcAdaptor {
+    private static final String debugPrefix = "debug.";
+
+    public void bindCommands(Supervisor supervisor) {
+        addTraceMethod(supervisor, "query", PacketType.query);
+        addTraceMethod(supervisor, "result", PacketType.result);
+        addMethod(supervisor, "output-search-chain", new OutputSearchChain());
+        addMethod(supervisor, "backend-statistics", new BackendStatistics());
+    }
+
+    private void addTraceMethod(Supervisor supervisor, String name, PacketType packetType) {
+        addMethod(supervisor, constructTraceMethodName(name), new TracePackets(packetType));
+    }
+
+    private void addMethod(Supervisor supervisor, String name, DebugMethodHandler handler) {
+        JrtMethodSignature typeStrings = handler.getSignature();
+        supervisor.addMethod(
+                new Method(debugPrefix + name,
+                        typeStrings.parametersTypes,
+                        typeStrings.returnTypes,
+                        handler));
+
+    }
+
+    //example: debug.dump-query-packets
+    private String constructTraceMethodName(String name) {
+        return debugPrefix + "dump-" + name + "-packets";
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/debug/IndentStringBuilder.java b/container-search/src/main/java/com/yahoo/search/debug/IndentStringBuilder.java
new file mode 100644
index 00000000000..acb9be8294f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/IndentStringBuilder.java
@@ -0,0 +1,102 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import java.io.Serializable;
+
+/**
+ * A StringBuilder that also handles indentation for append operations.
+ * @author tonytv
+ */
+@SuppressWarnings("serial")
+final class IndentStringBuilder implements Serializable, Appendable, CharSequence {
+    private final StringBuilder builder = new StringBuilder();
+    private final String singleIndentation;
+
+    private int level = 0;
+    private boolean newline = true;
+
+    private void appendIndentation() {
+        if (newline) {
+            for (int i=0; i<level; i++) {
+                builder.append(singleIndentation);
+            }
+        }
+        newline  = false;
+    }
+
+    public IndentStringBuilder(String singleIndentation) {
+        this.singleIndentation = singleIndentation;
+    }
+
+    public IndentStringBuilder() {
+        this("  ");
+    }
+
+    public void resetIndentLevel(int level) {
+        this.level = level;
+    }
+
+    //returns the indent level before indenting.
+    public int newlineAndIndent() {
+        newline();
+        return indent();
+    }
+
+    //returns the indent level before indenting.
+    public int indent() {
+        return level++;
+    }
+
+    public IndentStringBuilder newline() {
+        newline = true;
+        builder.append('\n');
+        return this;
+    }
+
+    public IndentStringBuilder append(Object o) {
+        appendIndentation();
+        builder.append(o);
+        return this;
+    }
+
+    public IndentStringBuilder append(String s) {
+        appendIndentation();
+        builder.append(s);
+        return this;
+    }
+
+    public IndentStringBuilder append(CharSequence charSequence) {
+        appendIndentation();
+        builder.append(charSequence);
+        return this;
+    }
+
+    public IndentStringBuilder append(CharSequence charSequence, int i, int i1) {
+        appendIndentation();
+        builder.append(charSequence, i, i1);
+        return this;
+    }
+
+    public IndentStringBuilder append(char c) {
+        appendIndentation();
+        builder.append(c);
+        return this;
+    }
+
+    public String toString() {
+        return builder.toString();
+    }
+
+    public int length() {
+        return builder.length();
+    }
+
+    public char charAt(int i) {
+        return builder.charAt(i);
+    }
+
+    public CharSequence subSequence(int i, int i1) {
+        return builder.subSequence(i, i1);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/debug/JrtMethodSignature.java b/container-search/src/main/java/com/yahoo/search/debug/JrtMethodSignature.java
new file mode 100644
index 00000000000..0383360487f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/JrtMethodSignature.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+/**
+ * Represents the signatures of a jrt method.
+ *
+ * @author tonytv
+ */
+final class JrtMethodSignature {
+    final String returnTypes;
+    final String parametersTypes;
+
+    JrtMethodSignature(String returnTypes, String parametersTypes) {
+        this.returnTypes = returnTypes;
+        this.parametersTypes = parametersTypes;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/debug/OutputSearchChain.java b/container-search/src/main/java/com/yahoo/search/debug/OutputSearchChain.java
new file mode 100644
index 00000000000..4413ea462c8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/OutputSearchChain.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import static com.yahoo.protect.Validator.ensureNotNull;
+
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.jrt.Value;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.handler.SearchHandler;
+import com.yahoo.search.searchchain.SearchChain;
+import com.yahoo.search.searchchain.SearchChainRegistry;
+
+/**
+ * Outputs a human readable representation of a given search chain.
+ *
+ * @author tonytv
+ */
+final class OutputSearchChain implements DebugMethodHandler {
+      private String getSearchChainName(Request request) {
+        final int numParameters = request.parameters().size();
+
+        if (numParameters == 0)
+            return SearchHandler.defaultSearchChainName;
+        else if (numParameters == 1)
+            return request.parameters().get(0).asString();
+        else
+            throw new RuntimeException("Too many parameters given.");
+    }
+
+    private SearchChain getSearchChain(SearchChainRegistry registry, String searchChainName) {
+        SearchChain searchChain = registry.getComponent(searchChainName);
+        ensureNotNull("There is no search chain named '" + searchChainName + "'", searchChain);
+        return searchChain;
+    }
+
+    public JrtMethodSignature getSignature() {
+        String returnTypes = "" + (char)Value.STRING;
+        String parametersTypes = "*"; //optional string
+        return new JrtMethodSignature(returnTypes, parametersTypes);
+    }
+
+    public void invoke(Request request) {
+        try {
+            SearchHandler searchHandler = SearcherUtils.getSearchHandler();
+            SearchChainRegistry searchChainRegistry = searchHandler.getSearchChainRegistry();
+            SearchChain searchChain = getSearchChain(searchChainRegistry,
+                    getSearchChainName(request));
+
+            SearchChainTextRepresentation textRepresentation = new SearchChainTextRepresentation(searchChain, searchChainRegistry);
+            request.returnValues().add(new StringValue(textRepresentation.toString()));
+        } catch (Exception e) {
+            request.setError(1000, Exceptions.toMessageString(e));
+        }
+    }
+
+
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/debug/SearchChainTextRepresentation.java b/container-search/src/main/java/com/yahoo/search/debug/SearchChainTextRepresentation.java
new file mode 100644
index 00000000000..2e9da99f85b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/SearchChainTextRepresentation.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.ForkingSearcher;
+import com.yahoo.search.searchchain.SearchChain;
+import com.yahoo.search.searchchain.SearchChainRegistry;
+
+import java.util.Collection;
+
+/**
+ * Text representation of a given search chain intended for debugging purposes.
+ *
+ * @author tonytv
+ */
+public class SearchChainTextRepresentation {
+
+    private final SearchChainRegistry searchChainRegistry;
+
+    private static class Block {
+        private static final String openBlock = " {";
+        private static final char closeBlock = '}';
+        private final IndentStringBuilder str;
+        private final int level;
+
+        Block(IndentStringBuilder str) {
+            this.str = str;
+            level = str.append(openBlock).newlineAndIndent();
+        }
+
+        void close() {
+            str.resetIndentLevel(level);
+            str.append(closeBlock).newline();
+        }
+    }
+
+    private final String textRepresentation;
+
+    private void outputChain(IndentStringBuilder str, Chain<Searcher> chain) {
+        if (chain == null) {
+            str.append(" [Unresolved Searchchain]");
+        } else {
+            str.append(chain.getId()).append(" [Searchchain] ");
+            Block block = new Block(str);
+
+            for (Searcher searcher : chain.components())
+                outputSearcher(str, searcher);
+
+            block.close();
+        }
+    }
+
+    private void outputSearcher(IndentStringBuilder str, Searcher searcher) {
+        str.append(searcher.getId()).append(" [Searcher]");
+        if ( ! (searcher instanceof ForkingSearcher) ) {
+            str.newline();
+            return;
+        }
+        Collection<ForkingSearcher.CommentedSearchChain> chains =
+                ((ForkingSearcher)searcher).getSearchChainsForwarded(searchChainRegistry);
+        if (chains.isEmpty()) {
+            str.newline();
+            return;
+        }
+        Block block = new Block(str);
+        for (ForkingSearcher.CommentedSearchChain chain : chains) {
+            if (chain.comment != null)
+                str.append(chain.comment).newline();
+            outputChain(str, chain.searchChain);
+        }
+        block.close();
+    }
+
+    @Override
+    public String toString() {
+        return textRepresentation;
+    }
+
+    public SearchChainTextRepresentation(SearchChain searchChain, SearchChainRegistry searchChainRegistry) {
+        this.searchChainRegistry = searchChainRegistry;
+
+        IndentStringBuilder stringBuilder = new IndentStringBuilder();
+        outputChain(stringBuilder, searchChain);
+        textRepresentation = stringBuilder.toString();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/debug/SearcherUtils.java b/container-search/src/main/java/com/yahoo/search/debug/SearcherUtils.java
new file mode 100644
index 00000000000..1633196a585
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/SearcherUtils.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import static com.yahoo.protect.Validator.ensureNotNull;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import com.yahoo.component.provider.ComponentRegistry;
+import org.apache.commons.collections.CollectionUtils;
+
+import com.yahoo.container.Container;
+import com.yahoo.jrt.Request;
+import com.yahoo.prelude.cluster.ClusterSearcher;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.handler.SearchHandler;
+import com.yahoo.search.searchchain.SearchChainRegistry;
+
+/**
+ * Utility functions for searchers and search chains.
+ *
+ * @author tonytv
+ */
+final class SearcherUtils {
+    private static Collection<Searcher> allSearchers() {
+        SearchChainRegistry searchChainRegistry = getSearchHandler().getSearchChainRegistry();
+        ComponentRegistry<Searcher> searcherRegistry = searchChainRegistry.getSearcherRegistry();
+        return searcherRegistry.allComponents();
+    }
+
+    private static Collection<ClusterSearcher> allClusterSearchers() {
+        return filter(allSearchers(), ClusterSearcher.class);
+    }
+
+    private static <T> Collection<T> filter(Collection<?> collection, Class<T> classToMatch) {
+        List<T> filtered = new ArrayList<>();
+        for (Object candidate : collection) {
+            if (classToMatch.isInstance(candidate))
+                filtered.add(classToMatch.cast(candidate));
+        }
+        return filtered;
+    }
+
+    public static Collection<ClusterSearcher> clusterSearchers(final String clusterName) {
+        Collection<ClusterSearcher> searchers = allClusterSearchers();
+        CollectionUtils.filter(searchers,
+                o -> clusterName.equalsIgnoreCase(((ClusterSearcher)o).getClusterModelName()));
+        return searchers;
+    }
+
+    //Return value is never null
+    static SearchHandler getSearchHandler() {
+        SearchHandler searchHandler = (SearchHandler) Container.get().getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.SearchHandler");
+        ensureNotNull("The standard search handler is not available.", searchHandler);
+        return searchHandler;
+    }
+
+    //Retrieve all the cluster searchers as specified by the first parameter of the request.
+    static Collection<ClusterSearcher> clusterSearchers(Request request) {
+        String clusterName = request.parameters().get(0).asString();
+        Collection<ClusterSearcher> searchers = clusterSearchers(clusterName);
+        if (searchers.isEmpty())
+            throw new RuntimeException("No cluster named " + clusterName);
+        return searchers;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/debug/TracePackets.java b/container-search/src/main/java/com/yahoo/search/debug/TracePackets.java
new file mode 100644
index 00000000000..de71b2e3f26
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/debug/TracePackets.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.debug;
+
+import static com.yahoo.search.debug.SearcherUtils.clusterSearchers;
+
+import java.util.Collection;
+
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.Value;
+import com.yahoo.prelude.cluster.ClusterSearcher;
+import com.yahoo.fs4.PacketDumper;
+import com.yahoo.yolean.Exceptions;
+
+/**
+ * Rpc method for enabling packet dumping for a specific packet type.
+ *
+ * @author tonytv
+ */
+final class TracePackets implements DebugMethodHandler {
+    private final PacketDumper.PacketType packetType;
+
+    public void invoke(Request request) {
+        try {
+            Collection<ClusterSearcher> searchers = clusterSearchers(request);
+            boolean on = request.parameters().get(1).asInt8() != 0;
+
+            for (ClusterSearcher searcher : searchers)
+                searcher.dumpPackets(packetType, on);
+
+        } catch (Exception e) {
+            request.setError(1000, Exceptions.toMessageString(e));
+        }
+    }
+
+    TracePackets(PacketDumper.PacketType packetType) {
+        this.packetType = packetType;
+    }
+
+    public JrtMethodSignature getSignature() {
+        String returnTypes = "";
+        String parametersTypes = "" + (char)Value.STRING + (char)Value.INT8;
+        return new JrtMethodSignature(returnTypes, parametersTypes);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/Client.java
new file mode 100644
index 00000000000..19d6a0c523b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Client.java
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.prelude.fastsearch.FastHit;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A dispatch client.
+ *
+ * @author bratseth
+ */
+interface Client {
+
+    void getDocsums(List<FastHit> hits, NodeConnection node, CompressionType compression,
+                    int uncompressedLength, byte[] compressedSlime, Dispatcher.GetDocsumsResponseReceiver responseReceiver,
+                    double timeoutSeconds);
+
+    /** Creates a connection to a particular node in this */
+    NodeConnection createConnection(String hostname, int port);
+
+    class GetDocsumsResponseOrError {
+
+        // One of these will be non empty and the other not
+        private Optional<GetDocsumsResponse> response;
+        private Optional<String> error;
+
+        public static GetDocsumsResponseOrError fromResponse(GetDocsumsResponse response) {
+            return new GetDocsumsResponseOrError(Optional.of(response), Optional.empty());
+        }
+
+        public static GetDocsumsResponseOrError fromError(String error) {
+            return new GetDocsumsResponseOrError(Optional.empty(), Optional.of(error));
+        }
+
+        private GetDocsumsResponseOrError(Optional<GetDocsumsResponse> response, Optional<String> error) {
+            this.response = response;
+            this.error = error;
+        }
+
+        /** Returns the response, or empty if there is an error */
+        public Optional<GetDocsumsResponse> response() { return response; }
+
+        /** Returns the error or empty if there is a response */
+        public Optional<String> error() { return error; }
+
+    }
+
+    class GetDocsumsResponse {
+
+        private final byte compression;
+        private final int uncompressedSize;
+        private final byte[] compressedSlimeBytes;
+        private final List<FastHit> hitsContext;
+
+        public GetDocsumsResponse(byte compression, int uncompressedSize, byte[] compressedSlimeBytes, List<FastHit> hitsContext) {
+            this.compression = compression;
+            this.uncompressedSize = uncompressedSize;
+            this.compressedSlimeBytes = compressedSlimeBytes;
+            this.hitsContext = hitsContext;
+        }
+
+        public byte compression() {
+            return compression;
+        }
+
+        public int uncompressedSize() {
+            return uncompressedSize;
+        }
+
+        public byte[] compressedSlimeBytes() {
+            return compressedSlimeBytes;
+        }
+
+        public List<FastHit> hitsContext() {
+            return hitsContext;
+        }
+
+    }
+
+    interface NodeConnection {
+
+        /** Closes this connection */
+        void close();
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
new file mode 100644
index 00000000000..e4d1fb0b1d5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -0,0 +1,228 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import com.yahoo.collections.ListMap;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.compress.CompressionType;
+import com.yahoo.compress.Compressor;
+import com.yahoo.data.access.slime.SlimeAdapter;
+import com.yahoo.prelude.fastsearch.FastHit;
+import com.yahoo.prelude.fastsearch.TimeoutException;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.slime.BinaryFormat;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.data.access.Inspector;
+import com.yahoo.vespa.config.search.DispatchConfig;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A dispatcher communicates with search nodes to (in the future) perform queries and (now) fill hits.
+ * This class is multithread safe.
+ *
+ * @author bratseth
+ */
+public class Dispatcher extends AbstractComponent {
+
+    private final static Logger log = Logger.getLogger(Dispatcher.class.getName());
+    private final Client client;
+
+    /** Connections to the search nodes this talks to, indexed by node id ("partid") */
+    private final ImmutableMap<Integer, Client.NodeConnection> nodes;
+
+    private final Compressor compressor = new Compressor();
+
+    @Inject
+    public Dispatcher(DispatchConfig dispatchConfig) {
+        this.client = new RpcClient();
+        ImmutableMap.Builder<Integer, Client.NodeConnection> nodesBuilder = new ImmutableMap.Builder<>();
+        for (DispatchConfig.Node node : dispatchConfig.node()) {
+            nodesBuilder.put(node.key(), client.createConnection(node.host(), node.port()));
+        }
+        nodes = nodesBuilder.build();
+    }
+
+    /** For testing */
+    public Dispatcher(Map<Integer, Client.NodeConnection> nodeConnections, Client client) {
+        this.nodes = ImmutableMap.copyOf(nodeConnections);
+        this.client = client;
+    }
+
+    /** Fills the given summary class by sending RPC requests to the right search nodes */
+    public void fill(Result result, String summaryClass, CompressionType compression) {
+        try {
+            ListMap<Integer, FastHit> hitsByNode = hitsByNode(result);
+
+            GetDocsumsResponseReceiver responseReceiver = new GetDocsumsResponseReceiver(hitsByNode.size(), compressor, result);
+            for (Map.Entry<Integer, List<FastHit>> nodeHits : hitsByNode.entrySet()) {
+                sendGetDocsumsRequest(nodeHits.getKey(), nodeHits.getValue(), summaryClass, compression, result, responseReceiver);
+            }
+            responseReceiver.processResponses(result.getQuery());
+        }
+        catch (TimeoutException e) {
+            result.hits().addError(ErrorMessage.createTimeout("Summary data is incomplete: " + e.getMessage()));
+        }
+    }
+
+    /** Return a map of hits by their search node (partition) id */
+    private ListMap<Integer, FastHit> hitsByNode(Result result) {
+        ListMap<Integer, FastHit> hitsByPartition = new ListMap<>();
+        for (Iterator<Hit> i = result.hits().deepIterator() ; i.hasNext(); ) {
+            Hit h = i.next();
+            if ( ! (h instanceof FastHit)) continue;
+            FastHit hit = (FastHit)h;
+
+            hitsByPartition.put(hit.getDistributionKey(), hit);
+        }
+        return hitsByPartition;
+    }
+
+    /** Send a getDocsums request to a node. Responses will be added to the given receiver. */
+    private void sendGetDocsumsRequest(int nodeId, List<FastHit> hits, String summaryClass,
+                                       CompressionType compression,
+                                       Result result, GetDocsumsResponseReceiver responseReceiver) {
+        Client.NodeConnection node = nodes.get(nodeId);
+        if (node == null) {
+            result.hits().addError(ErrorMessage.createEmptyDocsums("Could not fill hits from unknown node " + nodeId));
+            log.warning("Got hits with partid " + nodeId + ", which is not included in the current dispatch config");
+            return;
+        }
+
+        byte[] serializedSlime = BinaryFormat.encode(toSlime(summaryClass, hits));
+        double timeoutSeconds = ((double)result.getQuery().getTimeLeft()-3.0)/1000.0;
+        Compressor.Compression compressionResult = compressor.compress(compression, serializedSlime);
+        client.getDocsums(hits, node, compressionResult.type(),
+                          serializedSlime.length, compressionResult.data(), responseReceiver, timeoutSeconds);
+    }
+
+    public Slime toSlime(String summaryClass, List<FastHit> hits) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        if (summaryClass != null) {
+            root.setString("class", summaryClass);
+        }
+        Cursor gids = root.setArray("gids");
+        for (FastHit hit : hits) {
+            gids.addData(hit.getGlobalId().getRawId());
+        }
+        return slime;
+    }
+
+    @Override
+    public void deconstruct() {
+        for (Client.NodeConnection nodeConnection : nodes.values())
+            nodeConnection.close();
+    }
+
+    /** Receiver of the responses to a set of getDocsums requests */
+    public static class GetDocsumsResponseReceiver {
+
+        private final BlockingQueue<Client.GetDocsumsResponseOrError> responses;
+        private final Compressor compressor;
+        private final Result result;
+
+        /** Whether we have already logged/notified about an error - to avoid spamming */
+        private boolean hasReportedError = false;
+
+        /** The number of responses we should receive (and process) before this is complete */
+        private int outstandingResponses;
+
+        public GetDocsumsResponseReceiver(int requestCount, Compressor compressor, Result result) {
+            this.compressor = compressor;
+            responses = new LinkedBlockingQueue<>(requestCount);
+            outstandingResponses = requestCount;
+            this.result = result;
+        }
+
+        /** Called by a thread belonging to the client when a valid response becomes available */
+        public void receive(Client.GetDocsumsResponseOrError response) {
+            responses.add(response);
+        }
+
+        private void throwTimeout() throws TimeoutException {
+            throw new TimeoutException("Timed out waiting for summary data. " + outstandingResponses + " responses outstanding.");
+        }
+
+        /**
+         * Call this from the dispatcher thread to initiate and complete processing of responses.
+         * This will block until all responses are available and processed, or to timeout.
+         */
+        public void processResponses(Query query) throws TimeoutException {
+            try {
+                while (outstandingResponses > 0) {
+                    long timeLeftMs = query.getTimeLeft();
+                    if (timeLeftMs <= 0) {
+                        throwTimeout();
+                    }
+                    Client.GetDocsumsResponseOrError response = responses.poll(timeLeftMs, TimeUnit.MILLISECONDS);
+                    if (response == null)
+                        throwTimeout();
+                    processResponse(response);
+                    outstandingResponses--;
+                }
+            }
+            catch (InterruptedException e) {
+                // TODO: Add error
+            }
+        }
+
+        private void processResponse(Client.GetDocsumsResponseOrError responseOrError) {
+            if (responseOrError.error().isPresent()) {
+                if (hasReportedError) return;
+                String error = responseOrError.error().get();
+                result.hits().addError(ErrorMessage.createBackendCommunicationError(error));
+                log.log(Level.WARNING, "Error fetching summary data: "+ error);
+            }
+            else {
+                Client.GetDocsumsResponse response = responseOrError.response().get();
+                CompressionType compression = CompressionType.valueOf(response.compression());
+                byte[] slimeBytes = compressor.decompress(response.compressedSlimeBytes(), compression, response.uncompressedSize());
+                fill(response.hitsContext(), slimeBytes);
+            }
+        }
+
+        private void fill(List<FastHit> hits, byte[] slimeBytes) {
+            Inspector summaries = new SlimeAdapter(BinaryFormat.decode(slimeBytes).get().field("docsums"));
+            if ( ! summaries.valid())
+                throw new IllegalArgumentException("Expected a Slime root object containing a 'docsums' field");
+            for (int i = 0; i < hits.size(); i++) {
+                fill(hits.get(i), summaries.entry(i).field("docsum"));
+            }
+        }
+
+        private void fill(FastHit hit, Inspector summary) {
+            summary.traverse((String name, Inspector value) -> {
+                hit.setField(name, nativeTypeOf(value));
+            });
+        }
+
+        private Object nativeTypeOf(Inspector inspector) {
+            switch (inspector.type()) {
+                case ARRAY: return inspector;
+                case OBJECT: return inspector;
+                case BOOL: return inspector.asBool();
+                case DATA: return inspector.asData();
+                case DOUBLE: return inspector.asDouble();
+                case LONG: return inspector.asLong();
+                case STRING: return inspector.asString(); // TODO: Keep as utf8
+                case EMPTY : return null;
+                default: throw new IllegalArgumentException("Unexpected Slime type " + inspector.type());
+            }
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/RpcClient.java
new file mode 100644
index 00000000000..0305b06e92f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/RpcClient.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch;
+
+import com.yahoo.compress.CompressionType;
+import com.yahoo.jrt.DataValue;
+import com.yahoo.jrt.Int32Value;
+import com.yahoo.jrt.Int8Value;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.RequestWaiter;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Target;
+import com.yahoo.jrt.Transport;
+import com.yahoo.jrt.Values;
+import com.yahoo.prelude.fastsearch.FastHit;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * A client which uses rpc request to search nodes to implement the Client API.
+ *
+ * @author bratseth
+ */
+class RpcClient implements Client {
+
+    private final Supervisor supervisor = new Supervisor(new Transport());
+
+    @Override
+    public NodeConnection createConnection(String hostname, int port) {
+        return new RpcNodeConnection(hostname, port, supervisor);
+    }
+
+    @Override
+    public void getDocsums(List<FastHit> hits, NodeConnection node, CompressionType compression, int uncompressedLength,
+                           byte[] compressedSlime, Dispatcher.GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds) {
+        Request request = new Request("proton.getDocsums");
+        request.parameters().add(new Int8Value(compression.getCode()));
+        request.parameters().add(new Int32Value(uncompressedLength));
+        request.parameters().add(new DataValue(compressedSlime));
+
+        request.setContext(hits);
+        RpcNodeConnection rpcNode = ((RpcNodeConnection) node);
+        rpcNode.invokeAsync(request, timeoutSeconds, new RpcResponseWaiter(rpcNode, responseReceiver));
+    }
+
+    private static class RpcNodeConnection implements NodeConnection {
+
+        // Information about the connected node
+        private final Supervisor supervisor;
+        private final String hostname;
+        private final int port;
+        private final String description;
+
+        // The current shared connection. This will be recycled when it becomes invalid.
+        // All access to this must be synchronized
+        private Target target = null;
+
+        public RpcNodeConnection(String hostname, int port, Supervisor supervisor) {
+            this.supervisor = supervisor;
+            this.hostname = hostname;
+            this.port = port;
+            description = "rpc node connection to " + hostname + ":" + port;
+        }
+
+        public void invokeAsync(Request req, double timeout, RequestWaiter waiter) {
+            // TODO: Consider replacing this by a watcher on the target
+            synchronized(this) { // ensure we have exactly 1 valid connection across threads
+                if (target == null || ! target.isValid())
+                    target = supervisor.connect(new Spec(hostname, port));
+            }
+            target.invokeAsync(req, timeout, waiter);
+        }
+
+        @Override
+        public void close() {
+            target.close();
+        }
+
+        @Override
+        public String toString() {
+            return description;
+        }
+
+    }
+
+    private static class RpcResponseWaiter implements RequestWaiter {
+
+        /** The node to which we made the request we are waiting for - for error messages only */
+        private final RpcNodeConnection node;
+
+        /** The handler to which the response is forwarded */
+        private final Dispatcher.GetDocsumsResponseReceiver handler;
+
+        public RpcResponseWaiter(RpcNodeConnection node, Dispatcher.GetDocsumsResponseReceiver handler) {
+            this.node = node;
+            this.handler = handler;
+        }
+
+        @Override
+        public void handleRequestDone(Request requestWithResponse) {
+            if (requestWithResponse.isError()) {
+                handler.receive(GetDocsumsResponseOrError.fromError("Error response from " + node + ": " +
+                                                                    requestWithResponse.errorMessage()));
+                return;
+            }
+
+            Values returnValues = requestWithResponse.returnValues();
+            if (returnValues.size() < 3) {
+                handler.receive(GetDocsumsResponseOrError.fromError("Invalid getDocsums response from " + node +
+                                                                    ": Expected 3 return arguments, got " +
+                                                                    returnValues.size()));
+                return;
+            }
+
+            byte compression = returnValues.get(0).asInt8();
+            int uncompressedSize = returnValues.get(1).asInt32();
+            byte[] compressedSlimeBytes = returnValues.get(2).asData();
+            List<FastHit> hits = (List<FastHit>) requestWithResponse.getContext();
+            handler.receive(GetDocsumsResponseOrError.fromResponse(new GetDocsumsResponse(compression,
+                                                                                          uncompressedSize,
+                                                                                          compressedSlimeBytes,
+                                                                                          hits)));
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/CommonFields.java b/container-search/src/main/java/com/yahoo/search/federation/CommonFields.java
new file mode 100644
index 00000000000..912a1db6202
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/CommonFields.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation;
+/**
+ * A set of string constants for common hit field names.
+ * @author laboisse
+ *
+ */
+public class CommonFields {
+
+	public static final String TITLE = "title";
+	public static final String URL = "url";
+	public static final String DESCRIPTION = "description";
+	public static final String DATE = "date";
+	public static final String SIZE = "size";
+	public static final String DISP_URL = "dispurl";
+	public static final String BASE_URL = "baseurl";
+	public static final String MIME_TYPE = "mimetype";
+	public static final String RELEVANCY = "relevancy";
+	public static final String THUMBNAIL_URL = "thumbnailUrl";
+	public static final String THUMBNAIL_WIDTH = "thumbnailWidth";
+	public static final String THUMBNAIL_HEIGHT = "thumbnailHeight";
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
new file mode 100644
index 00000000000..4ec04d0d577
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
@@ -0,0 +1,948 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation;
+
+import com.google.inject.Inject;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Chain;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.concurrent.CopyOnWriteHashMap;
+import com.yahoo.errorhandling.Results.Builder;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.federation.selection.FederationTarget;
+import com.yahoo.search.federation.selection.TargetSelector;
+import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec;
+import com.yahoo.search.federation.sourceref.SearchChainResolver;
+import com.yahoo.search.federation.sourceref.SingleTarget;
+import com.yahoo.search.federation.sourceref.SourceRefResolver;
+import com.yahoo.search.federation.sourceref.SourcesTarget;
+import com.yahoo.search.federation.sourceref.Target;
+import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.query.properties.QueryProperties;
+import com.yahoo.search.query.properties.SubProperties;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.search.result.HitOrderer;
+import com.yahoo.search.searchchain.AsyncExecution;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.ForkingSearcher;
+import com.yahoo.search.searchchain.FutureResult;
+import com.yahoo.search.searchchain.SearchChainRegistry;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+import com.yahoo.errorhandling.Results;
+
+import org.apache.commons.lang.StringUtils;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.yahoo.collections.CollectionUtil.first;
+import static com.yahoo.container.util.Util.quote;
+import static com.yahoo.search.federation.StrictContractsConfig.PropagateSourceProperties;
+
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * This searcher takes a set of sources, looks them up in config and fire off the correct searchchains.
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author tonytv
+ */
+@Provides(FederationSearcher.FEDERATION)
+@After("*")
+public class FederationSearcher extends ForkingSearcher {
+    public static final String FEDERATION = "Federation";
+
+    private static abstract class TargetHandler {
+        abstract Chain<Searcher> getChain();
+        abstract void modifyTargetQuery(Query query);
+        abstract void modifyTargetResult(Result result);
+
+        ComponentId getId() {
+            return getChain().getId();
+        }
+
+        public abstract FederationOptions federationOptions();
+
+        @Override
+        public String toString() {
+            return getChain().getId().stringValue();
+        }
+
+    }
+
+    private static class StandardTargetHandler extends TargetHandler {
+        private final SearchChainInvocationSpec target;
+        private final Chain<Searcher> chain;
+
+        public StandardTargetHandler(SearchChainInvocationSpec target, Chain<Searcher> chain) {
+            this.target = target;
+            this.chain = chain;
+        }
+
+        @Override
+        Chain<Searcher> getChain() {
+            return chain;
+        }
+
+        @Override
+        void modifyTargetQuery(Query query) {}
+        @Override
+        void modifyTargetResult(Result result) {}
+
+        @Override
+        public FederationOptions federationOptions() {
+            return target.federationOptions;
+        }
+    }
+
+
+    private static class CustomTargetHandler<T> extends TargetHandler {
+        private final TargetSelector<T> selector;
+        private final FederationTarget<T> target;
+
+        CustomTargetHandler(TargetSelector<T> selector, FederationTarget<T> target) {
+            this.selector = selector;
+            this.target = target;
+        }
+
+        @Override
+        Chain<Searcher> getChain() {
+            return target.getChain();
+        }
+
+        @Override
+        public void modifyTargetQuery(Query query) {
+            selector.modifyTargetQuery(target, query);
+        }
+
+        @Override
+        public void modifyTargetResult(Result result) {
+            selector.modifyTargetResult(target, result);
+        }
+
+        @Override
+        public FederationOptions federationOptions() {
+            return target.getFederationOptions();
+        }
+    }
+
+
+
+    private static class ExecutionInfo {
+        final TargetHandler targetHandler;
+        final FederationOptions federationOptions;
+        final FutureResult futureResult;
+
+        public ExecutionInfo(TargetHandler targetHandler, FederationOptions federationOptions, FutureResult futureResult) {
+            this.targetHandler = targetHandler;
+            this.federationOptions = federationOptions;
+            this.futureResult = futureResult;
+        }
+    }
+
+    private static class CompoundKey {
+        private final String sourceName;
+        private final String propertyName;
+        CompoundKey(String sourceName, String propertyName) {
+            this.sourceName = sourceName;
+            this.propertyName = propertyName;
+        }
+
+        @Override
+        public int hashCode() {
+            return sourceName.hashCode() ^ propertyName.hashCode();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            CompoundKey rhs = (CompoundKey) o;
+            return sourceName.equals(rhs.sourceName) && propertyName.equals(rhs.propertyName);
+        }
+
+        @Override
+        public String toString() {
+            return sourceName + '.' + propertyName;
+        }
+    }
+
+    private static class SourceKey extends CompoundKey {
+        public static final String SOURCE = "source.";
+        SourceKey(String sourceName, String propertyName) {
+            super(sourceName, propertyName);
+        }
+
+        @Override
+        public int hashCode() {
+            return super.hashCode() ^ 7;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            return (o instanceof SourceKey) && super.equals(o);
+        }
+
+        @Override
+        public String toString() {
+            return SOURCE + super.toString();
+        }
+    }
+    private static class ProviderKey extends CompoundKey {
+        public static final String PROVIDER = "provider.";
+        ProviderKey(String sourceName, String propertyName) {
+            super(sourceName, propertyName);
+        }
+
+        @Override
+        public int hashCode() {
+            return super.hashCode() ^ 17;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            return (o instanceof ProviderKey) && super.equals(o);
+        }
+
+        @Override
+        public String toString() {
+            return PROVIDER + super.toString();
+        }
+    }
+
+    private static final Logger log = Logger.getLogger(FederationSearcher.class.getName());
+
+    /** The name of the query property containing the source name added to the query to each source by this */
+    public final static CompoundName SOURCENAME = new CompoundName("sourceName");
+    public final static CompoundName PROVIDERNAME = new CompoundName("providerName");
+
+
+    /** Logging field name constants */
+    public static final String LOG_COUNT_PREFIX = "count_";
+
+    private final SearchChainResolver searchChainResolver;
+    private final PropagateSourceProperties.Enum propagateSourceProperties;
+    private final SourceRefResolver sourceRefResolver;
+    private final CopyOnWriteHashMap<CompoundKey, CompoundName> map = new CopyOnWriteHashMap<>();
+
+    private final boolean strictSearchchain;
+    private final TargetSelector<?> targetSelector;
+
+
+    @Inject
+    public FederationSearcher(FederationConfig config, StrictContractsConfig strict,
+                              ComponentRegistry<TargetSelector> targetSelectors) {
+        this(createResolver(config), strict.searchchains(), strict.propagateSourceProperties(),
+                resolveSelector(config.targetSelector(), targetSelectors));
+    }
+
+    private static TargetSelector resolveSelector(String selectorId, ComponentRegistry<TargetSelector> targetSelectors) {
+        if (selectorId.isEmpty())
+            return null;
+
+        return checkNotNull(
+                targetSelectors.getComponent(selectorId),
+                "Missing target selector with id" + quote(selectorId));
+    }
+
+    //for testing
+    public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) {
+        this(searchChainResolver, false, PropagateSourceProperties.ALL, null);
+    }
+
+    private FederationSearcher(SearchChainResolver searchChainResolver, boolean strictSearchchain,
+                               PropagateSourceProperties.Enum propagateSourceProperties,
+                               TargetSelector targetSelector) {
+        this.searchChainResolver = searchChainResolver;
+        sourceRefResolver = new SourceRefResolver(searchChainResolver);
+        this.strictSearchchain = strictSearchchain;
+        this.propagateSourceProperties = propagateSourceProperties;
+        this.targetSelector = targetSelector;
+    }
+
+
+    private static SearchChainResolver createResolver(FederationConfig config) {
+        SearchChainResolver.Builder builder = new SearchChainResolver.Builder();
+
+        for (FederationConfig.Target target : config.target()) {
+            boolean isDefaultProviderForSource = true;
+
+            for (FederationConfig.Target.SearchChain searchChain : target.searchChain()) {
+                if (searchChain.providerId() == null || searchChain.providerId().isEmpty()) {
+                    addSearchChain(builder, target, searchChain);
+                } else {
+                    addSourceForProvider(builder, target, searchChain, isDefaultProviderForSource);
+                    isDefaultProviderForSource = false;
+                }
+            }
+
+            //Allow source groups to use by default.
+            if (target.useByDefault())
+                builder.useTargetByDefault(target.id());
+        }
+
+        return builder.build();
+    }
+
+    private static void addSearchChain(SearchChainResolver.Builder builder,
+                                       FederationConfig.Target target, FederationConfig.Target.SearchChain searchChain) {
+        if (!target.id().equals(searchChain.searchChainId()))
+            throw new RuntimeException("Invalid federation config, " + target.id() + " != " + searchChain.searchChainId());
+
+        builder.addSearchChain(ComponentId.fromString(searchChain.searchChainId()),
+                federationOptions(searchChain), searchChain.documentTypes());
+    }
+
+    private static void addSourceForProvider(SearchChainResolver.Builder builder, FederationConfig.Target target,
+                                             FederationConfig.Target.SearchChain searchChain, boolean isDefaultProvider) {
+        builder.addSourceForProvider(
+                ComponentId.fromString(target.id()),
+                ComponentId.fromString(searchChain.providerId()),
+                ComponentId.fromString(searchChain.searchChainId()),
+                isDefaultProvider, federationOptions(searchChain),
+                searchChain.documentTypes());
+    }
+
+    private static FederationOptions federationOptions(FederationConfig.Target.SearchChain searchChain) {
+        return new FederationOptions().
+                setOptional(searchChain.optional()).
+                setUseByDefault(searchChain.useByDefault()).
+                setTimeoutInMilliseconds(searchChain.timeoutMillis()).
+                setRequestTimeoutInMilliseconds(searchChain.requestTimeoutMillis());
+    }
+
+    private static long calculateTimeout(Query query, List<TargetHandler> targets) {
+
+        class PartitionByOptional {
+            final List<TargetHandler> mandatoryTargets;
+            final List<TargetHandler> optionalTargets;
+
+            PartitionByOptional(List<TargetHandler> targets) {
+                List<TargetHandler> mandatoryTargets = new ArrayList<>();
+                List<TargetHandler> optionalTargets = new ArrayList<>();
+
+                for (TargetHandler target : targets) {
+                    if (target.federationOptions().getOptional()) {
+                        optionalTargets.add(target);
+                    } else {
+                        mandatoryTargets.add(target);
+                    }
+                }
+
+                this.mandatoryTargets = Collections.unmodifiableList(mandatoryTargets);
+                this.optionalTargets = Collections.unmodifiableList(optionalTargets);
+            }
+        }
+
+        if (query.requestHasProperty("timeout") || targets.isEmpty()) {
+            return query.getTimeLeft();
+        } else {
+            PartitionByOptional partition = new PartitionByOptional(targets);
+            long queryTimeout = query.getTimeout();
+
+            return partition.mandatoryTargets.isEmpty() ?
+                    maximumTimeout(partition.optionalTargets, queryTimeout) :
+                    maximumTimeout(partition.mandatoryTargets, queryTimeout);
+        }
+    }
+
+    private static long maximumTimeout(List<TargetHandler> invocationSpecs, long queryTimeout) {
+        long timeout = 0;
+        for (TargetHandler target : invocationSpecs) {
+            timeout = Math.max(timeout,
+                               target.federationOptions().getSearchChainExecutionTimeoutInMilliseconds(queryTimeout));
+        }
+        return timeout;
+    }
+
+    private void addSearchChainTimedOutError(Query query,
+                                             ComponentId searchChainId) {
+        ErrorMessage timeoutMessage=
+                ErrorMessage.createTimeout("The search chain '" + searchChainId + "' timed out.");
+        timeoutMessage.setSource(searchChainId.stringValue());
+        query.errors().add(timeoutMessage);
+    }
+
+    private void mergeResult(Query query, TargetHandler targetHandler,
+                             Result mergedResults, Result result) {
+
+
+        targetHandler.modifyTargetResult(result);
+        final ComponentId searchChainId = targetHandler.getId();
+        Chain<Searcher> searchChain = targetHandler.getChain();
+
+        mergedResults.mergeWith(result);
+        HitGroup group = result.hits();
+        group.setId("source:" + searchChainId.getName());
+
+        group.setSearcherSpecificMetaData(this, searchChain);
+        group.setMeta(false); // Set hit groups as non-meta as a default
+        group.setAuxiliary(true); // Set hit group as auxiliary so that it doesn't contribute to count
+        group.setSource(searchChainId.getName());
+        group.setQuery(result.getQuery());
+
+        for (Iterator<Hit> it = group.unorderedDeepIterator(); it.hasNext();) {
+            Hit hit = it.next();
+            hit.setSearcherSpecificMetaData(this, searchChain);
+            hit.setSource(searchChainId.stringValue());
+
+            // This is the backend request meta hit, that is holding logging information
+            // See HTTPBackendSearcher, where this hit is created
+            if (hit.isMeta() && hit.types().contains("logging")) {
+                // Augment this hit with count fields
+                hit.setField(LOG_COUNT_PREFIX + "deep", result.getDeepHitCount());
+                hit.setField(LOG_COUNT_PREFIX + "total", result.getTotalHitCount());
+                int offset = result.getQuery().getOffset();
+                hit.setField(LOG_COUNT_PREFIX + "first", offset + 1);
+                hit.setField(LOG_COUNT_PREFIX + "last", result.getConcreteHitCount() + offset);
+            }
+
+        }
+        if (query.getTraceLevel()>=4)
+            query.trace("Got " + group.getConcreteSize() + " hits from " + group.getId(),false, 4);
+        mergedResults.hits().add(group);
+    }
+
+    private boolean successfullyCompleted(FutureResult result) {
+        return result.isDone() && !result.isCancelled();
+    }
+
+    private Query setupSingleQuery(Query query, long timeout, TargetHandler targetHandler) {
+        if (strictSearchchain) {
+            query.resetTimeout();
+            return setupFederationQuery(query, query,
+                    windowParameters(query.getHits(), query.getOffset()), timeout, targetHandler);
+        } else {
+            return cloneFederationQuery(query,
+                    windowParameters(query.getHits(), query.getOffset()), timeout, targetHandler);
+        }
+    }
+
+    private Result startExecuteSingleQuery(Query query, TargetHandler chain, long timeout, Execution execution) {
+        Query outgoing = setupSingleQuery(query, timeout, chain);
+        Execution exec = new Execution(chain.getChain(), execution.context());
+        return exec.search(outgoing);
+    }
+
+    private List<ExecutionInfo> startExecuteQueryForEachTarget(
+            Query query, Collection<TargetHandler> targets, long timeout, Execution execution) {
+
+        List<ExecutionInfo> results = new ArrayList<>();
+
+        Map<String, Object> windowParameters;
+        if (targets.size()==1) // preserve requested top-level offset by default as an optimization
+            windowParameters = Collections.unmodifiableMap(windowParameters(query.getHits(), query.getOffset()));
+        else // request from offset 0 to enable correct upstream blending into a single top-level hit list
+            windowParameters = Collections.unmodifiableMap(windowParameters(query.getHits() + query.getOffset(), 0));
+
+        for (TargetHandler targetHandler : targets) {
+            long executeTimeout = timeout;
+            if (targetHandler.federationOptions().getRequestTimeoutInMilliseconds() != -1)
+                executeTimeout = targetHandler.federationOptions().getRequestTimeoutInMilliseconds();
+            results.add(new ExecutionInfo(targetHandler, targetHandler.federationOptions(),
+                                          createFutureSearch(query, windowParameters, targetHandler, executeTimeout, execution)));
+        }
+
+        return results;
+    }
+
+    private Map<String, Object> windowParameters(int hits, int offset) {
+        Map<String, Object> params = new HashMap<>();
+        params.put(Query.HITS.toString(), hits);
+        params.put(Query.OFFSET.toString(), offset);
+        return params;
+    }
+
+    private FutureResult createFutureSearch(Query query, Map<String, Object> windowParameters, TargetHandler targetHandler,
+                                            long timeout, Execution execution) {
+        Query clonedQuery = cloneFederationQuery(query, windowParameters, timeout, targetHandler);
+        return new AsyncExecution(targetHandler.getChain(), execution).search(clonedQuery);
+    }
+
+
+    private Query cloneFederationQuery(Query query,
+                                       Map<String, Object> windowParameters, long timeout, TargetHandler targetHandler) {
+        Query clonedQuery = Query.createNewQuery(query);
+        return setupFederationQuery(query, clonedQuery, windowParameters, timeout, targetHandler);
+    }
+
+    private Query setupFederationQuery(Query query, Query outgoing,
+                                       Map<String, Object> windowParameters, long timeout, TargetHandler targetHandler) {
+
+        ComponentId chainId = targetHandler.getChain().getId();
+
+        String sourceName = chainId.getName();
+        outgoing.properties().set(SOURCENAME, sourceName);
+        String providerName = chainId.getName();
+        if (chainId.getNamespace() != null)
+            providerName = chainId.getNamespace().getName();
+        outgoing.properties().set(PROVIDERNAME, providerName);
+
+        outgoing.setTimeout(timeout);
+
+        switch (propagateSourceProperties) {
+            case ALL:
+                propagatePerSourceQueryProperties(query, outgoing, windowParameters, sourceName, providerName,
+                                                  QueryProperties.PER_SOURCE_QUERY_PROPERTIES);
+                break;
+            case OFFSET_HITS:
+                propagatePerSourceQueryProperties(query, outgoing, windowParameters, sourceName, providerName,
+                                                  new CompoundName[]{Query.OFFSET, Query.HITS});
+                break;
+        }
+
+        //TODO: FederationTarget
+        //TODO: only for target produced by this, not others
+        targetHandler.modifyTargetQuery(outgoing);
+        return outgoing;
+    }
+
+    private void propagatePerSourceQueryProperties(Query original, Query outgoing,
+                                                   Map<String, Object> windowParameters,
+                                                   String sourceName, String providerName,
+                                                   CompoundName[] queryProperties) {
+
+        for (CompoundName key : queryProperties) {
+            Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, windowParameters.get(key.toString()));
+            if (value != null) {
+                outgoing.properties().set(key, value);
+            }
+        }
+    }
+
+    private Object getSourceOrProviderProperty(Query query, CompoundName propertyName,
+                                                String sourceName, String providerName,
+                                                Object defaultValue) {
+        Object result = getProperty(query, new SourceKey(sourceName, propertyName.toString()));
+        if (result == null)
+            result = getProperty(query, new ProviderKey(providerName, propertyName.toString()));
+        if (result == null)
+            result = defaultValue;
+
+        return result;
+    }
+
+    private Object getProperty(Query query, CompoundKey key) {
+
+        CompoundName name = map.get(key);
+        if (name == null) {
+            name = new CompoundName(key.toString());
+            map.put(key, name);
+        }
+        return query.properties().get(name);
+    }
+
+    private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(StringUtils.join(getMessagesSet(unresolvedSearchChainExceptions), ' '));
+
+
+        sb.append(" Valid source refs are ");
+        sb.append(
+                StringUtils.join(allSourceRefDescriptions().iterator(),
+                        ", ")).append('.');
+
+        return ErrorMessage.createInvalidQueryParameter(sb.toString());
+    }
+
+    private List<String> allSourceRefDescriptions() {
+        List<String> descriptions = new ArrayList<>();
+
+        for (Target target : searchChainResolver.allTopLevelTargets()) {
+            descriptions.add(target.searchRefDescription());
+        }
+        return descriptions;
+    }
+
+    private Set<String> getMessagesSet(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) {
+        Set<String> messages = new LinkedHashSet<>();
+        for (UnresolvedSearchChainException exception : unresolvedSearchChainExceptions) {
+            messages.add(exception.getMessage());
+        }
+        return messages;
+    }
+
+    private void warnIfUnresolvedSearchChains(List<UnresolvedSearchChainException> missingTargets,
+                                      HitGroup errorHitGroup) {
+
+        if (!missingTargets.isEmpty()) {
+            errorHitGroup.addError(missingSearchChainsErrorMessage(missingTargets));
+        }
+    }
+
+    @Override
+    public Collection<CommentedSearchChain> getSearchChainsForwarded(SearchChainRegistry registry) {
+        List<CommentedSearchChain> searchChains = new ArrayList<>();
+
+        for (Target target : searchChainResolver.allTopLevelTargets()) {
+            if (target instanceof SourcesTarget) {
+                searchChains.addAll(commentedSourceProviderSearchChains((SourcesTarget)target, registry));
+            } else if (target instanceof SingleTarget) {
+                searchChains.add(commentedSearchChain((SingleTarget)target, registry));
+            } else {
+                log.warning("Invalid target type " + target.getClass().getName());
+            }
+        }
+
+        return searchChains;
+    }
+
+    private CommentedSearchChain commentedSearchChain(SingleTarget singleTarget, SearchChainRegistry registry) {
+        return new CommentedSearchChain("If source refs contains '" + singleTarget.getId() + "'.",
+                registry.getChain(singleTarget.getId()));
+    }
+
+    private List<CommentedSearchChain> commentedSourceProviderSearchChains(SourcesTarget sourcesTarget,
+                                                                           SearchChainRegistry registry) {
+
+        List<CommentedSearchChain> commentedSearchChains = new ArrayList<>();
+        String ifMatchingSourceRefPrefix = "If source refs contains '" + sourcesTarget.getId() + "' and provider is '";
+
+        commentedSearchChains.add(
+                new CommentedSearchChain(ifMatchingSourceRefPrefix + sourcesTarget.defaultProviderSource().provider +
+                        "'(or not given).", registry.getChain(sourcesTarget.defaultProviderSource().searchChainId)));
+
+        for (SearchChainInvocationSpec providerSource : sourcesTarget.allProviderSources()) {
+            if (!providerSource.equals(sourcesTarget.defaultProviderSource())) {
+                commentedSearchChains.add(
+                        new CommentedSearchChain(ifMatchingSourceRefPrefix + providerSource.provider + "'.",
+                                registry.getChain(providerSource.searchChainId)));
+            }
+        }
+        return commentedSearchChains;
+    }
+
+    /** Returns the set of properties set for the source or provider given in the query (if any).
+     *
+     * If the query has not set sourceName or providerName, null will be returned */
+    public static Properties getSourceProperties(Query query) {
+        String sourceName = query.properties().getString(SOURCENAME);
+        String providerName = query.properties().getString(PROVIDERNAME);
+        if (sourceName == null || providerName == null)
+            return null;
+        Properties sourceProperties = new SubProperties("source." + sourceName, query.properties());
+        Properties providerProperties = new SubProperties("provider." + providerName, query.properties());
+        sourceProperties.chain(providerProperties);
+        return sourceProperties;
+    }
+
+    @Override
+    public void fill(final Result result, final String summaryClass, Execution execution) {
+        List<FutureResult> filledResults = new ArrayList<>();
+        UniqueExecutionsToResults uniqueExecutionsToResults = new UniqueExecutionsToResults();
+        addResultsToFill(result.hits(), result, summaryClass, uniqueExecutionsToResults);
+        final Set<Entry<Chain<Searcher>, Map<Query, Result>>> resultsForAllChains = uniqueExecutionsToResults.resultsToFill
+                .entrySet();
+        int numberOfCallsToFillNeeded = 0;
+
+        for (Entry<Chain<Searcher>, Map<Query, Result>> resultsToFillForAChain : resultsForAllChains) {
+            numberOfCallsToFillNeeded += resultsToFillForAChain.getValue().size();
+        }
+
+        for (Entry<Chain<Searcher>, Map<Query, Result>> resultsToFillForAChain : resultsForAllChains) {
+            Chain<Searcher> chain = resultsToFillForAChain.getKey();
+            Execution chainExecution = (chain == null) ? execution : new Execution(chain, execution.context());
+
+            for (Entry<Query, Result> resultsToFillForAChainAndQuery : resultsToFillForAChain.getValue().entrySet()) {
+                Result resultToFill = resultsToFillForAChainAndQuery.getValue();
+                if (numberOfCallsToFillNeeded == 1) {
+                    chainExecution.fill(resultToFill, summaryClass);
+                    propagateErrors(resultToFill, result);
+                } else {
+                    AsyncExecution asyncFill = new AsyncExecution(chainExecution);
+                    filledResults.add(asyncFill.fill(resultToFill, summaryClass));
+                }
+            }
+        }
+        for (FutureResult filledResult : filledResults) {
+            propagateErrors(filledResult.get(result.getQuery().getTimeLeft(), TimeUnit.MILLISECONDS), result);
+        }
+    }
+
+    private void propagateErrors(Result source, Result destination) {
+        ErrorMessage error = source.hits().getError();
+        if (error != null)
+            destination.hits().addError(error);
+    }
+
+    /** A map from a unique search chain and query instance to a result */
+    private static class UniqueExecutionsToResults {
+
+        /** Implemented as a nested identity hashmap */
+        final Map<Chain<Searcher>,Map<Query,Result>> resultsToFill = new IdentityHashMap<>();
+
+        /** Returns a result to fill for a query and chain, by creating it if necessary */
+        public Result get(Chain<Searcher> chain, Query query) {
+            Map<Query,Result> resultsToFillForAChain = resultsToFill.get(chain);
+            if (resultsToFillForAChain == null) {
+                resultsToFillForAChain = new IdentityHashMap<>();
+                resultsToFill.put(chain,resultsToFillForAChain);
+            }
+
+            Result resultsToFillForAChainAndQuery = resultsToFillForAChain.get(query);
+            if (resultsToFillForAChainAndQuery == null) {
+                resultsToFillForAChainAndQuery = new Result(query);
+                resultsToFillForAChain.put(query,resultsToFillForAChainAndQuery);
+            }
+
+            return resultsToFillForAChainAndQuery;
+        }
+
+    }
+
+    private void addResultsToFill(HitGroup hitGroup, Result result, String summaryClass,
+                                  UniqueExecutionsToResults uniqueExecutionsToResults) {
+        for (Hit hit : hitGroup) {
+            if (hit instanceof HitGroup) {
+                addResultsToFill((HitGroup) hit, result, summaryClass, uniqueExecutionsToResults);
+            } else {
+                if ( ! hit.isFilled(summaryClass))
+                    getSearchChainGroup(hit,result,uniqueExecutionsToResults).hits().add(hit);
+            }
+        }
+    }
+
+    private Result getSearchChainGroup(Hit hit, Result result, UniqueExecutionsToResults uniqueExecutionsToResults) {
+        @SuppressWarnings("unchecked")
+        Chain<Searcher> chain = (Chain<Searcher>) hit.getSearcherSpecificMetaData(this);
+        Query query = hit.getQuery() !=null ? hit.getQuery() : result.getQuery();
+
+        return uniqueExecutionsToResults.get(chain,query);
+    }
+
+    private void searchMultipleTargets(Query query, Result mergedResults,
+                                       Collection<TargetHandler> targets,
+                                       long timeout,
+                                       Execution execution) {
+
+        List<ExecutionInfo> executionInfos = startExecuteQueryForEachTarget(query, targets, timeout, execution);
+        waitForMandatoryTargets(executionInfos, query.getTimeout());
+
+        HitOrderer s=null;
+        for (ExecutionInfo executionInfo : executionInfos) {
+            if ( ! successfullyCompleted(executionInfo.futureResult)) {
+                addSearchChainTimedOutError(query, executionInfo.targetHandler.getId());
+            } else {
+                if (s == null) {
+                    s = dirtyCopyIfModifiedOrderer(mergedResults.hits(), executionInfo.futureResult.get().hits().getOrderer());
+                }
+                mergeResult(query, executionInfo.targetHandler, mergedResults, executionInfo.futureResult.get());
+
+            }
+        }
+    }
+
+    /**
+     * TODO This is probably a dirty hack for bug 4711376. There are probably better ways.
+     * But I will leave that to trd-processing@
+     *
+     * @param group  The merging hitgroup to be updated if necessary
+     * @param orderer The per provider hit orderer.
+     * @return The hitorderer chosen
+     */
+    private HitOrderer dirtyCopyIfModifiedOrderer(HitGroup group, HitOrderer orderer) {
+        if (orderer != null) {
+            HitOrderer old = group.getOrderer();
+            if ((old == null) || ! orderer.equals(old)) {
+                group.setOrderer(orderer);
+            }
+        }
+
+        return orderer;
+    }
+
+    private void waitForMandatoryTargets(List<ExecutionInfo> executionInfos, long queryTimeout) {
+        FutureWaiter futureWaiter = new FutureWaiter();
+
+        boolean hasMandatoryTargets = false;
+        for (ExecutionInfo executionInfo : executionInfos) {
+            if (isMandatory(executionInfo)) {
+                futureWaiter.add(executionInfo.futureResult,
+                                 getSearchChainExecutionTimeoutInMilliseconds(executionInfo, queryTimeout));
+                hasMandatoryTargets = true;
+            }
+        }
+
+        if (!hasMandatoryTargets) {
+            for (ExecutionInfo executionInfo : executionInfos) {
+                futureWaiter.add(executionInfo.futureResult,
+                                 getSearchChainExecutionTimeoutInMilliseconds(executionInfo, queryTimeout));
+            }
+        }
+
+        futureWaiter.waitForFutures();
+    }
+
+    private long getSearchChainExecutionTimeoutInMilliseconds(ExecutionInfo executionInfo, long queryTimeout) {
+        return executionInfo.federationOptions.
+                getSearchChainExecutionTimeoutInMilliseconds(queryTimeout);
+    }
+
+    private boolean isMandatory(ExecutionInfo executionInfo) {
+        return !executionInfo.federationOptions.getOptional();
+    }
+
+    private void searchSingleTarget(Query query, Result mergedResults,
+                                    TargetHandler targetHandler,
+                                    long timeout,
+                                    Execution execution) {
+        Result result = startExecuteSingleQuery(query, targetHandler, timeout, execution);
+        mergeResult(query, targetHandler, mergedResults, result);
+    }
+
+
+    private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> getTargets(Set<String> sources, Properties properties, IndexFacts indexFacts) {
+        return sources.isEmpty() ?
+                defaultSearchChains(properties):
+                resolveSources(sources, properties, indexFacts);
+    }
+
+    private Results<SearchChainInvocationSpec, UnresolvedSearchChainException> resolveSources(Set<String> sources, Properties properties, IndexFacts indexFacts) {
+        Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>();
+
+        for (String source : sources) {
+            try {
+                result.addAllData(sourceRefResolver.resolve(asSourceSpec(source), properties, indexFacts));
+            } catch (UnresolvedSearchChainException e) {
+                result.addError(e);
+            }
+        }
+
+        return result.build();
+    }
+
+
+    public Results<SearchChainInvocationSpec, UnresolvedSearchChainException> defaultSearchChains(Properties sourceToProviderMap) {
+        Results.Builder<SearchChainInvocationSpec, UnresolvedSearchChainException> result = new Builder<>();
+
+        for (Target target : searchChainResolver.defaultTargets()) {
+            try {
+                result.addData(target.responsibleSearchChain(sourceToProviderMap));
+            } catch (UnresolvedSearchChainException e) {
+                result.addError(e);
+            }
+        }
+
+        return result.build();
+    }
+
+
+    private ComponentSpecification asSourceSpec(String source) {
+        try {
+            return new ComponentSpecification(source);
+        } catch(Exception e) {
+            throw new IllegalArgumentException("The source ref '" + source
+                    + "' used for federation is not valid.", e);
+        }
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        Result mergedResults = execution.search(query);
+
+        Results<SearchChainInvocationSpec, UnresolvedSearchChainException> targets =
+                getTargets(query.getModel().getSources(), query.properties(), execution.context().getIndexFacts());
+        warnIfUnresolvedSearchChains(targets.errors(), mergedResults.hits());
+
+        Collection<SearchChainInvocationSpec> prunedTargets =
+                pruneTargetsWithoutDocumentTypes(query.getModel().getRestrict(), targets.data());
+
+        Results<TargetHandler, ErrorMessage> regularTargetHandlers = resolveSearchChains(prunedTargets, execution.searchChainRegistry());
+        query.errors().addAll(regularTargetHandlers.errors());
+
+        List<TargetHandler> targetHandlers = new ArrayList<>(regularTargetHandlers.data());
+        targetHandlers.addAll(getAdditionalTargets(query, execution, targetSelector));
+
+        final long targetsTimeout = calculateTimeout(query, targetHandlers);
+        if (targetsTimeout < 0)
+            return new Result(query, ErrorMessage.createTimeout("Timed out when about to federate"));
+
+        traceTargets(query, targetHandlers);
+
+        if (targetHandlers.size() == 0) {
+            return mergedResults;
+        } else if (targetHandlers.size() == 1 &&  ! shouldExecuteTargetLongerThanThread(query, targetHandlers.get(0))) {
+            TargetHandler chain = first(targetHandlers);
+            searchSingleTarget(query, mergedResults, chain, targetsTimeout, execution);
+        } else {
+            searchMultipleTargets(query, mergedResults, targetHandlers, targetsTimeout, execution);
+        }
+
+        return mergedResults;
+    }
+
+    private void traceTargets(Query query, List<TargetHandler> targetHandlers) {
+        final int traceFederationLevel = 2;
+        if ( ! query.isTraceable(traceFederationLevel)) return;
+        query.trace("Federating to " + targetHandlers, traceFederationLevel);
+    }
+
+    /**
+     * Returns true if we are requested to keep executing a target longer than we're waiting for it.
+     * This is useful to populate caches inside targets.
+     */
+    private boolean shouldExecuteTargetLongerThanThread(Query query, TargetHandler target) {
+        return target.federationOptions().getRequestTimeoutInMilliseconds() > query.getTimeout();
+    }
+
+    private static Results<TargetHandler, ErrorMessage> resolveSearchChains(
+            Collection<SearchChainInvocationSpec> prunedTargets,
+            SearchChainRegistry registry) {
+
+        Results.Builder<TargetHandler, ErrorMessage> targetHandlers = new Results.Builder<>();
+
+        for (SearchChainInvocationSpec target: prunedTargets) {
+            Chain<Searcher> chain = registry.getChain(target.searchChainId);
+            if (chain == null) {
+                targetHandlers.addError(ErrorMessage.createIllegalQuery(
+                        "Could not find search chain '" + target.searchChainId + "'"));
+            } else {
+                targetHandlers.addData(new StandardTargetHandler(target, chain));
+            }
+        }
+
+        return targetHandlers.build();
+    }
+
+    private static <T> List<TargetHandler> getAdditionalTargets(Query query, Execution execution, TargetSelector<T> targetSelector) {
+        if (targetSelector == null)
+            return Collections.emptyList();
+
+        ArrayList<TargetHandler> result = new ArrayList<>();
+        for (FederationTarget<T> target: targetSelector.getTargets(query, execution.searchChainRegistry()))
+            result.add(new CustomTargetHandler<>(targetSelector, target));
+
+        return result;
+    }
+
+    private Collection<SearchChainInvocationSpec> pruneTargetsWithoutDocumentTypes(Set<String> restrict, List<SearchChainInvocationSpec> targets) {
+        if (restrict.isEmpty())
+            return targets;
+
+        Collection<SearchChainInvocationSpec> prunedTargets = new ArrayList<>();
+
+        for (SearchChainInvocationSpec target : targets) {
+            if (target.documentTypes.isEmpty() || documentTypeIntersectionIsNonEmpty(restrict, target))
+                prunedTargets.add(target);
+        }
+
+        return prunedTargets;
+    }
+
+    private boolean documentTypeIntersectionIsNonEmpty(Set<String> restrict, SearchChainInvocationSpec target) {
+        for (String documentType : target.documentTypes) {
+            if (restrict.contains(documentType))
+                return true;
+        }
+
+        return false;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java
new file mode 100644
index 00000000000..b43798113de
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java
@@ -0,0 +1,106 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Chain;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.prelude.Ping;
+import com.yahoo.prelude.Pong;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.cluster.PingableSearcher;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * A lightweight searcher to forward all incoming requests to a single search
+ * chain defined in config. An alternative to federation searcher when standard
+ * semantics are not necessary for the application.
+ *
+ * @see FederationSearcher
+ * @since 5.0.13
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@After("*")
+public class ForwardingSearcher extends PingableSearcher {
+    private final ComponentSpecification target;
+
+    public ForwardingSearcher(final SearchchainForwardConfig config) {
+        if (config.target() == null) {
+            throw new RuntimeException(
+                    "Configuration value searchchain-forward.target was null.");
+        }
+        try {
+            target = new ComponentSpecification(config.target());
+        } catch (RuntimeException e) {
+            throw new RuntimeException(
+                    "Failed constructing the component specification from searchchain-forward.target: "
+                            + config.target(), e);
+        }
+    }
+
+    @Override
+    public Result search(final Query query, final Execution execution) {
+        Execution next = createForward(execution);
+
+        if (next == null) {
+            return badResult(query);
+        } else {
+            return next.search(query);
+        }
+    }
+
+    private Result badResult(final Query query) {
+        final ErrorMessage error = noSearchchain();
+        return new Result(query, error);
+    }
+
+    @Override
+    public Pong ping(final Ping ping, final Execution execution) {
+        Execution next = createForward(execution);
+
+        if (next == null) {
+            return badPong();
+        } else {
+            return next.ping(ping);
+        }
+    }
+
+    private Pong badPong() {
+        final Pong pong = new Pong();
+        pong.addError(noSearchchain());
+        return pong;
+    }
+
+    @Override
+    public void fill(final Result result, final String summaryClass,
+            final Execution execution) {
+        Execution next = createForward(execution);
+        if (next == null) {
+            badFill(result.hits());
+            return;
+        } else {
+            next.fill(result, summaryClass);
+        }
+    }
+
+    private void badFill(HitGroup hits) {
+        hits.addError(noSearchchain());
+    }
+
+    private Execution createForward(Execution execution) {
+        Chain<Searcher> targetChain = execution.context().searchChainRegistry()
+                .getComponent(target);
+        if (targetChain == null) {
+            return null;
+        }
+        return new Execution(targetChain, execution.context());
+    }
+
+    private ErrorMessage noSearchchain() {
+        return ErrorMessage
+                .createServerIsMisconfigured("Could not get search chain matching component specification: " + target);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/FutureWaiter.java b/container-search/src/main/java/com/yahoo/search/federation/FutureWaiter.java
new file mode 100644
index 00000000000..52cd5397489
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/FutureWaiter.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation;
+
+import com.yahoo.search.searchchain.FutureResult;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author tonytv
+ */
+class FutureWaiter {
+    private class Future {
+        final FutureResult result;
+        final long timeoutInMilliseconds;
+
+        public Future(FutureResult result, long timeoutInMilliseconds) {
+            this.result = result;
+            this.timeoutInMilliseconds = timeoutInMilliseconds;
+        }
+    }
+
+    private List<Future> futures = new ArrayList<>();
+
+    public void add(FutureResult futureResult, long timeoutInMilliseconds) {
+        futures.add(new Future(futureResult, timeoutInMilliseconds));
+    }
+
+    public void waitForFutures() {
+        sortFuturesByTimeoutDescending();
+
+        final long startTime = System.currentTimeMillis();
+
+        for (Future future : futures) {
+            long timeToWait = startTime + future.timeoutInMilliseconds - System.currentTimeMillis();
+            if (timeToWait <= 0)
+                break;
+
+            future.result.get(timeToWait, TimeUnit.MILLISECONDS);
+        }
+    }
+
+    private void sortFuturesByTimeoutDescending() {
+        Collections.sort(futures, new Comparator<Future>() {
+            @Override
+            public int compare(Future lhs, Future rhs) {
+                return -compareLongs(lhs.timeoutInMilliseconds, rhs.timeoutInMilliseconds);
+            }
+
+            private int compareLongs(long lhs, long rhs) {
+                return new Long(lhs).compareTo(rhs);
+            }
+        });
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/TimeoutException.java
new file mode 100644
index 00000000000..8b7e8a1d9d5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/TimeoutException.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation;
+
+/**
+ * Thrown on timeouts
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+@SuppressWarnings("serial")
+public class TimeoutException extends RuntimeException {
+
+    public TimeoutException(String message) {
+        super(message);
+    }
+
+    public TimeoutException(String message,Throwable cause) {
+        super(message,cause);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java
new file mode 100644
index 00000000000..576c16f68db
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import java.util.Collections;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.search.federation.ProviderConfig;
+import com.yahoo.search.Result;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.statistics.Statistics;
+
+
+/**
+ * Superclass for http client searchers which depends on config. All this is doing is translating
+ * the provider and cache configurations to parameters which are passed upwards.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher {
+
+    /** Create this from a configuraton */
+    public ConfiguredHTTPClientSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) {
+        super(id, ConfiguredSearcherHelper.toConnectionList(providerConfig), new HTTPParameters(providerConfig), manager);
+    }
+
+    /** Create an instance from direct parameters having a single connection. Useful for testing */
+    public ConfiguredHTTPClientSearcher(String idString,String host,int port,String path, Statistics manager) {
+        super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager);
+    }
+
+    /** Forwards to the next in chain fill(result,summaryName) */
+    public @Override void fill(Result result,String summaryName, Execution execution,Connection connection) {
+        execution.fill(result,summaryName);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java
new file mode 100644
index 00000000000..25253f768bd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.search.federation.ProviderConfig;
+import com.yahoo.search.cache.QrBinaryCacheConfig;
+import com.yahoo.search.cache.QrBinaryCacheRegionConfig;
+import com.yahoo.search.Result;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.statistics.Statistics;
+
+import java.util.Collections;
+
+
+/**
+ * Superclass for http provider searchers which depends on config. All this is doing is translating
+ * the provider and cache configurations to parameters which are passed upwards.
+ *
+ * @author  <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author  bratseth
+ */
+public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearcher {
+
+    /** Create this from a configuraton */
+    public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) {
+        super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager);
+    }
+
+    /** Create this from a configuraton */
+    public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig,
+                                          HTTPParameters parameters, Statistics manager) {
+        super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager);
+    }
+
+    /** Create this from a configuraton with a configured cache */
+    public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig,
+                                          final QrBinaryCacheConfig cacheConfig,
+    	                                  final QrBinaryCacheRegionConfig regionConfig, Statistics manager) {
+        super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager);
+        configureCache(cacheConfig,regionConfig);
+    }
+
+    /** Create this from a configuraton with a configured cache */
+    public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig,
+                                          final QrBinaryCacheConfig cacheConfig,
+    	                                  final QrBinaryCacheRegionConfig regionConfig, HTTPParameters parameters, Statistics manager) {
+        super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager);
+        configureCache(cacheConfig,regionConfig);
+    }
+
+    /** Create an instance from direct parameters having a single connection. Useful for testing */
+    public ConfiguredHTTPProviderSearcher(String idString,String host,int port,String path, Statistics manager) {
+        super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager);
+    }
+
+    /** Create an instance from direct parameters having a single connection. Useful for testing */
+    public ConfiguredHTTPProviderSearcher(String idString,String host,int port,HTTPParameters parameters, Statistics manager) {
+        super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),parameters, manager);
+    }
+
+    /**
+     * Override this to provider multi-phase result filling towards a backend.
+     * This default implementation does nothing.
+     */
+    public @Override void fill(Result result,String summaryName, Execution execution,Connection connection) {
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java
new file mode 100644
index 00000000000..8d3ee016b4f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.yahoo.search.federation.ProviderConfig;
+
+/**
+ * Some static helper classes for configured*Searcher classes
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class ConfiguredSearcherHelper {
+
+    /** No instantiation */
+    private ConfiguredSearcherHelper() { }
+
+    public static List<Connection> toConnectionList(ProviderConfig providerConfig) {
+        List<Connection> connections=new ArrayList<>();
+        for(ProviderConfig.Node node : providerConfig.node()) {
+            connections.add(new Connection(node.host(), node.port()));
+        }
+        return connections;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java
new file mode 100644
index 00000000000..88e2c6ad0a0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+/**
+ * Represents a connection to a particular node (host/port).
+ * Right now this is just a container of connection parameters, but might be extended to
+ * contain an open connection later.
+ * The host and port state is immutable.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Connection {
+
+    private String host;
+    private int port;
+
+    public Connection(String host,int port) {
+        this.host=host;
+        this.port=port;
+    }
+
+    public String getHost() { return host; }
+
+    public int getPort() { return port; }
+
+    public String toString() {
+        return "http connection '" + host + ":" + port + "'";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java
new file mode 100644
index 00000000000..1dc58ecd65e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java
@@ -0,0 +1,125 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.HttpEntityWrapper;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * Used by HTTPSearcher when talking to services returning compressed content.
+ *
+ * @author <a href="mailto:mainak@yahoo-inc.com">Mainak Mandal</a>
+ */
+public class GzipDecompressingEntity extends HttpEntityWrapper {
+
+    private static class Resources {
+
+        byte [] buffer;
+        int total;
+
+        Resources() {
+            total = 0;
+            buffer = new byte[65536];
+        }
+        void drain(InputStream zipStream) throws IOException {
+            int numRead = zipStream.read(buffer, total, buffer.length);
+            while (numRead != -1) {
+                total += numRead;
+                if ((total + 65536) > buffer.length) {
+                    buffer = Arrays.copyOf(buffer, buffer.length + numRead);
+                }
+                numRead = zipStream.read(buffer, total, buffer.length - total);
+            }
+        }
+
+    }
+
+    private final Resources resources = new Resources();
+
+    public GzipDecompressingEntity(final HttpEntity entity) throws IllegalStateException, IOException {
+        super(entity);
+        GZIPInputStream gz = new GZIPInputStream(entity.getContent());
+        InputStream zipStream = new BufferedInputStream(gz);
+        try {
+            resources.drain(zipStream);
+        } catch (IOException e) {
+            throw e;
+        } finally {
+            zipStream.close();
+        }
+    }
+
+    @Override
+    public InputStream getContent() throws IOException, IllegalStateException {
+
+        final ByteBuffer buff = ByteBuffer.wrap(resources.buffer, 0, resources.total);
+        return new InputStream() {
+
+            @Override
+            public int available() throws IOException {
+                return buff.remaining();
+            }
+
+            @Override
+            public int read() throws IOException {
+                if (buff.hasRemaining())
+                    return buff.get() & 0xFF;
+
+                return -1;
+            }
+
+            @Override
+            public int read(byte[] b) throws IOException {
+                if (!buff.hasRemaining())
+                    return -1;
+
+                int len = b.length;
+                if (len > buff.remaining())
+                    len = buff.remaining();
+                buff.get(b, 0, len);
+                return len;
+            }
+
+            @Override
+            public int read(byte[] b, int off, int len) throws IOException {
+                if (!buff.hasRemaining())
+                    return -1;
+
+                if (len > buff.remaining())
+                    len = buff.remaining();
+                buff.get(b, off, len);
+                return len;
+            }
+
+            @Override
+            public long skip(long n) throws IOException {
+                if (!buff.hasRemaining())
+                    return -1;
+
+                if (n > buff.remaining())
+                    n = buff.remaining();
+
+                buff.position(buff.position() + (int) n);
+                return n;
+            }
+        };
+    }
+
+    @Override
+    public long getContentLength() {
+        return resources.total;
+    }
+
+    @Override
+    public void writeTo(OutputStream outstream) throws IOException {
+        outstream.write(resources.buffer, 0, resources.total);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java
new file mode 100644
index 00000000000..1459fb6f226
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java
@@ -0,0 +1,276 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.jdisc.http.CertificateStore;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.statistics.Statistics;
+
+import org.apache.http.HttpEntity;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * A utility parent for searchers which gets data from web services which is incorporated into the query.
+ * This searcher will take care of implementing the search method while the extending class implements
+ * {@link #getQueryMap} and {@link #handleResponse} to create the http request and handle the response, respectively.
+ *
+ * <p>This class automatically adds a meta hit containing latency and other
+ * meta information about the obtained HTTP data using createRequestMeta().
+ * The fields available in the hit are:</p>
+ *
+  * <dl><dt>
+ * HTTPSearcher.LOG_LATENCY_START
+ * <dd>
+ *     The latency of the external provider answering a request.
+ * <dt>
+ * HTTPSearcher.LOG_LATENCY_FINISH
+ * <dd>
+ *     Total time of the HTTP traffic, but also decoding of the data, is this
+ *     happens at the same time.
+ * <dt>
+ * HTTPSearcher.LOG_URI
+ * <dd>
+ *     The complete URI used for external service.
+ * <dt>
+ * HTTPSearcher.LOG_SCHEME
+ * <dd>
+ *     The scheme of the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_HOST
+ * <dd>
+ *     The host used for the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_PORT
+ * <dd>
+ *     The port used for the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_PATH
+ * <dd>
+ *     Path element of the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_STATUS
+ * <dd>
+ *     Status code of the HTTP response.
+ * <dt>
+ * HTTPSearcher.LOG_PROXY_TYPE
+ * <dd>
+ *     The proxy type used, if any. Default is "http".
+ * <dt>
+ * HTTPSearcher.LOG_PROXY_HOST
+ * <dd>
+ *     The proxy host, if any.
+ * <dt>
+ * HTTPSearcher.LOG_PROXY_PORT
+ * <dd>
+ *     The proxy port, if any.
+ * <dt>
+ * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name
+ * <dd>
+ *     The content of any additional request header fields.
+ * <dt>
+ * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name
+ * <dd>
+ *     The content of any additional response header fields.
+ * </dl>
+
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author bratseth
+ */
+public abstract class HTTPClientSearcher extends HTTPSearcher {
+
+    static final CompoundName REQUEST_META_CARRIER = new CompoundName("com.yahoo.search.federation.http.HTTPClientSearcher_requestMeta");
+
+    protected final static Logger log = Logger.getLogger(HTTPClientSearcher.class.getName());
+
+    /**
+     * Creates a client searcher
+     *
+     * @param id the id of this instance
+     * @param connections the connections this will load balance and fail over between
+     * @param path the path portion of the url to be used
+     */
+    public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics) {
+        super(id, connections, path, statistics);
+    }
+
+    public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics,
+                              CertificateStore certificateStore) {
+        super(id, connections, path, statistics, certificateStore);
+    }
+
+    public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, Statistics statistics) {
+        super(id, connections, parameters, statistics);
+    }
+    /**
+     * Creates a client searcher
+     *
+     * @param id the id of this instance
+     * @param connections the connections this will load balance and fail over between
+     * @param parameters the parameters to use when making http calls
+     * @param certificateStore the certificate store to use to pass certificates in requests
+     */
+    public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters,
+                              Statistics statistics, CertificateStore certificateStore) {
+        super(id, connections, parameters, statistics, certificateStore);
+    }
+
+    /** Overridden to avoid interfering with errors from nested searchers, which is inappropriate for a <i>client</i> */
+    @Override
+    public Result robustSearch(Query query, Execution execution, Connection connection) {
+        return search(query,execution,connection);
+    }
+
+    /** Implements a search towards the connection chosen by the cluster searcher for this query */
+    @Override
+    public Result search(Query query, Execution execution, Connection connection) {
+        Hit requestMeta = doHttpRequest(query, connection);
+        Result result = execution.search(query);
+        result.hits().add(requestMeta);
+        return result;
+    }
+
+    private Hit doHttpRequest(Query query, Connection connection) {
+        URI uri;
+        // Create default meta hit for holding logging information
+        Hit requestMeta = createRequestMeta();
+        query.properties().set(REQUEST_META_CARRIER, requestMeta);
+
+        query.trace("Created request information hit",false,9);
+        try {
+            uri = getURI(query, connection);
+        } catch (MalformedURLException e) {
+            query.errors().add(createMalformedUrlError(query,e));
+            return requestMeta;
+        } catch (URISyntaxException e) {
+            query.errors().add(createMalformedUrlError(query,e));
+            return requestMeta;
+        }
+
+        HttpEntity entity;
+        try {
+            if (query.getTraceLevel()>=1)
+                query.trace("Fetching " + uri.toString(), false, 1);
+            entity = getEntity(uri, requestMeta, query);
+        } catch (IOException e) {
+            query.errors().add(ErrorMessage.createBackendCommunicationError(
+                    "Error when trying to connect to HTTP backend in " + this + " using " + connection + " for " +
+                    query + ": " + Exceptions.toMessageString(e)));
+            return requestMeta;
+        } catch (TimeoutException e) {
+            query.errors().add(ErrorMessage.createTimeout("HTTP traffic timed out in "
+                    + this + " for " + query + ": " + e.getMessage()));
+            return requestMeta;
+        }
+        if (entity==null) {
+            query.errors().add(ErrorMessage.createBackendCommunicationError(
+                    "No result from connecting to HTTP backend in " + this + " using " + connection + " for " +  query));
+            return requestMeta;
+        }
+
+        try {
+            query = handleResponse(entity,query);
+        }
+        catch (IOException e) {
+            query.errors().add(ErrorMessage.createBackendCommunicationError(
+                    "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e)));
+        } finally {
+            cleanupHttpEntity(entity);
+        }
+        return requestMeta;
+    }
+
+    /** Overrides to pass the query on to the next searcher */
+    @Override
+    public Result search(Query query, Execution execution, ErrorMessage error) {
+        query.errors().add(error);
+        return execution.search(query);
+    }
+
+    /** Do nothing on fill in client searchers */
+    @Override
+    public void fill(Result result,String summaryClass,Execution execution,Connection connection) {
+    }
+
+    /**
+     * Convenience hook for unmarshalling the response and adding the information to the query.
+     * Implement this or <code>handleResponse(entity,query)</code> in any subclass.
+     * This default implementation throws an exception.
+     *
+     * @param inputStream the stream containing the data from the http service
+     * @param contentLength the length of the content in the stream in bytes, or a negative number if not known
+     * @param query the current query, to which information from the stream should be added
+     * @return query the query to propagate down the chain. This should almost always be the
+     *         query instance given as a parameter.
+     */
+    public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException {
+        throw new UnsupportedOperationException("handleResponse must be implemented by " + this);
+    }
+
+    /**
+     * Unmarshals the response and adds the resulting data to the given query.
+     * This default implementation calls
+     * <code>return handleResponse(entity.getContent(), entity.getContentLength(), query);</code>
+     * (and does some detailed query tracing).
+     *
+     * @param query the current query, to which information from the stream should be added
+     * @return query the query to propagate down the chain. This should almost always be the
+     *         query instance given as a parameter.
+     */
+    public Query handleResponse(HttpEntity entity, Query query) throws IOException {
+        long len = entity.getContentLength();
+        if (query.getTraceLevel()>=4)
+            query.trace("Received " + len + " bytes response in " + this, false, 4);
+        query = handleResponse(entity.getContent(), len, query);
+        if (query.getTraceLevel()>=2)
+            query.trace("Handled " + len + " bytes response in " + this, false, 2);
+        return query;
+    }
+
+    /** Never retry individual queries to clients for now */
+    @Override
+    protected boolean shouldRetry(Query query,Result result) { return false; }
+
+    /**
+     * numHits and offset should not be part of the cache key as cache supports
+     * partial read/write that is only one cache entry is maintained per query
+     * irrespective of the offset and numhits.
+     */
+    public abstract Map<String, String> getCacheKey(Query q);
+
+    /**
+     * Adds all key-values starting by "service." + getClientName() in query.properties().
+     * Returns the empty map if {@link #getServiceName} is not overridden.
+     */
+    @Override
+    public Map<String,String> getQueryMap(Query query) {
+        LinkedHashMap<String, String> queryMap=new LinkedHashMap<>();
+        if (getServiceName().isEmpty()) return queryMap;
+
+        for (Map.Entry<String,Object> objectProperty : query.properties().listProperties("service." + getServiceName()).entrySet()) // TODO: Make more efficient using CompoundName
+            queryMap.put(objectProperty.getKey(),objectProperty.getValue().toString());
+        return queryMap;
+    }
+
+    /**
+     * Override this to return the name of the service this is a client of.
+     * This is used to look up service specific properties as service.getServiceName.serviceSpecificProperty.
+     * This default implementation returns "", which means service specific parameters will not be used.
+     */
+    protected String getServiceName() { return ""; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java
new file mode 100644
index 00000000000..19fe1df3e2e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java
@@ -0,0 +1,315 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import com.google.common.base.Preconditions;
+import com.yahoo.search.federation.ProviderConfig.PingOption;
+import org.apache.http.conn.params.ConnManagerParams;
+import org.apache.http.conn.params.ConnPerRouteBean;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.params.HttpParams;
+
+import com.yahoo.search.federation.ProviderConfig;
+
+/**
+ * A set of parameters for talking to an http backend
+ *
+ * @author bratseth
+ */
+public final class HTTPParameters {
+
+    public static final String RETRIES = "com.yahoo.search.federation.http.retries";
+
+    private boolean frozen=false;
+
+    // All timing parameters below are in milliseconds
+    /** The url request path portion */
+    private String path="/";
+    private int connectionTimeout=2000;
+    private int readTimeout=5000;
+    private boolean persistentConnections=true;
+    private boolean enableProxy = false;
+    private String proxyHost = "localhost";
+    private int proxyPort = 1080;
+    private String method = "GET";
+    private String schema = "http";
+    private String inputEncoding = "utf-8";
+    private String outputEncoding = "utf-8";
+    private int maxTotalConnections=10000;
+    private int maxConnectionsPerRoute=10000;
+    private int socketBufferSizeBytes=-1;
+    private int retries = 1;
+    private int configuredReadTimeout = -1;
+    private int configuredConnectionTimeout = -1;
+    private int connectionPoolTimeout = -1;
+    private String ycaProxy = null;
+    private int ycaPort = 0;
+    private String ycaApplicationId = null;
+    private boolean ycaUseProxy = false;
+    private long ycaTtl = 0L;
+    private long ycaRetry = 0L;
+
+    private PingOption.Enum pingOption = PingOption.NORMAL;
+
+
+    private boolean followRedirects = true;
+
+    public HTTPParameters() {}
+
+    public HTTPParameters(String path) {
+        setPath(path);
+    }
+
+    public HTTPParameters(ProviderConfig providerConfig) {
+        configuredReadTimeout = (int) (providerConfig.readTimeout() * 1000.0d);
+        configuredConnectionTimeout = (int) (providerConfig.connectionTimeout() * 1000.0d);
+        connectionPoolTimeout = (int) (providerConfig.connectionPoolTimeout() * 1000.0d);
+        retries = providerConfig.retries();
+        setPath(providerConfig.path());
+        ycaUseProxy = providerConfig.yca().useProxy();
+        if (ycaUseProxy) {
+            ycaProxy = providerConfig.yca().host();
+            ycaPort = providerConfig.yca().port();
+        }
+        ycaApplicationId = providerConfig.yca().applicationId();
+        ycaTtl = providerConfig.yca().ttl() * 1000L;
+        ycaRetry = providerConfig.yca().retry() * 1000L;
+        followRedirects = providerConfig.followRedirects();
+        pingOption = providerConfig.pingOption();
+    }
+
+    /**
+     * Set the url path to use in queries to this. If the argument is null or empty the path is set to "/".
+     * If a leading "/" is missing, it is added automatically.
+     */
+    public final void setPath(String path) {
+        if (path==null || path.isEmpty()) path="/";
+
+        if (! path.startsWith("/"))
+            path="/" + path;
+        this.path = path;
+    }
+
+    public PingOption.Enum getPingOption() {
+        return pingOption;
+    }
+
+    public void setPingOption(PingOption.Enum pingOption) {
+        Preconditions.checkNotNull(pingOption);
+        ensureNotFrozen();
+        this.pingOption = pingOption;
+    }
+
+    /** Returns the url path. Default is "/". */
+    public String getPath() { return path; }
+
+    public boolean getFollowRedirects() {
+        return followRedirects;
+    }
+
+    public void setFollowRedirects(boolean followRedirects) {
+        ensureNotFrozen();
+        this.followRedirects = followRedirects;
+    }
+
+
+    public void setConnectionTimeout(int connectionTimeout) {
+        ensureNotFrozen();
+        this.connectionTimeout=connectionTimeout;
+    }
+
+    /** Returns the connection timeout in milliseconds. Default is 2000. */
+    public int getConnectionTimeout() { return connectionTimeout; }
+
+    public void setReadTimeout(int readTimeout) {
+        ensureNotFrozen();
+        this.readTimeout=readTimeout;
+    }
+
+    /** Returns the read timeout in milliseconds. Default is 5000. */
+    public int getReadTimeout() { return readTimeout; }
+
+    /**
+     * <b>Note: This is currently largely a noop: Connections are reused even when this is set to true.
+     * The setting will change from sharing connections between threads to only reusing it within a thread
+     * but it is still reused.</b>
+     */
+    public void setPersistentConnections(boolean persistentConnections) {
+        ensureNotFrozen();
+        this.persistentConnections=persistentConnections;
+    }
+
+    /** Returns whether this should use persistent connections. Default is true. */
+    public boolean getPersistentConnections() { return persistentConnections; }
+
+    /** Returns whether proxying should be enabled. Default is false. */
+    public boolean getEnableProxy() { return enableProxy; }
+
+    public void setEnableProxy(boolean enableProxy ) {
+        ensureNotFrozen();
+        this.enableProxy=enableProxy;
+    }
+
+    /** Returns the proxy type to use (if enabled). Default is "http". */
+    public String getProxyType() {
+        return "http";
+    }
+
+    public void setProxyHost(String proxyHost) {
+        ensureNotFrozen();
+        this.proxyHost=proxyHost;
+    }
+
+    /** Returns the proxy host to use (if enabled). Default is "localhost". */
+    public String getProxyHost() { return proxyHost; }
+
+    public void setProxyPort(int proxyPort) {
+        ensureNotFrozen();
+        this.proxyPort=proxyPort;
+    }
+
+    /** Returns the proxy port to use (if enabled). Default is 1080. */
+    public int getProxyPort() { return proxyPort; }
+
+    public void setMethod(String method) {
+        ensureNotFrozen();
+        this.method=method;
+    }
+
+    /** Returns the http method to use. Default is "GET". */
+    public String getMethod() { return method; }
+
+    public void setSchema(String schema) {
+        ensureNotFrozen();
+        this.schema=schema;
+    }
+
+    /** Returns the schema to use. Default is "http". */
+    public String getSchema() { return schema; }
+
+    public void setInputEncoding(String inputEncoding) {
+        ensureNotFrozen();
+        this.inputEncoding=inputEncoding;
+    }
+
+    /** Returns the input encoding. Default is "utf-8". */
+    public String getInputEncoding() { return inputEncoding; }
+
+    public void setOutputEncoding(String outputEncoding) {
+        ensureNotFrozen();
+        this.outputEncoding=outputEncoding;
+    }
+
+    /** Returns the output encoding. Default is "utf-8". */
+    public String getOutputEncoding() { return outputEncoding; }
+
+    /** Make this unmodifiable. Note that any thread synchronization must be done outside this object. */
+    public void freeze() {
+        frozen=true;
+    }
+
+    private void ensureNotFrozen() {
+        if (frozen) throw new IllegalStateException("Cannot modify frozen " + this);
+    }
+
+    /**
+     * Returns the eligible subset of this as a HttpParams snapshot
+     * AND configures the Apache HTTP library with the parameters of this
+     */
+    public HttpParams toHttpParams() {
+        return toHttpParams(connectionTimeout, readTimeout);
+    }
+
+    /**
+     * Returns the eligible subset of this as a HttpParams snapshot
+     * AND configures the Apache HTTP library with the parameters of this
+     */
+    public HttpParams toHttpParams(int connectionTimeout, int readTimeout) {
+        HttpParams params = new BasicHttpParams();
+        // force use of configured value if available
+        if (configuredConnectionTimeout > 0) {
+            HttpConnectionParams.setConnectionTimeout(params, configuredConnectionTimeout);
+        } else {
+            HttpConnectionParams.setConnectionTimeout(params, connectionTimeout);
+        }
+        if (configuredReadTimeout > 0) {
+            HttpConnectionParams.setSoTimeout(params, configuredReadTimeout);
+        } else {
+            HttpConnectionParams.setSoTimeout(params, readTimeout);
+        }
+        if (socketBufferSizeBytes > 0) {
+            HttpConnectionParams.setSocketBufferSize(params, socketBufferSizeBytes);
+        }
+        if (connectionPoolTimeout > 0) {
+            ConnManagerParams.setTimeout(params, connectionPoolTimeout);
+        }
+        ConnManagerParams.setMaxTotalConnections(params, maxTotalConnections);
+        ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(maxConnectionsPerRoute));
+        if (retries >= 0) {
+            params.setIntParameter(RETRIES, retries);
+        }
+        params.setParameter("http.protocol.handle-redirects", followRedirects);
+        return params;
+    }
+
+    public int getMaxTotalConnections() {
+        return maxTotalConnections;
+    }
+
+    public void setMaxTotalConnections(int maxTotalConnections) {
+        ensureNotFrozen();
+        this.maxTotalConnections = maxTotalConnections;
+    }
+
+    public int getMaxConnectionsPerRoute() {
+        return maxConnectionsPerRoute;
+    }
+
+    public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) {
+        ensureNotFrozen();
+        this.maxConnectionsPerRoute = maxConnectionsPerRoute;
+    }
+
+    public int getSocketBufferSizeBytes() {
+        return socketBufferSizeBytes;
+    }
+
+    public void setSocketBufferSizeBytes(int socketBufferSizeBytes) {
+        ensureNotFrozen();
+        this.socketBufferSizeBytes = socketBufferSizeBytes;
+    }
+
+    public int getRetries() {
+        return retries;
+    }
+
+    public void setRetries(int retries) {
+        ensureNotFrozen();
+        this.retries = retries;
+    }
+
+    public String getYcaProxy() {
+        return ycaProxy;
+    }
+
+    public int getYcaPort() {
+        return ycaPort;
+    }
+
+    public String getYcaApplicationId() {
+        return ycaApplicationId;
+    }
+
+    public boolean getYcaUseProxy() {
+        return ycaUseProxy;
+    }
+
+    public long getYcaTtl() {
+        return ycaTtl;
+    }
+
+    public long getYcaRetry() {
+        return ycaRetry;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java
new file mode 100644
index 00000000000..c2bc6b2196b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java
@@ -0,0 +1,260 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.component.ComponentId;
+import com.yahoo.jdisc.http.CertificateStore;
+import com.yahoo.search.cache.QrBinaryCacheConfig;
+import com.yahoo.search.cache.QrBinaryCacheRegionConfig;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.federation.FederationSearcher;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.statistics.Counter;
+import com.yahoo.statistics.Statistics;
+import com.yahoo.statistics.Value;
+
+import org.apache.http.HttpEntity;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Superclass of searchers which talks to HTTP backends. Implement a subclass to talk to a backend
+ * over HTTP which is not supported by the platform out of the box.
+ * <p>
+ * Implementations must override one of the <code>unmarshal</code> methods to unmarshal the response.
+ * </p>
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author bratseth
+ */
+public abstract class HTTPProviderSearcher extends HTTPSearcher {
+
+    private final Counter emptyResults;
+    private final Value hitsPerQuery;
+    private final Value responseLatency;
+    private final Counter readTimeouts;
+
+    private final static List<String> excludedSourceProperties = ImmutableList.of("offset", "hits", "provider");
+
+    protected final static Logger log = Logger.getLogger(HTTPProviderSearcher.class.getName());
+
+    /** The name of the cache used (which is just getid().stringValue(), or null if no cache is used */
+    protected String cacheName=null;
+
+    public HTTPProviderSearcher(ComponentId id, List<Connection> connections,String path, Statistics statistics) {
+        this(id,connections,new HTTPParameters(path), statistics);
+    }
+
+    /** Creates a http provider searcher using id.getName as provider name */
+    public HTTPProviderSearcher(ComponentId id, List<Connection> connections, String path,
+                                Statistics statistics, CertificateStore certificateStore) {
+        this(id, connections, new HTTPParameters(path), statistics, certificateStore);
+    }
+
+    public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters,
+                                Statistics statistics) {
+        this(id, connections, parameters, statistics, new ThrowingCertificateStore());
+    }
+
+    /**
+     * Creates a provider searcher
+     *
+     * @param id the id of this instance
+     * @param connections the connections this will load balance and fail over between
+     * @param parameters the parameters to use when making http calls
+     */
+    public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters,
+                                Statistics statistics, CertificateStore certificateStore) {
+        super(id, connections, parameters, statistics, certificateStore);
+        String suffix = "_" + getId().getName().replace('.', '_');
+        hitsPerQuery = new Value("hits_per_query" + suffix, statistics,
+                new Value.Parameters().setLogRaw(false).setNameExtension(false).setLogMean(true));
+        responseLatency = new Value(LOG_LATENCY_START + suffix, statistics,
+                                    new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false));
+        emptyResults = new Counter("empty_results" + suffix, statistics, false);
+        readTimeouts = new Counter(LOG_READ_TIMEOUT_PREFIX + suffix, statistics, false);
+    }
+
+    /** @deprecated this method does nothing */
+    @Deprecated
+    protected void configureCache(final QrBinaryCacheConfig cacheConfig,final QrBinaryCacheRegionConfig regionConfig) {
+    }
+
+    /**
+     * Unmarshal the stream by converting it to hits and adding the hits to the given result.
+     * A convenience hook called by the default <code>unmarshal(entity,result).</code>
+     * Override this in subclasses which does not override <code>unmarshal(entity,result).</code>
+     * <p>
+     * This default implementation throws an exception.
+     *
+     * @param stream the stream of data returned
+     * @param contentLength the length of the content in bytes if known, or a negative number if unknown
+     * @param result the result to which unmarshalled data should be added
+     */
+    public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException {
+        throw new UnsupportedOperationException("Unmarshal must be implemented by " + this);
+    }
+
+    /**
+     * Unmarshal the result from an http entity. This default implementation calls
+     * <code>unmarshal(entity.getContent(), entity.getContentLength(), result)</code>
+     * (and does some detailed query tracing).
+     *
+     * @param entity the entity containing the data to unmarshal
+     * @param result the result to which unmarshalled data should be added
+     */
+    public void unmarshal(HttpEntity entity,Result result) throws IOException {
+        Query query=result.getQuery();
+        long len = entity.getContentLength();
+        if (query.getTraceLevel()>=4)
+            query.trace("Received " + len + " bytes response in " + this, false, 4);
+        query.trace("Unmarshaling result.", false, 6);
+        unmarshal(entity.getContent(), len, result);
+
+        if (query.getTraceLevel()>=2)
+            query.trace("Handled " + len + " bytes response in " + this, false, 2);
+
+    }
+
+    protected void addNonExcludedSourceProperties(Query query, Map<String, String> queryMap) {
+        Properties sourceProperties = FederationSearcher.getSourceProperties(query);
+        if (sourceProperties != null) {
+            for(Map.Entry<String, Object> entry : sourceProperties.listProperties("").entrySet()) {
+                if (!excludedSourceProperties.contains(entry.getKey())) {
+                    queryMap.put(entry.getKey(), entry.getValue().toString());
+                }
+            }
+        }
+    }
+
+    /**
+     * Hook called at the moment the result is returned from this searcher. This default implementation
+     * does <code>return result</code>.
+     *
+     * @param result the result which is to be returned
+     * @param requestMeta the request information hit, or null if none was created (e.g if this was a cache lookup)
+     * @param e the exception caused during execution of this query, or null if none
+     * @return the result which is returned upwards
+     */
+    protected Result inspectAndReturnFinalResult(Result result, Hit requestMeta, Exception e) {
+        return result;
+    }
+
+    private Result statisticsBeforeInspection(Result result,
+            Hit requestMeta, Exception e) {
+        int hitCount = result.getConcreteHitCount();
+        if (hitCount == 0) {
+            emptyResults.increment();
+        }
+        hitsPerQuery.put((double) hitCount);
+
+        if (requestMeta != null) {
+            requestMeta.setField(LOG_HITCOUNT, Integer.valueOf(hitCount));
+        }
+
+        return inspectAndReturnFinalResult(result,
+                requestMeta, e);
+    }
+
+
+    @Override
+    protected void logResponseLatency(long latency) {
+        responseLatency.put((double) latency);
+    }
+
+    @Override
+    public Result search(Query query, Execution execution,Connection connection) {
+        // Create default meta hit for holding logging information
+        Hit requestMeta = createRequestMeta();
+        Result result  = new Result(query);
+        result.hits().add(requestMeta);
+        query.trace("Created request information hit", false, 9);
+
+        try {
+            URI uri = getURI(query, requestMeta, connection);
+            if (query.getTraceLevel()>=1)
+                query.trace("Fetching " + uri.toString(), false, 1);
+            long requestStartTime = System.currentTimeMillis();
+
+            HttpEntity entity = getEntity(uri, requestMeta, query);
+
+            // Why should consumeEntity call inspectAndReturnFinalResult itself?
+            // Seems confusing to me.
+            return entity == null
+                    ? statisticsBeforeInspection(result, requestMeta, null)
+                    : consumeEntity(entity, query, result, requestMeta, requestStartTime);
+
+        } catch (MalformedURLException|URISyntaxException e) {
+            result.hits().addError(createMalformedUrlError(query,e));
+            return statisticsBeforeInspection(result, requestMeta, e);
+        } catch (TimeoutException e) {
+            result.hits().addError(ErrorMessage.createTimeout("No time left for HTTP traffic in "
+                    + this
+                    + " for " + query + ": " + e.getMessage()));
+            return statisticsBeforeInspection(result, requestMeta, e);
+        } catch (IOException e) {
+            result.hits().addError(ErrorMessage.createBackendCommunicationError(
+                    "Error when trying to connect to HTTP backend in " + this
+                            + " for " + query + ": " + Exceptions.toMessageString(e)));
+            return statisticsBeforeInspection(result, requestMeta, e);
+        }
+    }
+
+    private Result consumeEntity(HttpEntity entity, Query query, Result result, Hit logHit, long requestStartTime) {
+
+        try {
+            // remove some time from timeout to allow for close calls with return result
+            unmarshal(new TimedHttpEntity(entity, query.getStartTime(), Math.max(1, query.getTimeout() - 10)), result);
+            logHit.setField(LOG_LATENCY_FINISH, System.currentTimeMillis() - requestStartTime);
+            return statisticsBeforeInspection(result, logHit, null);
+        } catch (IOException e) {
+            result.hits().addError(ErrorMessage.createBackendCommunicationError(
+                    "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e)));
+            return statisticsBeforeInspection(result, logHit, e);
+        } catch (TimeoutException e) {
+            readTimeouts.increment();
+            result.hits().addError(ErrorMessage
+                    .createTimeout("Timed out while reading/unmarshaling from backend in "
+                            + this + " for " + query
+                            + ": " + e.getMessage()));
+            return statisticsBeforeInspection(result, logHit, e);
+        } finally { // TODO: The scope of this finally must be enlarged to release the connection also on errors
+            cleanupHttpEntity(entity);
+        }
+    }
+
+    /**
+     * Returns the key-value pairs that should be added as properties to the request url sent to the service.
+     * Must be overridden in subclasses to add the key-values expected by the service in question, unless
+     * {@link #getURI} (from which this is called) is overridden.
+     * <p>
+     * This default implementation returns the query.properties() prefixed by
+     * "source.[sourceName]" or "property.[propertyName]"
+     * (by calling {@link #addNonExcludedSourceProperties}).
+     */
+    @Override
+    public Map<String,String> getQueryMap(Query query) {
+        Map<String,String> queryMap = super.getQueryMap(query);
+        addNonExcludedSourceProperties(query, queryMap);
+        return queryMap;
+    }
+
+    /**
+     * @deprecated the cache key is ignored as there is no built-in caching support
+     */
+    @Deprecated
+    public abstract Map<String, String> getCacheKey(Query q);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java
new file mode 100644
index 00000000000..65ce7b3647c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java
@@ -0,0 +1,958 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import com.google.inject.Inject;
+import com.yahoo.component.ComponentId;
+import com.yahoo.jdisc.http.CertificateStore;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.Ping;
+import com.yahoo.prelude.Pong;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.Query;
+import com.yahoo.search.cluster.ClusterSearcher;
+import com.yahoo.search.federation.ProviderConfig.PingOption;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.statistics.Counter;
+import com.yahoo.statistics.Statistics;
+import com.yahoo.text.Utf8;
+
+import org.apache.http.*;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.conn.params.ConnManagerParams;
+import org.apache.http.conn.params.ConnRoutePNames;
+import org.apache.http.conn.routing.HttpRoutePlanner;
+import org.apache.http.conn.scheme.PlainSocketFactory;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.conn.DefaultHttpRoutePlanner;
+import org.apache.http.impl.conn.SingleClientConnManager;
+import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
+import org.apache.http.params.HttpParams;
+import org.apache.http.params.HttpProtocolParams;
+import org.apache.http.protocol.BasicHttpContext;
+import org.apache.http.protocol.ExecutionContext;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.protocol.HttpRequestExecutor;
+import org.apache.http.util.EntityUtils;
+
+import javax.net.ssl.SSLHandshakeException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.UnsupportedEncodingException;
+import java.net.*;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Generic superclass of searchers making connections to some HTTP service. This
+ * supports clustered connections - a list of alternative servers may be given,
+ * requests will be hashed across these and failed over in case some are down.
+ * <p>
+ * This simply provides some utility methods for working with http connections
+ * and implements ping against the service.
+ *
+ * <p>This searcher contains code from the Apache httpcomponents client library,
+ * licensed to the Apache Software Foundation under the Apache License, Version
+ * 2.0. Please refer to http://www.apache.org/licenses/LICENSE-2.0 for details.
+ *
+ * <p>This class automatically adds a meta hit containing latency and other
+ * meta information about the obtained HTTP data using createRequestMeta().
+ * The fields available in the hit are:</p>
+ *
+ * <dl><dt>
+ * HTTPSearcher.LOG_LATENCY_START
+ * <dd>
+ *     The latency of the external provider answering a request.
+ * <dt>
+ * HTTPSearcher.LOG_LATENCY_FINISH
+ * <dd>
+ *     Total time of the HTTP traffic, but also decoding of the data, as this
+ *     happens at the same time.
+ * <dt>
+ * HTTPSearcher.LOG_HITCOUNT
+ * <dd>
+ *     Number of concrete hits in the result returned by this provider.
+ * <dt>
+ * HTTPSearcher.LOG_URI
+ * <dd>
+ *     The complete URI used for external service.
+ * <dt>
+ * HTTPSearcher.LOG_SCHEME
+ * <dd>
+ *     The scheme of the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_HOST
+ * <dd>
+ *     The host used for the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_PORT
+ * <dd>
+ *     The port used for the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_PATH
+ * <dd>
+ *     Path element of the request URI sent.
+ * <dt>
+ * HTTPSearcher.LOG_STATUS
+ * <dd>
+ *     Status code of the HTTP response.
+ * <dt>
+ * HTTPSearcher.LOG_PROXY_TYPE
+ * <dd>
+ *     The proxy type used, if any. Default is "http".
+ * <dt>
+ * HTTPSearcher.LOG_PROXY_HOST
+ * <dd>
+ *     The proxy host, if any.
+ * <dt>
+ * HTTPSearcher.LOG_PROXY_PORT
+ * <dd>
+ *     The proxy port, if any.
+ * <dt>
+ * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name
+ * <dd>
+ *     The content of any additional request header fields.
+ * <dt>
+ * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name
+ * <dd>
+ *     The content of any additional response header fields.
+ * </dl>
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ */
+public abstract class HTTPSearcher extends ClusterSearcher<Connection> {
+
+    protected static final String YCA_HTTP_HEADER = "Yahoo-App-Auth";
+
+    private static final Charset iso8859Charset = Charset.forName("ISO-8859-1");
+
+    // Logging field name constants
+    public static final String LOG_PATH = "path";
+    public static final String LOG_PORT = "port";
+    public static final String LOG_HOST = "host";
+    public static final String LOG_IP_ADDRESS = "ip_address";
+    public static final String IP_ADDRESS_UNKNOWN = "unknown";
+
+    public static final String LOG_SCHEME = "scheme";
+    public static final String LOG_URI = "uri";
+    public static final String LOG_PROXY_PORT = "proxy_port";
+    public static final String LOG_PROXY_HOST = "proxy_host";
+    public static final String LOG_PROXY_TYPE = "proxy_type";
+    public static final String LOG_STATUS = "status";
+    public static final String LOG_LATENCY_FINISH = "latency_finish";
+    public static final String LOG_LATENCY_START = "latency_start";
+    public static final String LOG_LATENCY_CONNECT = "latency_connect";
+    public static final String LOG_QUERY_PARAM_PREFIX = "query_param_";
+    public static final String LOG_HEADER_PREFIX = "header_";
+    public static final String LOG_RESPONSE_HEADER_PREFIX = "response_header_";
+    public static final String LOG_HITCOUNT = "hit_count";
+    public static final String LOG_CONNECT_TIMEOUT_PREFIX = "connect_timeout_";
+    public static final String LOG_READ_TIMEOUT_PREFIX = "read_timeout_";
+
+    protected final Logger log = Logger.getLogger(HTTPSearcher.class.getName());
+
+    /** The HTTP parameters to use. Assigned in the constructor */
+    private HTTPParameters httpParameters;
+
+    private final Counter connectTimeouts;
+
+    /** Whether to use certificates */
+    protected boolean useCertificate = false;
+
+    private final CertificateStore certificateStore;
+
+    /** The (optional) YCA application ID. */
+    private String ycaApplicationId = null;
+
+    /** The (optional) YCA proxy */
+    protected HttpHost ycaProxy = null;
+
+    /** YCA cache TTL in ms */
+    private long ycaTtl = 0L;
+
+    /** YCA retry rate in the cache if no cert is found, in ms */
+    private long ycaRetry = 0L;
+
+    /** Set at construction if this is using persistent connections */
+    private ClientConnectionManager sharedConnectionManager = null;
+
+    /** Set at construction if using non-persistent connections */
+    private ThreadLocal<SingleClientConnManager> singleClientConnManagerThreadLocal = null;
+
+    private static final SchemeRegistry schemeRegistry = new SchemeRegistry();
+
+    static {
+        schemeRegistry.register(new Scheme("http", PlainSocketFactory
+                .getSocketFactory(), 80));
+        schemeRegistry.register(new Scheme("https", SSLSocketFactory
+                .getSocketFactory(), 443));
+    }
+
+    public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics) {
+        this(componentId, connections, new HTTPParameters(path), statistics, new ThrowingCertificateStore());
+    }
+
+    /** Creates a http searcher with default connection and read timeouts (currently 2 and 5s respectively) */
+    public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics,
+                        CertificateStore certificateStore) {
+        this(componentId, connections, new HTTPParameters(path), statistics, certificateStore);
+    }
+
+    public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters,
+                        Statistics statistics) {
+        this(componentId, connections, parameters, statistics, new ThrowingCertificateStore());
+    }
+    /**
+     * Creates a http searcher
+     *
+     * @param componentId the id of this instance
+     * @param connections the connections to establish to the backend nodes
+     * @param parameters the http parameters to use. This object will be frozen if it isn't already
+     */
+    @Inject
+    public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters,
+                        Statistics statistics, CertificateStore certificateStore) {
+        super(componentId,connections,false);
+        String suffix = "_" + getId().getName().replace('.', '_');
+
+        connectTimeouts = new Counter(LOG_CONNECT_TIMEOUT_PREFIX + suffix, statistics, false);
+
+        parameters.freeze();
+        this.httpParameters = parameters;
+        this.certificateStore = certificateStore;
+
+        if (parameters.getPersistentConnections()) {
+            HttpParams params=parameters.toHttpParams();
+            HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
+            ConnManagerParams.setTimeout(params, 10);
+            sharedConnectionManager = new ThreadSafeClientConnManager(params, schemeRegistry);
+            Thread connectionPurgerThread = new Thread(() -> {
+                //this is the default value in yahoo jvm installations
+                long DNSTTLSec = 120;
+                while (true) {
+                    try {
+                        Thread.sleep(DNSTTLSec * 1000);
+                        if (sharedConnectionManager == null)
+                            continue;
+
+                        sharedConnectionManager.closeExpiredConnections();
+                        DNSTTLSec = Long.valueOf(java.security.Security
+                                .getProperty("networkaddress.cache.ttl"));
+                        //No DNS TTL, no need to close idle connections
+                        if (DNSTTLSec <= 0) {
+                            DNSTTLSec = 120;
+                            continue;
+                        }
+                        sharedConnectionManager.closeIdleConnections(2 * DNSTTLSec, TimeUnit.SECONDS);
+                    } catch (InterruptedException e) {
+                        return;
+                    } catch (NumberFormatException e) {
+                        continue;
+                    }
+                }
+            });
+            connectionPurgerThread.setDaemon(true);
+            connectionPurgerThread.start();
+
+        }
+        else {
+            singleClientConnManagerThreadLocal =new ThreadLocal<>();
+        }
+
+        initializeYCA(httpParameters, certificateStore);
+    }
+
+    /**
+     * Initialize YCA certificate and proxy if they have been set to non-null,
+     * non-empty values. It will wrap thrown exceptions from the YCA layer into
+     * RuntimeException and propagate them.
+     */
+    private void initializeYCA(HTTPParameters parameters, CertificateStore certificateStore) {
+        String applicationId = parameters.getYcaApplicationId();
+        String proxy = parameters.getYcaProxy();
+        int port = parameters.getYcaPort();
+        long ttl = parameters.getYcaTtl();
+        long retry = parameters.getYcaRetry();
+
+        if (applicationId != null && !applicationId.trim().isEmpty()) {
+            initializeCertificate(applicationId, ttl, retry, certificateStore);
+        }
+
+        if (parameters.getYcaUseProxy()) {
+            initializeProxy(proxy, port);
+        }
+    }
+
+    /** Returns the HTTP parameters used in this. This is always frozen */
+    public HTTPParameters getParameters() { return httpParameters; }
+
+    /**
+     * Returns the key-value pairs that should be added as properties to the request url sent to the service.
+     * Must be overridden in subclasses to add the key-values expected by the service in question, unless
+     * {@link #getURI} (from which this is called) is overridden.
+     * <p>
+     * This default implementation returns an empty LinkedHashMap.
+     */
+    public Map<String,String> getQueryMap(Query query) {
+        return new LinkedHashMap<>();
+    }
+
+    /**
+     * Initialize the YCA certificate.
+     * This will warn but not throw if certificates could not be loaded, as the certificates
+     * are external state which can fail independently.
+     */
+    private void initializeCertificate(String applicationId, long ttl, long retry, CertificateStore certificateStore) {
+        try {
+            // get the certificate, i.e. init the cache and check integrity
+            String certificate = certificateStore.getCertificate(applicationId, ttl, retry);
+            if (certificate == null) {
+                getLogger().log(LogLevel.WARNING, "No certificate found for application '" + applicationId + "'");
+                return;
+            }
+
+            this.useCertificate = true;
+            this.ycaApplicationId = applicationId;
+            this.ycaTtl = ttl;
+            this.ycaRetry = retry;
+            getLogger().log(LogLevel.CONFIG, "Got certificate: " + certificate);
+        }
+        catch (Exception e) {
+            getLogger().log(LogLevel.WARNING,"Exception while initializing certificate for application '" +
+                                             applicationId + "' in " + this, e);
+        }
+    }
+
+    /**
+     * Initialize the YCA proxy setting.
+     */
+    private void initializeProxy(String host, int port) {
+        ycaProxy = new HttpHost(host, port);
+        getLogger().log(LogLevel.CONFIG,"Proxy is configured; will use proxy: " + ycaProxy);
+    }
+
+    /**
+     * Same a {@code getURI(query, offset, hits, null)}.
+     * @see #getURI(Query, Hit, Connection)
+     */
+    protected URI getURI(Query query,Connection connection) throws MalformedURLException, URISyntaxException {
+        Hit requestMeta;
+        try {
+            requestMeta = (Hit) query.properties().get(HTTPClientSearcher.REQUEST_META_CARRIER);
+        } catch (ClassCastException e) {
+            requestMeta = null;
+        }
+        return getURI(query, requestMeta, connection);
+    }
+
+    /**
+     * Creates the URI for a query.
+     * Populates the {@code requestMeta} meta hit with the created URI HTTP properties.
+     *
+     * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
+     */
+    protected URI getURI(Query query, Hit requestMeta, Connection connection)
+            throws MalformedURLException, URISyntaxException {
+        StringBuilder parameters = new StringBuilder();
+
+        Map<String, String> queries = getQueryMap(query);
+        if (queries.size() > 0) {
+            Iterator<Map.Entry<String, String>> mapIterator = queries.entrySet().iterator();
+            parameters.append("?");
+            try {
+                Map.Entry<String, String> entry;
+                while (mapIterator.hasNext()) {
+                    entry = mapIterator.next();
+
+                    if (requestMeta != null)
+                        requestMeta.setField(LOG_QUERY_PARAM_PREFIX
+                                + entry.getKey(), entry.getValue());
+
+                    parameters.append(entry.getKey() + "=" + URLEncoder.encode(entry.getValue(),
+                                                                               httpParameters.getInputEncoding()));
+                    if (mapIterator.hasNext()) {
+                        parameters.append("&");
+                    }
+                }
+            } catch (UnsupportedEncodingException e) {
+                throw new RuntimeException("Unknown input encoding set in " + this, e);
+            }
+        }
+
+        URI uri = new URL(httpParameters.getSchema(), connection.getHost(),
+                          connection.getPort(), getPath() + parameters.toString()).toURI();
+        if (requestMeta != null) {
+            requestMeta.setField(LOG_URI, uri.toString());
+            requestMeta.setField(LOG_SCHEME, uri.getScheme());
+            requestMeta.setField(LOG_HOST, uri.getHost());
+            requestMeta.setField(LOG_PORT, uri.getPort());
+            requestMeta.setField(LOG_PATH, uri.getPath());
+        }
+        return uri;
+    }
+
+    /**
+     * Called by getURI() to get the path of the URI for the external service.
+     * The default implementation returns httpParameters.getPath(); subclasses
+     * which only wants to override the path from httpParameters may use this
+     * method instead of overriding all of getURI().
+     *
+     * @return the path to use for getURI
+     */
+    protected String getPath() {
+        return httpParameters.getPath();
+    }
+
+    /**
+     * The URI that is used to check if the provider is up or down. This will again be used in the
+     * checkPing method by checking that we get a response that has a good status code (below 300). If better
+     * validation than just status code checking is needed, override the checkPing method.
+     */
+    protected URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException {
+        return new URL(httpParameters.getSchema(),connection.getHost(),connection.getPort(),getPingPath()).toURI();
+    }
+
+    /**
+     * Called by getPingURI() to get the path of the URI for pinging the
+     * external service. The default implementation returns
+     * httpParameters.getPath(); subclasses which only wants to override the
+     * path from httpParameters may use this method instead of overriding all of
+     * getPingURI().
+     *
+     * @return the path to use for getPingURI
+     */
+    protected String getPingPath() {
+        return httpParameters.getPath();
+    }
+
+    /**
+     * Checks if the response is valid.
+     * @param response The response from the ping request
+     * @param pong The pong result to return back to the calling method. This method
+     * will add an error to the pong result (using addError) if the status of the HTTP response is 300 or above.
+     */
+    protected void checkPing(HttpResponse response, Pong pong) {
+        if (response.getStatusLine().getStatusCode() >= 300) {
+            pong.addError(com.yahoo.search.result.ErrorMessage.createBackendCommunicationError(
+                    "Got error " + response.getStatusLine().getStatusCode()
+                    + " when contacting backend")
+            );
+        }
+    }
+
+    /**
+     * Pinging in HTTPBackend is done by creating a PING uri from http://host:port/path.
+     * If this returns a status that is below 300, the ping is considered good.
+     *
+     * If another uri is needed for pinging, reimplement getPingURI.
+     *
+     * Override either this method to change how ping
+     */
+    @Override
+    public Pong ping(Ping ping, Connection connection) {
+        URI uri = null;
+        Pong pong = new Pong();
+        HttpResponse response = null;
+
+        if (httpParameters.getPingOption() == PingOption.DISABLE)
+            return pong;
+
+        try {
+            uri = getPingURI(connection);
+            if (uri == null)
+                pong.addError(ErrorMessage.createIllegalQuery("Ping uri is null"));
+            if (uri.getHost()==null) {
+                pong.addError(ErrorMessage.createIllegalQuery("Ping uri has no host"));
+                uri=null;
+            }
+        } catch (MalformedURLException | URISyntaxException e) {
+            pong.addError(ErrorMessage.createIllegalQuery("Malformed ping uri '" + uri + "': " +
+                                                                              Exceptions.toMessageString(e)));
+        } catch (RuntimeException e) {
+            log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e);
+            pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " +
+                                                                              Exceptions.toMessageString(e)));
+        }
+
+        if (uri == null) return pong;
+        pong.setPingInfo("using uri '" + uri + "'");
+
+        try {
+            response = getPingResponse(uri, ping);
+            checkPing(response, pong);
+        } catch (IOException e) {
+            //We do not have a valid ping
+            pong.addError(ErrorMessage.createBackendCommunicationError(
+                    "Exception thrown when pinging with url '" + uri + "': " + Exceptions.toMessageString(e)));
+        } catch (TimeoutException e) {
+            pong.addError(ErrorMessage.createTimeout("Timeout for ping "
+                        + uri + " in " + this + ": " + e.getMessage()));
+        } catch (RuntimeException e) {
+            log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e);
+            pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " +
+                                                                              Exceptions.toMessageString(e)));
+        } finally {
+            if (response != null) {
+                cleanupHttpEntity(response.getEntity());
+            }
+        }
+
+        return pong;
+    }
+
+    private HttpResponse getPingResponse(URI uri, Ping ping) throws IOException {
+        long timeLeft = ping.getTimeout();
+        int connectionTimeout = (int) (timeLeft / 4L);
+        int readTimeout = (int) (timeLeft * 3L / 4L);
+
+        Map<String, String> requestHeaders = null;
+        if (httpParameters.getPingOption() == PingOption.YCA)
+            requestHeaders = generateYCAHeaders();
+
+        return getResponse(uri, null, requestHeaders, null, connectionTimeout, readTimeout);
+    }
+
+    /**
+     * Same a {@code getEntity(uri, null)}.
+     * @param uri resource to fetch
+     * @param query the originating query
+     * @throws TimeoutException If query.timeLeft() equal to or lower than 0
+     */
+    protected HttpEntity getEntity(URI uri, Query query) throws IOException{
+    	return getEntity(uri, null, query);
+    }
+
+
+    /**
+     * Gets the HTTP entity that holds the response contents.
+     * @param uri the request URI.
+     * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
+     * @param query the originating query
+     * @return the http entity, or null if none
+     * @throws java.io.IOException Whenever HTTP status code is in the 300 or higher range.
+     * @throws TimeoutException If query.timeLeft() equal to or lower than 0
+     */
+    protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException {
+        if (query.getTimeLeft() <= 0) {
+            throw new TimeoutException("No time left for querying external backend.");
+        }
+        HttpResponse response = getResponse(uri, requestMeta, query);
+        StatusLine statusLine = response.getStatusLine();
+
+        // Logging
+        if (requestMeta != null) {
+        	requestMeta.setField(LOG_STATUS, statusLine.getStatusCode());
+        	for (HeaderIterator headers = response.headerIterator(); headers.hasNext(); ) {
+        		Header h = headers.nextHeader();
+        		requestMeta.setField(LOG_RESPONSE_HEADER_PREFIX + h.getName(), h.getValue());
+        	}
+        }
+
+        if (statusLine.getStatusCode() >= 300) {
+            HttpEntity entity = response.getEntity();
+            String message = createServerReporterErrorMessage(statusLine, entity);
+            cleanupHttpEntity(response.getEntity());
+            throw new IOException(message);
+        }
+
+        return response.getEntity();
+    }
+
+    private String createServerReporterErrorMessage(StatusLine statusLine, HttpEntity entity) {
+        String message = "Error when trying to connect to HTTP backend: "
+                         + statusLine.getStatusCode() + " : " + statusLine.getReasonPhrase();
+
+        try {
+            if (entity != null) {
+                message += "(Message = " + EntityUtils.toString(entity) + ")";
+            }
+        } catch (Exception e) {
+            log.log(LogLevel.WARNING, "Could not get message.", e);
+        }
+
+        return message;
+    }
+
+    /**
+     * Creates a meta hit dedicated to holding logging information. This hit has
+     * the 'logging:[searcher's ID]' type.
+     */
+    protected Hit createRequestMeta() {
+        Hit requestMeta = new Hit("logging:" + getId().toString());
+        requestMeta.setMeta(true);
+        requestMeta.types().add("logging");
+        return requestMeta;
+    }
+
+    protected void cleanupHttpEntity(HttpEntity entity) {
+        if (entity == null) return;
+
+        try {
+            entity.consumeContent();
+        } catch (IOException e) {
+            // It is ok if do not consume it, the resource will be freed after
+            // timeout.
+            // But log it just in case.
+            log.log(LogLevel.getVespaLogLevel(LogLevel.DEBUG),
+                    "Not able to consume after processing: " + Exceptions.toMessageString(e));
+        }
+    }
+
+    /**
+     * Same as {@code getResponse(uri, null)}.
+     */
+    protected HttpResponse getResponse(URI uri, Query query) throws IOException{
+    	return getResponse(uri, null, query);
+    }
+
+    /**
+     * Executes an HTTP request and gets the response.
+     * @param uri the request URI.
+     * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
+     * @param query the originating query, used to calculate timeouts
+     */
+    protected HttpResponse getResponse(URI uri, Hit requestMeta, Query query) throws IOException {
+        long timeLeft = query.getTimeLeft();
+        int connectionTimeout = (int) (timeLeft / 4L);
+        int readTimeout = (int) (timeLeft * 3L / 4L);
+        connectionTimeout = connectionTimeout <= 0 ? 1 : connectionTimeout;
+        readTimeout = readTimeout <= 0 ? 1 : readTimeout;
+        HttpEntity reqEntity = getRequestEntity(query, requestMeta);
+        Map<String, String> reqHeaders = getRequestHeaders(query, requestMeta);
+        if ((reqEntity == null) && (reqHeaders == null)) {
+            return getResponse(uri, requestMeta, connectionTimeout, readTimeout);
+        } else {
+            return getResponse(uri, reqEntity, reqHeaders, requestMeta, connectionTimeout, readTimeout);
+        }
+    }
+
+    /**
+     * Returns the set of headers to be passed in the http request to provider backend. The default
+     * implementation returns null, unless YCA is in use. If YCA is used, it will return a map
+     * only containing the needed YCA headers.
+     */
+    protected Map<String, String> getRequestHeaders(Query query, Hit requestMeta) {
+        if (useCertificate) {
+            return generateYCAHeaders();
+        }
+        return null;
+    }
+
+ /**
+     * Returns the HTTP request entity to use when making the request for this query.
+     * This default implementation returns null.
+     *
+     * <p> Do return a repeatable entity if HTTP retry is active.
+     *
+     * @return the http request entity to use, or null to use the default entity
+     */
+    protected HttpEntity getRequestEntity(Query query, Hit requestMeta) {
+        return null;
+    }
+
+    /**
+     * Executes an HTTP request and gets the response.
+     * @param uri the request URI.
+     * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
+     * @param connectionTimeout how long to wait for getting a connection
+     * @param readTimeout timeout for reading HTTP data
+     */
+    protected HttpResponse getResponse(URI uri, Hit requestMeta, int connectionTimeout, int readTimeout)
+            throws IOException {
+        return getResponse(uri, null, null, requestMeta, connectionTimeout, readTimeout);
+    }
+
+
+    /**
+     * Executes an HTTP request and gets the response.
+     * @param uri the request URI.
+     * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
+     * @param connectionTimeout how long to wait for getting a connection
+     * @param readTimeout timeout for reading HTTP data
+     */
+    protected HttpResponse getResponse(URI uri, HttpEntity reqEntity,
+                                       Map<String, String> reqHeaders, Hit requestMeta,
+                                       int connectionTimeout, int readTimeout) throws IOException {
+
+        HttpParams httpParams = httpParameters.toHttpParams(connectionTimeout, readTimeout);
+        HttpClient httpClient = createClient(httpParams);
+        long start = 0L;
+        HttpUriRequest request;
+        if (httpParameters.getEnableProxy() && "http".equals(httpParameters.getProxyType())) {
+            HttpHost proxy = new HttpHost(httpParameters.getProxyHost(),
+                                          httpParameters.getProxyPort(), httpParameters.getProxyType());
+            httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
+            // Logging
+            if (requestMeta != null) {
+                requestMeta.setField(LOG_PROXY_TYPE, httpParameters.getProxyType());
+                requestMeta.setField(LOG_PROXY_HOST, httpParameters.getProxyHost());
+                requestMeta.setField(LOG_PROXY_PORT, httpParameters.getProxyPort());
+            }
+        }
+        if (reqEntity == null) {
+            request = createRequest(httpParameters.getMethod(), uri);
+        } else {
+            request = createRequest(httpParameters.getMethod(), uri, reqEntity);
+        }
+
+        if (reqHeaders != null) {
+            for (Entry<String, String> entry : reqHeaders.entrySet()) {
+                if (entry.getValue() == null || isAscii(entry.getValue())) {
+                    request.addHeader(entry.getKey(), entry.getValue());
+                } else {
+                    byte[] asBytes = Utf8.toBytes(entry.getValue());
+                    String asLyingString = new String(asBytes, 0, asBytes.length, iso8859Charset);
+                    request.addHeader(entry.getKey(), asLyingString);
+                }
+            }
+        }
+
+        // Logging
+        if (requestMeta != null) {
+            for (HeaderIterator headers = request.headerIterator(); headers.hasNext();) {
+                Header h = headers.nextHeader();
+                requestMeta.setField(LOG_HEADER_PREFIX + h.getName(), h.getValue());
+            }
+            start = System.currentTimeMillis();
+        }
+
+        HttpResponse response;
+
+        try {
+            HttpContext context = new BasicHttpContext();
+            response = httpClient.execute(request, context);
+
+            if (requestMeta != null) {
+                requestMeta.setField(LOG_IP_ADDRESS, getIpAddress(context));
+            }
+         } catch (ConnectTimeoutException e) {
+            connectTimeouts.increment();
+            throw e;
+        }
+
+        // Logging
+        long latencyStart = System.currentTimeMillis() - start;
+        if (requestMeta != null) {
+            requestMeta.setField(LOG_LATENCY_START, latencyStart);
+        }
+        logResponseLatency(latencyStart);
+        return response;
+    }
+
+    private String getIpAddress(HttpContext context) {
+        HttpConnection connection = (HttpConnection) context.getAttribute(ExecutionContext.HTTP_CONNECTION);
+        if (connection instanceof HttpInetConnection) {
+            InetAddress address = ((HttpInetConnection) connection).getRemoteAddress();
+            String hostAddress = address.getHostAddress();
+            return hostAddress == null ?
+                    IP_ADDRESS_UNKNOWN:
+                    hostAddress;
+        } else {
+            getLogger().log(LogLevel.DEBUG, "Unexpected connection type: " + connection.getClass().getName());
+            return IP_ADDRESS_UNKNOWN;
+        }
+    }
+
+    private boolean isAscii(String value) {
+        char[] scanBuffer = new char[value.length()];
+        value.getChars(0, value.length(), scanBuffer, 0);
+        for (char c: scanBuffer)
+            if (c > 127) return false;
+        return true;
+    }
+
+    protected void logResponseLatency(long latency) { }
+
+    /**
+     * Creates a http client for one request. Override to customize the client
+     * to use, e.g for testing. This default implementation will add the YCA
+     * proxy to params if is necessary, and then do
+     * <code>return new SearcherHttpClient(getConnectionManager(params), params);</code>
+     */
+    protected HttpClient createClient(HttpParams params) {
+        if (ycaProxy != null) {
+            params.setParameter(ConnRoutePNames.DEFAULT_PROXY, ycaProxy);
+        }
+        return new SearcherHttpClient(getConnectionManager(params), params);
+    }
+
+    /**
+     * Creates a HttpRequest. Override to customize the request.
+     * This default implementation does <code>return new HttpRequest(method,uri);</code>
+     */
+    protected HttpUriRequest createRequest(String method,URI uri) {
+        return createRequest(method, uri, null);
+    }
+
+    /**
+     * Creates a HttpRequest. Override to customize the request.
+     * This default implementation does <code>return new HttpRequest(method,uri);</code>
+     */
+    protected HttpUriRequest createRequest(String method,URI uri, HttpEntity entity) {
+        return new SearcherHttpRequest(method,uri);
+    }
+
+    /** Get a connection manager which may be used safely from this thread */
+    protected ClientConnectionManager getConnectionManager(HttpParams params) {
+        if (sharedConnectionManager != null) {// We are using shared connections
+            return sharedConnectionManager;
+        } else {
+            SingleClientConnManager singleClientConnManager = singleClientConnManagerThreadLocal.get();
+            if (singleClientConnManager == null) {
+                singleClientConnManager = new SingleClientConnManager(params, schemeRegistry);
+                singleClientConnManagerThreadLocal.set(singleClientConnManager);
+            }
+            return singleClientConnManager;
+        }
+    }
+
+    /** Utility method for creating error messages when a url is incorrect */
+    protected ErrorMessage createMalformedUrlError(Query query,Exception e) {
+        return ErrorMessage.createErrorInPluginSearcher("Malformed url in " + this + " for " + query +
+                                                        ": " + Exceptions.toMessageString(e));
+    }
+
+    private Map<String, String> generateYCAHeaders() {
+        Map<String, String> headers = new HashMap<>();
+        String certificate = certificateStore.getCertificate(ycaApplicationId, ycaTtl, ycaRetry);
+        headers.put(YCA_HTTP_HEADER, certificate);
+        return headers;
+    }
+
+    protected static class SearcherHttpClient extends DefaultHttpClient {
+
+        private final int retries;
+
+        public SearcherHttpClient(final ClientConnectionManager conman, final HttpParams params) {
+            super(conman, params);
+            retries = params.getIntParameter(HTTPParameters.RETRIES, 1);
+            addRequestInterceptor((request, context) -> {
+                if (!request.containsHeader("Accept-Encoding")) {
+                    request.addHeader("Accept-Encoding", "gzip");
+                }
+            });
+            addResponseInterceptor((response, context) -> {
+                HttpEntity entity = response.getEntity();
+                if (entity == null) return;
+                Header ceheader = entity.getContentEncoding();
+                if (ceheader == null) return;
+                for (HeaderElement codec : ceheader.getElements()) {
+                    if (codec.getName().equalsIgnoreCase("gzip")) {
+                        response.setEntity(new GzipDecompressingEntity(response.getEntity()));
+                        return;
+                    }
+                }
+            });
+        }
+
+        @Override
+        protected HttpRequestExecutor createRequestExecutor() {
+            return new HttpRequestExecutor();
+        }
+
+        @Override
+        protected HttpRoutePlanner createHttpRoutePlanner() {
+            return new DefaultHttpRoutePlanner(getConnectionManager().getSchemeRegistry());
+        }
+
+        @Override
+        protected HttpRequestRetryHandler createHttpRequestRetryHandler() {
+            return new SearcherHttpRequestRetryHandler(retries);
+        }
+    }
+
+    /** A retry handler which avoids retrying forever on errors misclassified as transient */
+    private static class SearcherHttpRequestRetryHandler implements HttpRequestRetryHandler {
+        private final int retries;
+
+        public SearcherHttpRequestRetryHandler(int retries) {
+            this.retries = retries;
+        }
+
+        @Override
+        public boolean retryRequest(IOException e, int executionCount, HttpContext httpContext) {
+            if (e == null) {
+                throw new IllegalArgumentException("Exception parameter may not be null");
+            }
+            if (executionCount > retries) {
+                return false;
+            }
+            if (e instanceof NoHttpResponseException) {
+                // Retry if the server dropped connection on us
+                return true;
+            }
+            if (e instanceof InterruptedIOException) {
+                // Timeout from federation layer
+                return false;
+            }
+            if (e instanceof UnknownHostException) {
+                // Unknown host
+                return false;
+            }
+            if (e instanceof SSLHandshakeException) {
+                // SSL handshake exception
+                return false;
+            }
+            return true;
+        }
+
+
+    }
+
+    private static class SearcherHttpRequest extends HttpRequestBase {
+        String method;
+
+        public SearcherHttpRequest(String method, final URI uri) {
+            super();
+            this.method = method;
+            setURI(uri);
+        }
+
+        @Override
+        public String getMethod() {
+            return method;
+        }
+    }
+
+    /**
+     * Only for testing.
+     */
+    public void shutdownConnectionManagers() {
+        ClientConnectionManager manager;
+        if (sharedConnectionManager != null) {
+            manager = sharedConnectionManager;
+        } else {
+            manager = singleClientConnManagerThreadLocal.get();
+        }
+        if (manager != null) {
+            manager.shutdown();
+        }
+    }
+
+    protected static final class ThrowingCertificateStore implements CertificateStore {
+
+        @Override
+        public String getCertificate(String key, long ttl, long retry) {
+            throw new UnsupportedOperationException("A certificate store is not available");
+        }
+
+    }
+
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java
new file mode 100644
index 00000000000..9d89a318c32
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+
+/**
+ * Wrapper for adding timeout to an HttpEntity instance.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class TimedHttpEntity implements HttpEntity {
+    /**
+     * The wrapped entity. Never null.
+     */
+    private final HttpEntity entity;
+    private final long startTime;
+    private final long timeout;
+
+    public TimedHttpEntity(HttpEntity entity, long startTime, long timeout) {
+        if (entity == null) {
+            throw new IllegalArgumentException("TimedHttpEntity cannot be instantiated with null HttpEntity.");
+        }
+        this.entity = entity;
+        this.startTime = startTime;
+        this.timeout = timeout;
+    }
+
+
+    @Override
+    public InputStream getContent() throws IOException, IllegalStateException {
+        InputStream content = entity.getContent();
+        if (content == null) {
+            return null;
+        } else {
+            return new TimedStream(content, startTime, timeout);
+        }
+    }
+
+
+    // START OF PURE FORWARDING METHODS
+    @Override
+    public void consumeContent() throws IOException {
+        entity.consumeContent();
+    }
+
+
+    @Override
+    public Header getContentEncoding() {
+        return entity.getContentEncoding();
+    }
+
+    @Override
+    public long getContentLength() {
+        return entity.getContentLength();
+    }
+
+    @Override
+    public Header getContentType() {
+        return entity.getContentType();
+    }
+
+    @Override
+    public boolean isChunked() {
+        return entity.isChunked();
+    }
+
+    @Override
+    public boolean isRepeatable() {
+        return entity.isRepeatable();
+    }
+
+    @Override
+    public boolean isStreaming() {
+        return entity.isStreaming();
+    }
+
+    @Override
+    public void writeTo(OutputStream outstream) throws IOException {
+        entity.writeTo(outstream);
+    }
+    // END OF PURE FORWARDING METHODS
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java
new file mode 100644
index 00000000000..02777afb43c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A stream which throws a TimeoutException if query timeout has been reached.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class TimedStream extends InputStream {
+
+    /**
+     * A time barrier value, the point in time from which on read operations will cause an exception.
+     */
+    private final long limit;
+
+    /**
+     * A wrapped InputStream instance.
+     */
+    private final InputStream content;
+
+    /**
+     * Wrap an InputStream to make read operations potentially fire off
+     * TimeoutException.
+     *
+     * <p>Typical use would be<br>
+     * <code>new TimedStream(httpEntity.getContent(), query.getStartTime(), query.getTimeout())</code>
+     *
+     * @param content
+     *                the InputStream to wrap
+     * @param startTime
+     *                start time of query
+     * @param timeout
+     *                how long the query is allowed to run
+     */
+    public TimedStream(InputStream content, long startTime, long timeout) {
+        if (content == null) {
+            throw new IllegalArgumentException("Cannot instantiate TimedStream with null InputStream");
+        }
+        this.content = content;
+        // The reasion for doing it in here instead of outside the constructor
+        // is this makes the usage of the class more intuitive IMHO
+        this.limit = startTime + timeout;
+    }
+
+    private void checkTime(String message) {
+        if (System.currentTimeMillis() >= limit) {
+            throw new TimeoutException(message);
+        }
+    }
+
+    // START FORWARDING METHODS:
+    // All methods below are forwarding methods to the contained stream, where
+    // some do a timeout check.
+    @Override
+    public int read() throws IOException {
+        int data = content.read();
+        checkTime("Timed out during read().");
+        return data;
+    }
+
+    @Override
+    public int available() throws IOException {
+        return content.available();
+    }
+
+    @Override
+    public void close() throws IOException {
+        content.close();
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+        content.mark(readlimit);
+    }
+
+    @Override
+    public boolean markSupported() {
+        return content.markSupported();
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        int length = content.read(b, off, len);
+        checkTime("Timed out during read(byte[], int, int)");
+        return length;
+    }
+
+    @Override
+    public int read(byte[] b) throws IOException {
+        int length = content.read(b);
+        checkTime("Timed out during read(byte[])");
+        return length;
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+        content.reset();
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+        long skipped = content.skip(n);
+        checkTime("Timed out during skip(long)");
+        return skipped;
+    }
+    // END FORWARDING METHODS
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java
new file mode 100644
index 00000000000..9e0536ea053
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.http;
+
+/**
+ * Timeout marker for slow HTTP connections.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class TimeoutException extends RuntimeException {
+
+    /**
+     * Auto-generated version ID.
+     */
+    private static final long serialVersionUID = 7084147598258586559L;
+
+    public TimeoutException(String message) {
+        super(message);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java
new file mode 100644
index 00000000000..aa3d249ab66
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.federation.http;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/federation/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/package-info.java
new file mode 100644
index 00000000000..008e339db4b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/package-info.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * The federation layer on top of the search container. This contains
+ *
+ * <ul>
+ * <li>A model of Sources which can be selected in and for a Query and which are implemented
+ * by a Search Chain, and Providers which represents the connection to specific backends (these
+ * two are often 1-1 but not always)
+ * <li>The federation searcher responsible for forking a query to multiple sources in parallel
+ * <li>A simple searcher which can talk to other vespa services
+ * </ul>
+ */
+@ExportPackage
+package com.yahoo.search.federation;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/federation/selection/FederationTarget.java b/container-search/src/main/java/com/yahoo/search/federation/selection/FederationTarget.java
new file mode 100644
index 00000000000..676292d6a3a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/selection/FederationTarget.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.selection;
+
+import java.util.Optional;
+import com.yahoo.component.chain.Chain;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Represents a search chain that the federation searcher should send a query to,
+ * along with a timeout and
+ * custom data reserved for use by the TargetSelector.
+ *
+ * @author tonytv
+ */
+public final class FederationTarget<T> {
+    private final Chain<Searcher> chain;
+    private final FederationOptions federationOptions;
+    private final T customData;
+
+    public FederationTarget(Chain<Searcher> chain, FederationOptions federationOptions, T customData) {
+        checkNotNull(chain);
+        checkNotNull(federationOptions);
+
+        this.chain = chain;
+        this.federationOptions = federationOptions;
+        this.customData = customData;
+    }
+
+    public Chain<Searcher> getChain() {
+        return chain;
+    }
+
+    public FederationOptions getFederationOptions() {
+        return federationOptions;
+    }
+
+    /**
+     * Any data that the TargetSelector wants to associate with this target.
+     * Owned exclusively by the TargetSelector that created this instance.
+     */
+    public T getCustomData() {
+        return customData;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        FederationTarget that = (FederationTarget) o;
+
+        if (!chain.equals(that.chain)) return false;
+        if (customData != null ? !customData.equals(that.customData) : that.customData != null) return false;
+        if (!federationOptions.equals(that.federationOptions)) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = chain.hashCode();
+        result = 31 * result + federationOptions.hashCode();
+        return result;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/selection/TargetSelector.java b/container-search/src/main/java/com/yahoo/search/federation/selection/TargetSelector.java
new file mode 100644
index 00000000000..0f6bf2d5b71
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/selection/TargetSelector.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.selection;
+
+import com.yahoo.processing.execution.chain.ChainRegistry;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.federation.selection.FederationTarget;
+
+import java.util.Collection;
+
+/**
+ * Allows adding extra targets that the federation searcher should federate to.
+ *
+ * For each federation search call, the federation searcher will call targetSelector.getTargets.
+ *
+ * Then, for each target, it will:
+ * 1) call modifyTargetQuery(target, query)
+ * 2) call modifyTargetResult(target, result)
+ *
+ * @author tonytv
+ */
+public interface TargetSelector<T> {
+    Collection<FederationTarget<T>> getTargets(Query query, ChainRegistry<Searcher> searcherChainRegistry);
+
+    /**
+     * For modifying the query before sending it to a the target
+     */
+    void modifyTargetQuery(FederationTarget<T> target, Query query);
+
+    /**
+     * For modifying the result produced by the target.
+     */
+    void modifyTargetResult(FederationTarget<T> target, Result result);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/selection/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/selection/package-info.java
new file mode 100644
index 00000000000..f3c289f6b43
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/selection/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.federation.selection;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainInvocationSpec.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainInvocationSpec.java
new file mode 100644
index 00000000000..7e82801d85f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainInvocationSpec.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+
+import java.util.List;
+
+/**
+ * Specifices which search chain should be run and how it should be run.
+ *
+ * @author tonytv
+ */
+public class SearchChainInvocationSpec implements Cloneable {
+    public final ComponentId searchChainId;
+
+    public final ComponentId source;
+    public final ComponentId provider;
+
+    public final FederationOptions federationOptions;
+    public final List<String> documentTypes;
+
+    SearchChainInvocationSpec(ComponentId searchChainId,
+                              ComponentId source, ComponentId provider, FederationOptions federationOptions,
+                              List<String> documentTypes) {
+        this.searchChainId = searchChainId;
+        this.source = source;
+        this.provider = provider;
+        this.federationOptions = federationOptions;
+        this.documentTypes = documentTypes;
+    }
+
+    @Override
+    public SearchChainInvocationSpec clone() throws CloneNotSupportedException {
+        return (SearchChainInvocationSpec)super.clone();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java
new file mode 100644
index 00000000000..fc70fb5e5e7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SearchChainResolver.java
@@ -0,0 +1,160 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.processing.request.Properties;
+import com.yahoo.search.searchchain.model.federation.FederationOptions;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+/**
+ * Resolves (source, provider) component specifications to a search chain invocation spec.
+ * The provider component specification is given by the entry in the queryMap with key
+ * 'source.&lt;source-name&gt;.provider'.
+ *
+ * <p>
+ * The diagram shows the relationship between source, provider and the result:
+ * (source is used to select row, provider is used to select column.)
+ * Provider id = null is used for regular search chains.
+ * </p>
+ *
+ * <pre>
+ *                   Provider id
+ *                 null
+ *                |----+---+---+---|
+ *                | o  |   |   |   |
+ *                |----+---+---+---|
+ * Source id      |    | o | o |   |
+ *                |----+---+---+---|
+ *                |    |   |   | o |
+ *                |----+---+---+---|
+ *
+ *                    o: SearchChainInvocationSpec
+ * </pre>
+ *
+ * @author tonytv
+ */
+public class SearchChainResolver {
+    private final ComponentRegistry<Target> targets;
+    private final SortedSet<Target> defaultTargets;
+
+    public static class Builder {
+
+        private SortedSet<Target> defaultTargets = new TreeSet<>();
+
+        private final ComponentRegistry<Target> targets = new ComponentRegistry<Target>() {
+            @Override
+            public void freeze() {
+                for (Target target : allComponents()) {
+                    target.freeze();
+                }
+                super.freeze();
+            }
+        };
+
+        public Builder addSearchChain(ComponentId searchChainId) {
+            return addSearchChain(searchChainId, Collections.<String>emptyList());
+        }
+
+        public Builder addSearchChain(ComponentId searchChainId, FederationOptions federationOptions) {
+            return addSearchChain(searchChainId, federationOptions, Collections.<String>emptyList());
+        }
+
+        public Builder addSearchChain(ComponentId searchChainId, List<String> documentTypes) {
+            return addSearchChain(searchChainId, new FederationOptions(), documentTypes);
+        }
+
+        public Builder addSearchChain(ComponentId searchChainId, FederationOptions federationOptions,
+                                      List<String> documentTypes) {
+            registerTarget(new SingleTarget(searchChainId,
+                                            new SearchChainInvocationSpec(searchChainId, null, null, federationOptions, documentTypes), false));
+            return this;
+        }
+
+        private Builder registerTarget(SingleTarget singleTarget) {
+            targets.register(singleTarget.getId(), singleTarget);
+            if (singleTarget.useByDefault()) {
+                defaultTargets.add(singleTarget);
+            }
+            return this;
+        }
+
+        public Builder addSourceForProvider(ComponentId sourceId, ComponentId providerId, ComponentId searchChainId,
+                                         boolean isDefaultProviderForSource, FederationOptions federationOptions,
+                                         List<String> documentTypes) {
+
+            SearchChainInvocationSpec searchChainInvocationSpec =
+                    new SearchChainInvocationSpec(searchChainId, sourceId, providerId, federationOptions, documentTypes);
+
+            SourcesTarget sourcesTarget = getOrRegisterSourceTarget(sourceId);
+            sourcesTarget.addSource(providerId, searchChainInvocationSpec, isDefaultProviderForSource);
+
+            registerTarget(new SingleTarget(searchChainId, searchChainInvocationSpec, true));
+            return this;
+        }
+
+        private SourcesTarget getOrRegisterSourceTarget(ComponentId sourceId) {
+            Target sourcesTarget = targets.getComponent(sourceId);
+            if (sourcesTarget == null) {
+                targets.register(sourceId, new SourcesTarget(sourceId));
+                return getOrRegisterSourceTarget(sourceId);
+            } else if (sourcesTarget instanceof SourcesTarget) {
+                return (SourcesTarget) sourcesTarget;
+            } else {
+                throw new IllegalStateException("Expected " + sourceId + " to be a source.");
+            }
+        }
+
+        public void useTargetByDefault(String targetId) {
+            Target target = targets.getComponent(targetId);
+            assert target != null : "Target not added yet.";
+
+            defaultTargets.add(target);
+        }
+
+        public SearchChainResolver build() {
+            targets.freeze();
+            return new SearchChainResolver(targets, defaultTargets);
+        }
+    }
+
+    private SearchChainResolver(ComponentRegistry<Target> targets, SortedSet<Target> defaultTargets) {
+        this.targets = targets;
+        this.defaultTargets = Collections.unmodifiableSortedSet(defaultTargets);
+    }
+
+
+    public SearchChainInvocationSpec resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap)
+            throws UnresolvedSearchChainException {
+
+        Target target = resolveTarget(sourceRef);
+        return target.responsibleSearchChain(sourceToProviderMap);
+    }
+
+    private Target resolveTarget(ComponentSpecification sourceRef) throws UnresolvedSearchChainException {
+        Target target = targets.getComponent(sourceRef);
+        if (target == null) {
+            throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef);
+        }
+        return target;
+    }
+
+    public SortedSet<Target> allTopLevelTargets() {
+        SortedSet<Target> topLevelTargets = new TreeSet<>();
+        for (Target target : targets.allComponents()) {
+            if (!target.isDerived) {
+                topLevelTargets.add(target);
+            }
+        }
+        return topLevelTargets;
+    }
+
+    public SortedSet<Target> defaultTargets() {
+        return defaultTargets;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java
new file mode 100644
index 00000000000..4210b56a501
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SingleTarget.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.processing.request.Properties;
+
+/**
+ * TODO: What is this?
+ *
+* @author tonytv
+*/
+public class SingleTarget extends Target {
+    private final SearchChainInvocationSpec searchChainInvocationSpec;
+
+    public SingleTarget(ComponentId id, SearchChainInvocationSpec searchChainInvocationSpec, boolean isDerived) {
+        super(id, isDerived);
+        this.searchChainInvocationSpec = searchChainInvocationSpec;
+    }
+
+    @Override
+    public SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) {
+        return searchChainInvocationSpec;
+    }
+
+    @Override
+    public String searchRefDescription() {
+        return localId.toString();
+    }
+
+    @Override
+    void freeze() {}
+
+    public final boolean useByDefault() {
+        return searchChainInvocationSpec.federationOptions.getUseByDefault();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java
new file mode 100644
index 00000000000..8de6635e517
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourceRefResolver.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+import static com.yahoo.container.util.Util.quote;
+
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.processing.request.Properties;
+
+/**
+ * Maps a source reference to search chain invocation specs.
+ *
+ * @author tonytv
+ */
+public class SourceRefResolver {
+    private final SearchChainResolver searchChainResolver;
+
+    public SourceRefResolver(SearchChainResolver searchChainResolver) {
+        this.searchChainResolver = searchChainResolver;
+    }
+    public Set<SearchChainInvocationSpec> resolve(ComponentSpecification sourceRef, Properties sourceToProviderMap,
+            IndexFacts indexFacts)
+            throws UnresolvedSearchChainException {
+
+        try {
+            return new LinkedHashSet<>(Arrays.asList(searchChainResolver.resolve(sourceRef, sourceToProviderMap)));
+        } catch (UnresolvedSourceRefException e) {
+            return resolveClustersWithDocument(sourceRef, sourceToProviderMap, indexFacts);
+        }
+    }
+
+    private Set<SearchChainInvocationSpec> resolveClustersWithDocument(ComponentSpecification sourceRef,
+                                                                       Properties sourceToProviderMap,
+                                                                       IndexFacts indexFacts)
+            throws UnresolvedSearchChainException {
+
+        if (hasOnlyName(sourceRef)) {
+            Set<SearchChainInvocationSpec> clusterSearchChains = new LinkedHashSet<>();
+
+            List<String> clusters = indexFacts.clustersHavingSearchDefinition(sourceRef.getName());
+            for (String cluster : clusters) {
+                clusterSearchChains.add(resolveClusterSearchChain(cluster, sourceRef, sourceToProviderMap));
+            }
+
+            if (!clusterSearchChains.isEmpty())
+                return clusterSearchChains;
+        }
+
+        throw UnresolvedSourceRefException.createForMissingSourceRef(sourceRef);
+
+    }
+
+    private SearchChainInvocationSpec resolveClusterSearchChain(String cluster, ComponentSpecification sourceRef,
+                                                                Properties sourceToProviderMap) throws UnresolvedSearchChainException {
+        try {
+            return searchChainResolver.resolve(new ComponentSpecification(cluster), sourceToProviderMap);
+        } catch (UnresolvedSearchChainException e) {
+            throw new UnresolvedSearchChainException("Failed to resolve cluster search chain " + quote(cluster) +
+                    " when using source ref " + quote(sourceRef) + " as a document name.");
+        }
+    }
+
+    private boolean hasOnlyName(ComponentSpecification sourceSpec) {
+        return new ComponentSpecification(sourceSpec.getName()).equals(sourceSpec);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
new file mode 100644
index 00000000000..bb1de051ed0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/SourcesTarget.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+
+import com.google.common.base.Joiner;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.model.ComponentAdaptor;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.processing.request.Properties;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+
+public class SourcesTarget extends Target {
+    private ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>> providerSources =
+            new ComponentRegistry<ComponentAdaptor<SearchChainInvocationSpec>>() {};
+    private SearchChainInvocationSpec defaultProviderSource;
+
+    public SourcesTarget(ComponentId sourceId) {
+        super(sourceId);
+    }
+
+    @Override
+    public SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) throws UnresolvedSearchChainException {
+        ComponentSpecification providerSpecification = providerSpecificationForSource(queryProperties);
+        if (providerSpecification == null) {
+            return defaultProviderSource;
+        } else {
+            return lookupProviderSource(providerSpecification);
+        }
+    }
+
+    @Override
+    public String searchRefDescription() {
+        StringBuilder builder = new StringBuilder(sourceId().stringValue());
+        builder.append("[provider = ").
+                append(Joiner.on(", ").join(allProviderIdsStringValue())).
+                append("]");
+        return builder.toString();
+    }
+
+    private SortedSet<String> allProviderIdsStringValue() {
+        SortedSet<String> result = new TreeSet<>();
+        for (ComponentAdaptor<SearchChainInvocationSpec> providerSource : providerSources.allComponents()) {
+            result.add(providerSource.getId().stringValue());
+        }
+        return result;
+    }
+
+    private SearchChainInvocationSpec lookupProviderSource(ComponentSpecification providerSpecification)
+            throws UnresolvedSearchChainException {
+        ComponentAdaptor<SearchChainInvocationSpec> providerSource = providerSources.getComponent(providerSpecification);
+
+        if (providerSource == null)
+            throw UnresolvedProviderException.createForMissingProvider(sourceId(), providerSpecification);
+
+        return providerSource.model;
+    }
+
+    public void freeze() {
+        if (defaultProviderSource == null)
+            throw new RuntimeException("Null default provider source for source " + sourceId() + ".");
+
+        providerSources.freeze();
+    }
+
+    public void addSource(ComponentId providerId, SearchChainInvocationSpec searchChainInvocationSpec,
+                          boolean isDefaultProviderForSource) {
+        providerSources.register(providerId, new ComponentAdaptor<>(providerId, searchChainInvocationSpec));
+
+        if (isDefaultProviderForSource) {
+            setDefaultProviderSource(searchChainInvocationSpec);
+        }
+    }
+
+    private void setDefaultProviderSource(SearchChainInvocationSpec searchChainInvocationSpec) {
+        if (defaultProviderSource != null)
+            throw new RuntimeException("Tried to set two default providers for source " + sourceId() + ".");
+
+        defaultProviderSource = searchChainInvocationSpec;
+    }
+
+    ComponentId sourceId() {
+        return localId;
+    }
+
+
+    /**
+     * Looks up source.(sourceId).provider in the query properties.
+     * @return null if the default provider should be used
+     */
+    private ComponentSpecification providerSpecificationForSource(Properties queryProperties) {
+        String spec = queryProperties.getString("source." + sourceId().stringValue() + ".provider");
+        return ComponentSpecification.fromString(spec);
+    }
+
+    public SearchChainInvocationSpec defaultProviderSource() {
+        return defaultProviderSource;
+    }
+
+    public List<SearchChainInvocationSpec> allProviderSources() {
+        List<SearchChainInvocationSpec> allProviderSources = new ArrayList<>();
+        for (ComponentAdaptor<SearchChainInvocationSpec> component : providerSources.allComponents()) {
+            allProviderSources.add(component.model);
+        }
+        return allProviderSources;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java
new file mode 100644
index 00000000000..4cf5d406959
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/Target.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
+import com.yahoo.processing.request.Properties;
+
+/**
+ * TODO: What's this?
+ *
+* @author tonytv
+*/
+public abstract class Target extends AbstractComponent {
+    final ComponentId localId;
+    final boolean isDerived;
+
+    Target(ComponentId localId, boolean derived) {
+        super(localId);
+        this.localId = localId;
+        isDerived = derived;
+    }
+
+    Target(ComponentId localId) {
+        this(localId, false);
+    }
+
+    public abstract SearchChainInvocationSpec responsibleSearchChain(Properties queryProperties) throws UnresolvedSearchChainException;
+    public abstract String searchRefDescription();
+
+    abstract void freeze();
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java
new file mode 100644
index 00000000000..50b2dc95660
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedProviderException.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+
+import static com.yahoo.container.util.Util.quote;
+
+/**
+ * @author tonytv
+ */
+@SuppressWarnings("serial")
+class UnresolvedProviderException extends UnresolvedSearchChainException {
+    UnresolvedProviderException(String msg) {
+        super(msg);
+    }
+
+    static UnresolvedSearchChainException createForMissingProvider(ComponentId source,
+                                                                   ComponentSpecification provider) {
+        return new UnresolvedProviderException("No provider " + quote(provider) + " for source " + quote(source) + ".");
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java
new file mode 100644
index 00000000000..b8417a3d05a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSearchChainException.java
@@ -0,0 +1,13 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+/**
+ * Thrown if a search chain can not be resolved from one or more ids.
+ * @author tonytv
+ */
+@SuppressWarnings("serial")
+public class UnresolvedSearchChainException extends Exception {
+    public UnresolvedSearchChainException(String msg) {
+        super(msg);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java
new file mode 100644
index 00000000000..4c15366914b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/sourceref/UnresolvedSourceRefException.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.sourceref;
+
+import com.yahoo.component.ComponentSpecification;
+
+import static com.yahoo.container.util.Util.quote;
+
+/**
+ * @author tonytv
+ */
+@SuppressWarnings("serial")
+class UnresolvedSourceRefException extends UnresolvedSearchChainException {
+    UnresolvedSourceRefException(String msg) {
+        super(msg);
+    }
+
+
+    static UnresolvedSearchChainException createForMissingSourceRef(ComponentSpecification source) {
+        return new UnresolvedSourceRefException("Could not resolve source ref " + quote(source) + ".");
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java
new file mode 100644
index 00000000000..554424c267f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java
@@ -0,0 +1,170 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.vespa;
+
+import java.util.Iterator;
+
+import com.yahoo.prelude.query.*;
+
+/**
+ * Marshal a query stack into an advanced query string suitable for
+ * passing to another QRS.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author <a href="mailto:rafan@yahoo-inc.com">Rong-En Fan</a>
+ */
+public class QueryMarshaller {
+    private boolean atRoot = true;
+
+    public String marshal(Item root) {
+        if (root == null || root instanceof NullItem) {
+            return null;
+        }
+        StringBuilder s = new StringBuilder();
+        marshal(root, s);
+        atRoot = true;
+        return s.toString();
+    }
+
+    /**
+     * We do not yet care about exact match indices
+     */
+    private void marshal(Item root, StringBuilder s) {
+        switch (root.getItemType()) {
+        case OR:
+            marshalOr((OrItem) root, s);
+            break;
+        case AND:
+            marshalAnd((CompositeItem) root, s);
+            break;
+        case NOT:
+            marshalNot((NotItem) root, s);
+            break;
+        case RANK:
+            marshalRank((RankItem) root, s);
+            break;
+        case WORD:
+        case INT:
+        case PREFIX:
+        case SUBSTRING:
+        case SUFFIX:
+            marshalWord((TermItem) root, s);
+            break;
+        case PHRASE:
+            // PhraseItem and PhraseSegmentItem don't add quotes for segmented
+            // termse
+            if (root instanceof PhraseSegmentItem) {
+                marshalPhrase((PhraseSegmentItem) root, s);
+            } else {
+                marshalPhrase((PhraseItem) root, s);
+            }
+            break;
+        case NEAR:
+            marshalNear((NearItem) root, s);
+            break;
+        case ONEAR:
+            marshalNear((ONearItem) root, s);
+            break;
+        case WEAK_AND:
+            marshalWeakAnd((WeakAndItem)root, s);
+        default:
+            break;
+        }
+    }
+
+
+    private void marshalWord(TermItem item, StringBuilder s) {
+        String index = item.getIndexName();
+        if (index.length() != 0) {
+            s.append(item.getIndexName()).append(':');
+        }
+        s.append(item.stringValue());
+        if (item.getWeight() != Item.DEFAULT_WEIGHT)
+            s.append("!").append(item.getWeight());
+    }
+
+    private void marshalRank(RankItem root, StringBuilder s) {
+        marshalComposite("RANK", root, s);
+    }
+
+    private void marshalNot(NotItem root, StringBuilder s) {
+        marshalComposite("ANDNOT", root, s);
+    }
+
+    private void marshalOr(OrItem root, StringBuilder s) {
+        marshalComposite("OR", root, s);
+    }
+
+    /**
+     * Dump WORD items, and add space between each of them unless those
+     * words came from segmentation.
+     *
+     * @param root CompositeItem
+     * @param s current marshaled query
+     */
+    private void dumpWords(CompositeItem root, StringBuilder s) {
+        for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) {
+            Item word = i.next();
+            boolean useSeparator = true;
+            if (word instanceof TermItem) {
+                s.append(((TermItem) word).stringValue());
+                if (word instanceof WordItem) {
+                    useSeparator = !((WordItem) word).isFromSegmented();
+                }
+            } else {
+                dumpWords((CompositeItem) word, s);
+            }
+            if (useSeparator && i.hasNext()) {
+                s.append(' ');
+            }
+        }
+    }
+
+    private void marshalPhrase(PhraseItem root, StringBuilder s) {
+        marshalPhrase(root, s, root.isExplicit(), false);
+    }
+
+    private void marshalPhrase(PhraseSegmentItem root, StringBuilder s) {
+        marshalPhrase(root, s, root.isExplicit(), true);
+    }
+
+    private void marshalPhrase(IndexedItem root, StringBuilder s, boolean isExplicit, boolean isSegmented) {
+        String index = root.getIndexName();
+        if (index.length() != 0) {
+            s.append(root.getIndexName()).append(':');
+        }
+        if (isExplicit || !isSegmented) s.append('"');
+        dumpWords((CompositeItem) root, s);
+        if (isExplicit || !isSegmented) s.append('"');
+    }
+
+    private void marshalNear(NearItem root, StringBuilder s) {
+        marshalComposite(root.getName() + "(" + root.getDistance() + ")", root, s);
+    }
+
+    // Not only AndItem returns ItemType.AND
+    private void marshalAnd(CompositeItem root, StringBuilder s) {
+        marshalComposite("AND", root, s);
+    }
+
+    private void marshalWeakAnd(WeakAndItem root, StringBuilder s) {
+        marshalComposite("WAND(" + root.getN() + ")", root, s);
+    }
+
+    private void marshalComposite(String operator, CompositeItem root, StringBuilder s) {
+        boolean useParen = !atRoot;
+        if (useParen) {
+            s.append("( ");
+        } else {
+            atRoot = false;
+        }
+        for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) {
+            Item item = i.next();
+            marshal(item, s);
+            if (i.hasNext())
+                s.append(' ').append(operator).append(' ');
+        }
+        if (useParen) {
+            s.append(" )");
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java
new file mode 100644
index 00000000000..1361c7c14db
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java
@@ -0,0 +1,642 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.vespa;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.hitfield.XMLString;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.search.result.Relevance;
+import com.yahoo.text.XML;
+import com.yahoo.text.DoubleParser;
+import org.xml.sax.*;
+import org.xml.sax.helpers.DefaultHandler;
+import org.xml.sax.helpers.XMLReaderFactory;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * Parse Vespa XML results and create Result instances.
+ *
+ * <p> TODO: Ripe for a rewrite or major refactoring.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@SuppressWarnings("deprecation")
+public class ResultBuilder extends DefaultHandler {
+    private static final String ERROR = "error";
+
+    private static final String FIELD = "field";
+
+    private static Logger log = Logger.getLogger(ResultBuilder.class.getName());
+
+    /** Namespaces feature id (http://xml.org/sax/features/namespaces). */
+    protected static final String NAMESPACES_FEATURE_ID = "http://xml.org/sax/features/namespaces";
+
+    /**
+     * Namespace prefixes feature id
+     * (http://xml.org/sax/features/namespace-prefixes).
+     */
+    protected static final String NAMESPACE_PREFIXES_FEATURE_ID = "http://xml.org/sax/features/namespace-prefixes";
+
+    /** Validation feature id (http://xml.org/sax/features/validation). */
+    protected static final String VALIDATION_FEATURE_ID = "http://xml.org/sax/features/validation";
+
+    /**
+     * Schema validation feature id
+     * (http://apache.org/xml/features/validation/schema).
+     */
+    protected static final String SCHEMA_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/schema";
+
+    /**
+     * Dynamic validation feature id
+     * (http://apache.org/xml/features/validation/dynamic).
+     */
+    protected static final String DYNAMIC_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/dynamic";
+
+    // default settings
+
+    /** Default parser name. */
+    protected static final String DEFAULT_PARSER_NAME = "org.apache.xerces.parsers.SAXParser";
+
+    /** Default namespaces support (false). */
+    protected static final boolean DEFAULT_NAMESPACES = false;
+
+    /** Default namespace prefixes (false). */
+    protected static final boolean DEFAULT_NAMESPACE_PREFIXES = false;
+
+    /** Default validation support (false). */
+    protected static final boolean DEFAULT_VALIDATION = false;
+
+    /** Default Schema validation support (false). */
+    protected static final boolean DEFAULT_SCHEMA_VALIDATION = false;
+
+    /** Default dynamic validation support (false). */
+    protected static final boolean DEFAULT_DYNAMIC_VALIDATION = false;
+
+    private StringBuilder fieldContent;
+
+    private String fieldName;
+
+    private int fieldLevel = 0;
+
+    private boolean hasLiteralTags = false;
+
+    private Map<String, Object> hitFields = new HashMap<>();
+    private String hitType;
+    private String hitRelevance;
+    private String hitSource;
+
+    private int offset = 0;
+
+    private List<Tag> tagStack = new ArrayList<>();
+
+    private final XMLReader parser;
+
+    private Query query;
+
+    private Result result;
+
+    private static enum ResultPart {
+        ROOT, ERRORDETAILS, HIT, HITGROUP;
+    }
+
+    Deque<ResultPart> location = new ArrayDeque<>(10);
+
+    private String currentErrorCode;
+
+    private String currentError;
+
+    private Deque<HitGroup> hitGroups = new ArrayDeque<>(5);
+
+    private static class Tag {
+        public final String name;
+
+        /**
+         * Offset is a number which is generated for all data and tags inside
+         * fields, used to determine whether a tag was closed without enclosing
+         * any characters or other tags.
+         */
+        public final int offset;
+
+        public Tag(final String name, final int offset) {
+            this.name = name;
+            this.offset = offset;
+        }
+
+        @Override
+        public String toString() {
+            return name + '(' + Integer.valueOf(offset) + ')';
+        }
+    }
+
+    /** Default constructor. */
+    public ResultBuilder() throws RuntimeException {
+        this(createParser());
+    }
+
+    public ResultBuilder(XMLReader parser) {
+        this.parser = parser;
+        this.parser.setContentHandler(this);
+        this.parser.setErrorHandler(this);
+    }
+
+    public static XMLReader createParser() {
+        ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(org.apache.xerces.parsers.SAXParser.class.getClassLoader());
+
+        try {
+            XMLReader reader = XMLReaderFactory.createXMLReader(DEFAULT_PARSER_NAME);
+            setParserFeatures(reader);
+            return reader;
+        } catch (Exception e) {
+            throw new RuntimeException("error: Unable to instantiate parser ("
+                    + DEFAULT_PARSER_NAME + ")", e);
+        } finally {
+            Thread.currentThread().setContextClassLoader(savedContextClassLoader);
+        }
+    }
+
+    private static void setParserFeatures(XMLReader reader) {
+        try {
+            reader.setFeature(NAMESPACES_FEATURE_ID, DEFAULT_NAMESPACES);
+        } catch (SAXException e) {
+            log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
+                    + NAMESPACES_FEATURE_ID + ")");
+        }
+        try {
+            reader.setFeature(NAMESPACE_PREFIXES_FEATURE_ID,
+                    DEFAULT_NAMESPACE_PREFIXES);
+        } catch (SAXException e) {
+            log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
+                    + NAMESPACE_PREFIXES_FEATURE_ID + ")");
+        }
+        try {
+            reader.setFeature(VALIDATION_FEATURE_ID, DEFAULT_VALIDATION);
+        } catch (SAXException e) {
+            log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
+                    + VALIDATION_FEATURE_ID + ")");
+        }
+        try {
+            reader.setFeature(SCHEMA_VALIDATION_FEATURE_ID,
+                    DEFAULT_SCHEMA_VALIDATION);
+        } catch (SAXNotRecognizedException e) {
+            log.log(LogLevel.WARNING, "warning: Parser does not recognize feature ("
+                    + SCHEMA_VALIDATION_FEATURE_ID + ")");
+
+        } catch (SAXNotSupportedException e) {
+            log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
+                    + SCHEMA_VALIDATION_FEATURE_ID + ")");
+        }
+
+        try {
+            reader.setFeature(DYNAMIC_VALIDATION_FEATURE_ID,
+                    DEFAULT_DYNAMIC_VALIDATION);
+        } catch (SAXNotRecognizedException e) {
+            log.log(LogLevel.WARNING, "warning: Parser does not recognize feature ("
+                    + DYNAMIC_VALIDATION_FEATURE_ID + ")");
+
+        } catch (SAXNotSupportedException e) {
+            log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
+                    + DYNAMIC_VALIDATION_FEATURE_ID + ")");
+        }
+    }
+
+    @Override
+    public void startDocument() throws SAXException {
+        reset();
+        result = new Result(query);
+        hitGroups.addFirst(result.hits());
+        location.addFirst(ResultPart.ROOT);
+        return;
+    }
+
+    private void reset() {
+        result = null;
+        fieldLevel = 0;
+        hasLiteralTags = false;
+        tagStack = null;
+        fieldContent = null;
+        offset = 0;
+        currentError = null;
+        currentErrorCode = null;
+        hitGroups.clear();
+        location.clear();
+    }
+
+    @Override
+    public void startElement(String uri, String local, String raw,
+            Attributes attrs) throws SAXException {
+        // "Everybody" wants this switch to be moved into the
+        // enum class instead, but in this case, I find the classic
+        // approach more readable.
+        switch (location.peekFirst()) {
+        case HIT:
+            if (fieldLevel > 0) {
+                tagInField(raw, attrs, FIELD);
+                ++offset;
+                return;
+            }
+            if (FIELD.equals(raw)) {
+                ++fieldLevel;
+                fieldName = attrs.getValue("name");
+                fieldContent = new StringBuilder();
+                hasLiteralTags = false;
+            }
+            break;
+        case ERRORDETAILS:
+            if (fieldLevel > 0) {
+                tagInField(raw, attrs, ERROR);
+                ++offset;
+                return;
+            }
+            if (ERROR.equals(raw)) {
+                if (attrs != null) {
+                    currentErrorCode = attrs.getValue("code");
+                    currentError = attrs.getValue("error");
+                }
+                ++fieldLevel;
+                fieldContent = new StringBuilder();
+                hasLiteralTags = false;
+            }
+            break;
+        case HITGROUP:
+            if ("hit".equals(raw)) {
+                startHit(attrs);
+            } else if ("group".equals(raw)) {
+                startHitGroup(attrs);
+            }
+            break;
+        case ROOT:
+            if ("hit".equals(raw)) {
+                startHit(attrs);
+            } else if ("errordetails".equals(raw)) {
+                location.addFirst(ResultPart.ERRORDETAILS);
+            } else if ("result".equals(raw)) {
+                if (attrs != null) {
+                    String total = attrs.getValue("total-hit-count");
+                    if (total != null) {
+                        result.setTotalHitCount(Long.valueOf(total));
+                    }
+                }
+            } else if ("group".equals(raw)) {
+                startHitGroup(attrs);
+            } else if (ERROR.equals(raw)) {
+                if (attrs != null) {
+                    currentErrorCode = attrs.getValue("code");
+                    fieldContent = new StringBuilder();
+                }
+            }
+            break;
+        }
+        ++offset;
+    }
+
+    private void startHitGroup(Attributes attrs) {
+        HitGroup g = new HitGroup();
+        Set<String> types = g.types();
+
+        final String source;
+        if (attrs != null) {
+            String groupType = attrs.getValue("type");
+            if (groupType != null) {
+                for (String s : groupType.split(" ")) {
+                    if (s.length() > 0) {
+                        types.add(s);
+                    }
+                }
+            }
+
+            source = attrs.getValue("source");
+        } else {
+            source = null;
+        }
+
+        g.setId((source != null) ? source : "dummy");
+
+        hitGroups.peekFirst().add(g);
+        hitGroups.addFirst(g);
+        location.addFirst(ResultPart.HITGROUP);
+    }
+
+    private void startHit(Attributes attrs) {
+        hitFields.clear();
+        location.addFirst(ResultPart.HIT);
+        if (attrs != null) {
+            hitRelevance = attrs.getValue("relevancy");
+            hitSource = attrs.getValue("source");
+            hitType = attrs.getValue("type");
+        } else {
+            hitRelevance = null;
+            hitSource = null;
+            hitType = null;
+        }
+    }
+
+    private void tagInField(String tag, Attributes attrs, String enclosingTag) {
+        if (!hasLiteralTags) {
+            hasLiteralTags = true;
+            String fieldTillNow = XML.xmlEscape(fieldContent.toString(), false);
+            fieldContent = new StringBuilder(fieldTillNow);
+            tagStack = new ArrayList<>();
+        }
+        if (enclosingTag.equals(tag)) {
+            ++fieldLevel;
+        }
+        if (tagStack.size() > 0) {
+            Tag prevTag = tagStack.get(tagStack.size() - 1);
+            if (prevTag != null && (prevTag.offset + 1) == offset) {
+                fieldContent.append(">");
+            }
+        }
+        fieldContent.append("<").append(tag);
+        if (attrs != null) {
+            int attrCount = attrs.getLength();
+            for (int i = 0; i < attrCount; i++) {
+                fieldContent.append(" ").append(attrs.getQName(i))
+                        .append("=\"").append(
+                                XML.xmlEscape(attrs.getValue(i), true)).append(
+                                "\"");
+            }
+        }
+        tagStack.add(new Tag(tag, offset));
+    }
+
+    private void endElementInField(String qName, String enclosingTag) {
+        Tag prevTag = tagStack.get(tagStack.size() - 1);
+        if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) {
+            fieldContent.append(" />");
+        } else {
+            fieldContent.append("</").append(qName).append('>');
+        }
+        if (prevTag.name.equals(qName)) {
+            tagStack.remove(tagStack.size() - 1);
+        }
+    }
+
+    private void endElementInHitField(String qName) {
+        if (FIELD.equals(qName) && --fieldLevel == 0) {
+            Object content;
+            if (hasLiteralTags) {
+                content = new XMLString(fieldContent.toString());
+            } else {
+                content = fieldContent.toString();
+            }
+            hitFields.put(fieldName, content);
+            if ("collapseId".equals(fieldName)) {
+                hitFields.put(fieldName, Integer.valueOf(content.toString()));
+            }
+            fieldName = null;
+            fieldContent = null;
+            tagStack = null;
+        } else {
+            Tag prevTag = tagStack.get(tagStack.size() - 1);
+            if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) {
+                fieldContent.append(" />");
+            } else {
+                fieldContent.append("</").append(qName).append('>');
+            }
+            if (prevTag.name.equals(qName)) {
+                tagStack.remove(tagStack.size() - 1);
+            }
+        }
+    }
+    @Override
+    public void characters(char ch[], int start, int length)
+            throws SAXException {
+
+        switch (location.peekFirst()) {
+        case ERRORDETAILS:
+        case HIT:
+            if (fieldLevel > 0) {
+                if (hasLiteralTags) {
+                    if (tagStack.size() > 0) {
+                        Tag tag = tagStack.get(tagStack.size() - 1);
+                        if (tag != null && (tag.offset + 1) == offset) {
+                            fieldContent.append(">");
+                        }
+                    }
+                    fieldContent.append(
+                            XML.xmlEscape(new String(ch, start, length), false));
+                } else {
+                    fieldContent.append(ch, start, length);
+                }
+            }
+            break;
+        default:
+            if (fieldContent != null) {
+                fieldContent.append(ch, start, length);
+            }
+            break;
+        }
+        ++offset;
+    }
+
+    @Override
+    public void ignorableWhitespace(char ch[], int start, int length)
+            throws SAXException {
+        return;
+    }
+
+    @Override
+    public void processingInstruction(String target, String data)
+            throws SAXException {
+        return;
+    }
+
+    @Override
+    public void endElement(String namespaceURI, String localName, String qName)
+            throws SAXException {
+        switch (location.peekFirst()) {
+        case HITGROUP:
+            if ("group".equals(qName)) {
+                hitGroups.removeFirst();
+                location.removeFirst();
+            }
+            break;
+        case HIT:
+            if (fieldLevel > 0) {
+                endElementInHitField(qName);
+            }  else if ("hit".equals(qName)) {
+                 //assert(hitKeys.size() == hitValues.size());
+                 //We try to get either uri or documentID and use that as id
+                 Object docId = extractDocumentID();
+                 Hit newHit = new Hit(docId.toString());
+                 if (hitRelevance != null) newHit.setRelevance(new Relevance(DoubleParser.parse(hitRelevance)));
+                 if(hitSource != null) newHit.setSource(hitSource);
+                 if(hitType != null) {
+                     for(String type: hitType.split(" ")) {
+                         newHit.types().add(type);
+                     }
+                 }
+                 for(Map.Entry<String, Object> field : hitFields.entrySet()) {
+                     newHit.setField(field.getKey(), field.getValue());
+                 }
+
+                 hitGroups.peekFirst().add(newHit);
+                 location.removeFirst();
+            }
+            break;
+        case ERRORDETAILS:
+            if (fieldLevel == 1 && ERROR.equals(qName)) {
+                ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode),
+                        currentError,
+                        fieldContent.toString());
+                hitGroups.peekFirst().addError(error);
+                currentError = null;
+                currentErrorCode = null;
+                fieldContent = null;
+                tagStack = null;
+                fieldLevel = 0;
+            } else if (fieldLevel > 0) {
+                endElementInField(qName, ERROR);
+            } else if ("errordetails".equals(qName)) {
+                location.removeFirst();
+            }
+            break;
+        case ROOT:
+            if (ERROR.equals(qName)) {
+                ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode),
+                        fieldContent.toString());
+                hitGroups.peekFirst().setError(error);
+                currentErrorCode = null;
+                fieldContent = null;
+            }
+            break;
+        default:
+            break;
+        }
+        ++offset;
+    }
+
+    private Object extractDocumentID() {
+        Object docId = null;
+        if (hitFields.containsKey("uri")) {
+            docId = hitFields.get("uri");
+        } else {
+            final String documentId = "documentId";
+            if (hitFields.containsKey(documentId)) {
+                docId = hitFields.get(documentId);
+            } else {
+                final String lcDocumentId = toLowerCase(documentId);
+                for (Map.Entry<String, Object> e : hitFields.entrySet()) {
+                    String key = e.getKey();
+                    // case insensitive matching, checking length first hoping to avoid some lowercasing
+                    if (documentId.length() == key.length() && lcDocumentId.equals(toLowerCase(key))) {
+                        docId = e.getValue();
+                        break;
+                    }
+                }
+            }
+        }
+        if (docId == null) {
+            docId = "dummy";
+            log.info("Results from vespa backend did not contain either uri or documentId");
+        }
+        return docId;
+    }
+
+    @Override
+    public void warning(SAXParseException ex) throws SAXException {
+        printError("Warning", ex);
+    }
+
+    @Override
+    public void error(SAXParseException ex) throws SAXException {
+        printError("Error", ex);
+    }
+
+    @Override
+    public void fatalError(SAXParseException ex) throws SAXException {
+        printError("Fatal Error", ex);
+        // throw ex;
+    }
+
+    /** Prints the error message. */
+    protected void printError(String type, SAXParseException ex) {
+        StringBuilder errorMessage = new StringBuilder();
+
+        errorMessage.append(type);
+        if (ex != null) {
+            String systemId = ex.getSystemId();
+            if (systemId != null) {
+                int index = systemId.lastIndexOf('/');
+                if (index != -1)
+                    systemId = systemId.substring(index + 1);
+                errorMessage.append(' ').append(systemId);
+            }
+        }
+        errorMessage.append(':')
+            .append(ex.getLineNumber())
+            .append(':')
+            .append(ex.getColumnNumber())
+            .append(": ")
+            .append(ex.getMessage());
+        log.log(LogLevel.WARNING, errorMessage.toString());
+
+    }
+
+    public Result parse(String identifier, Query query) {
+        Result toReturn;
+
+        setQuery(query);
+        try {
+            parser.parse(identifier);
+        } catch (SAXParseException e) {
+            // ignore
+        } catch (Exception e) {
+            log.log(LogLevel.WARNING, "Error parsing result from Vespa",e);
+            Exception se = e;
+            if (e instanceof SAXException) {
+                se = ((SAXException) e).getException();
+            }
+            if (se != null)
+                se.printStackTrace(System.err);
+            else
+                e.printStackTrace(System.err);
+        }
+        toReturn = result;
+        reset();
+        return toReturn;
+    }
+
+    public Result parse(InputSource input, Query query) {
+        Result toReturn;
+
+        setQuery(query);
+        try {
+            parser.parse(input);
+        } catch (SAXParseException e) {
+            // ignore
+        } catch (Exception e) {
+            log.log(LogLevel.WARNING, "Error parsing result from Vespa",e);
+            Exception se = e;
+            if (e instanceof SAXException) {
+                se = ((SAXException) e).getException();
+            }
+            if (se != null)
+                se.printStackTrace(System.err);
+            else
+                e.printStackTrace(System.err);
+        }
+        toReturn = result;
+        reset();
+        return toReturn;
+    }
+
+
+    private void setQuery(Query query) {
+        this.query = query;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
new file mode 100644
index 00000000000..26c9b8ad2cd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
@@ -0,0 +1,270 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.federation.vespa;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Map;
+import java.util.Set;
+
+import org.xml.sax.InputSource;
+import org.xml.sax.XMLReader;
+
+import com.google.inject.Inject;
+import com.yahoo.collections.Tuple2;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.Version;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.language.Linguistics;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.QueryCanonicalizer;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.cache.QrBinaryCacheConfig;
+import com.yahoo.search.cache.QrBinaryCacheRegionConfig;
+import com.yahoo.search.federation.FederationSearcher;
+import com.yahoo.search.federation.ProviderConfig;
+import com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher;
+import com.yahoo.search.federation.http.Connection;
+import com.yahoo.search.intent.model.IntentModel;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.search.query.textserialize.TextSerialize;
+import com.yahoo.search.yql.MinimalQueryInserter;
+import com.yahoo.statistics.Statistics;
+
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+/**
+ * Backend searcher for external Vespa clusters (queried over http).
+ *
+ * <p>If the "sources" argument should be honored on an external cluster
+ * when using YQL+, override {@link #chooseYqlSources(Set)}.</p>
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@Provides("Vespa")
+@After("*")
+public class VespaSearcher extends ConfiguredHTTPProviderSearcher {
+    private final ThreadLocal<XMLReader> readerHolder = new ThreadLocal<>();
+    private final Query.Type queryType;
+    private final Tuple2<String, Version> segmenterVersion;
+
+    private static final CompoundName select = new CompoundName("select");
+    private static final CompoundName streamingUserid = new CompoundName(
+            "streaming.userid");
+    private static final CompoundName streamingGroupname = new CompoundName(
+            "streaming.groupname");
+    private static final CompoundName streamingSelection = new CompoundName(
+            "streaming.selection");
+
+    /** Create an instance from configuration */
+    public VespaSearcher(ComponentId id, ProviderConfig config,
+            QrBinaryCacheConfig c, QrBinaryCacheRegionConfig r,
+            Statistics statistics) {
+        this(id, config, c, r, statistics, null);
+    }
+
+    /**
+     * Create an instance from configuration
+     *
+     * @param linguistics used for generating meta info for YQL+
+     */
+    @Inject
+    public VespaSearcher(ComponentId id, ProviderConfig config,
+                         QrBinaryCacheConfig c, QrBinaryCacheRegionConfig r,
+                         Statistics statistics, @Nullable Linguistics linguistics) {
+        super(id, config, c, r, statistics);
+        queryType = toQueryType(config.queryType());
+        if (linguistics == null) {
+            segmenterVersion = null;
+        } else {
+            segmenterVersion = linguistics.getVersion(Linguistics.Component.SEGMENTER);
+        }
+    }
+
+    /**
+     * Create an instance from direct parameters having a single connection.
+     * Useful for testing
+     */
+    public VespaSearcher(String idString, String host, int port, String path) {
+        super(idString, host, port, path, Statistics.nullImplementation);
+        queryType = toQueryType(ProviderConfig.QueryType.LEGACY);
+        segmenterVersion = null;
+    }
+
+    void addProperty(Map<String, String> queryMap, Query query,
+            CompoundName property) {
+        Object o = query.properties().get(property);
+        if (o != null) {
+            queryMap.put(property.toString(), o.toString());
+        }
+    }
+
+    @Override
+    public Map<String, String> getQueryMap(Query query) {
+        Map<String, String> queryMap = getQueryMapWithoutHitsOffset(query);
+        queryMap.put("offset", Integer.toString(query.getOffset()));
+        queryMap.put("hits", Integer.toString(query.getHits()));
+        queryMap.put("presentation.format", "xml");
+
+        addProperty(queryMap, query, select);
+        addProperty(queryMap, query, streamingUserid);
+        addProperty(queryMap, query, streamingGroupname);
+        addProperty(queryMap, query, streamingSelection);
+        return queryMap;
+    }
+
+    @Override
+    public Map<String, String> getCacheKey(Query q) {
+        return getQueryMapWithoutHitsOffset(q);
+    }
+
+    private Map<String, String> getQueryMapWithoutHitsOffset(Query query) {
+        Map<String, String> queryMap = super.getQueryMap(query);
+        if (queryType == Query.Type.YQL) {
+            queryMap.put(MinimalQueryInserter.YQL.toString(), marshalQuery(query));
+        } else {
+            queryMap.put("query", marshalQuery(query.getModel().getQueryTree()));
+            queryMap.put("type", queryType.toString());
+        }
+
+        addNonExcludedSourceProperties(query, queryMap);
+        return queryMap;
+    }
+
+    Query.Type toQueryType(ProviderConfig.QueryType.Enum providerQueryType) {
+        if (providerQueryType == ProviderConfig.QueryType.LEGACY) {
+            return Query.Type.ADVANCED;
+        } else if (providerQueryType == ProviderConfig.QueryType.PROGRAMMATIC) {
+            return Query.Type.PROGRAMMATIC;
+        } else if (providerQueryType == ProviderConfig.QueryType.YQL) {
+            return Query.Type.YQL;
+        } else {
+            throw new RuntimeException("Query type " + providerQueryType
+                    + " unsupported.");
+        }
+    }
+
+    /**
+     * Serialize the query parameter for outgoing queries. For YQL+ queries,
+     * sources and fields will be set to all sources and all fields, to follow
+     * the behavior of other query types.
+     *
+     * @param query
+     *            the current, outgoing query
+     * @return a string to include in an HTTP request
+     */
+    public String marshalQuery(Query query) {
+        if (queryType != Query.Type.YQL) {
+            return marshalQuery(query.getModel().getQueryTree());
+        }
+
+        Query workQuery = query.clone();
+        String error = QueryCanonicalizer.canonicalize(workQuery);
+        if (error != null) {
+            getLogger().log(LogLevel.WARNING,
+                    "Could not normalize [" + query.toString() + "]: " + error);
+            // Just returning null here is the pattern from existing code...
+            return null;
+        }
+        chooseYqlSources(workQuery.getModel().getSources());
+        chooseYqlSummaryFields(workQuery.getPresentation().getSummaryFields());
+        return workQuery.yqlRepresentation(getSegmenterVersion(), false);
+    }
+
+    public String marshalQuery(QueryTree root) {
+        QueryCanonicalizer.QueryWrapper qw = new QueryCanonicalizer.QueryWrapper();
+        root = root.clone();
+        qw.setRoot(root.getRoot());
+        boolean could = QueryCanonicalizer.treeCanonicalize(qw, root.getRoot(),
+                null);
+        if (!could) {
+            return null;
+        }
+        return marshalRoot(qw.getRoot());
+    }
+
+    private String marshalRoot(Item root) {
+        switch (queryType) {
+        case ADVANCED:
+            QueryMarshaller marshaller = new QueryMarshaller();
+            return marshaller.marshal(root);
+        case PROGRAMMATIC:
+            return TextSerialize.serialize(root);
+        default:
+            throw new RuntimeException("Unsupported query type.");
+        }
+    }
+
+    private XMLReader getReader() {
+        XMLReader reader = readerHolder.get();
+        if (reader == null) {
+            reader = ResultBuilder.createParser();
+            readerHolder.set(reader);
+        }
+        return reader;
+    }
+
+    @Override
+    public void unmarshal(InputStream stream, long contentLength, Result result) {
+        ResultBuilder parser = new ResultBuilder(getReader());
+        Result mResult = parser.parse(new InputSource(stream),
+                result.getQuery());
+        result.mergeWith(mResult);
+        result.hits().addAll(mResult.hits().asUnorderedHits());
+    }
+
+    /** Returns the canonical Vespa ping URI, http://host:port/status.html */
+    @Override
+    public URI getPingURI(Connection connection) throws MalformedURLException,
+            URISyntaxException {
+        return new URL(getParameters().getSchema(), connection.getHost(),
+                connection.getPort(), "/status.html").toURI();
+    }
+
+    /**
+     * Get the segmenter version data used when creating YQL queries. Useful if
+     * overriding {@link #marshalQuery(Query)}.
+     *
+     * @return a tuple with the name of the segmenting engine in use, and its
+     *         version
+     */
+    protected Tuple2<String, Version> getSegmenterVersion() {
+        return segmenterVersion;
+    }
+
+    /**
+     * Choose which source arguments to use for the external cluster when
+     * generating a YQL+ query string. This is called from
+     * {@link #marshalQuery(Query)}. The default implementation clears the set,
+     * i.e. requests all sources. Other implementations may modify the source
+     * set as they see fit, or simply do nothing.
+     *
+     * @param sources
+     *            the set of source names to use for the outgoing query
+     */
+    protected void chooseYqlSources(Set<String> sources) {
+        sources.clear();
+    }
+
+    /**
+     * Choose which summary fields to request from the external cluster when
+     * generating a YQL+ query string. This is called from
+     * {@link #marshalQuery(Query)}. The default implementation clears the set,
+     * i.e. requests all fields. Other implementations may modify the summary
+     * field set as they see fit, or simply do nothing.
+     *
+     * @param summaryFields
+     *            the set of source names to use for the outgoing query
+     */
+    protected void chooseYqlSummaryFields(Set<String> summaryFields) {
+       summaryFields.clear();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java
new file mode 100644
index 00000000000..6a9f1decb21
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.federation.vespa;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java
new file mode 100644
index 00000000000..63139348ab3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping;
+
+import com.yahoo.search.grouping.vespa.ContinuationDecoder;
+
+/**
+ * <p>This class represents a piece of data stored by the grouping framework within a grouping result, which can
+ * subsequently be sent back along with the original request to navigate across a large result set. It is an opaque
+ * data object that is not intended to be human readable.</p>
+ *
+ * <p>To render a Cookie within a result set, you simply need to call {@link #toString()}.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class Continuation {
+
+    public static final String NEXT_PAGE = "next";
+    public static final String PREV_PAGE = "prev";
+    public static final String THIS_PAGE = "this";
+
+    public static Continuation fromString(String str) {
+        return ContinuationDecoder.decode(str);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java
new file mode 100644
index 00000000000..39bdd48c05e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping;
+
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Before;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.PhaseNames;
+
+import java.util.*;
+
+/**
+ * This searcher is responsible for turning the "select" parameter into a corresponding {@link GroupingRequest}. It will
+ * also parse any "timezone" parameter as the timezone for time expressions such as {@link
+ * com.yahoo.search.grouping.request.DayOfMonthFunction} and {@link com.yahoo.search.grouping.request.HourOfDayFunction}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@After(PhaseNames.RAW_QUERY)
+@Before(PhaseNames.TRANSFORMED_QUERY)
+@Provides(GroupingQueryParser.SELECT_PARAMETER_PARSING)
+public class GroupingQueryParser extends Searcher {
+
+    public static final String SELECT_PARAMETER_PARSING = "SelectParameterParsing";
+    public static final CompoundName PARAM_CONTINUE = new CompoundName("continue");
+    public static final CompoundName PARAM_REQUEST = new CompoundName("select");
+    public static final CompoundName PARAM_TIMEZONE = new CompoundName("timezone");
+    private static final ThreadLocal<ZoneCache> zoneCache = new ThreadLocal<>();
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        String reqParam = query.properties().getString(PARAM_REQUEST);
+        if (reqParam == null) {
+            return execution.search(query);
+        }
+        List<Continuation> continuations = getContinuations(query.properties().getString(PARAM_CONTINUE));
+        TimeZone zone = getTimeZone(query.properties().getString(PARAM_TIMEZONE, "utc"));
+        for (GroupingOperation op : GroupingOperation.fromStringAsList(reqParam)) {
+            GroupingRequest grpRequest = GroupingRequest.newInstance(query);
+            grpRequest.setRootOperation(op);
+            grpRequest.setTimeZone(zone);
+            grpRequest.continuations().addAll(continuations);
+        }
+        return execution.search(query);
+    }
+
+    private List<Continuation> getContinuations(String param) {
+        if (param == null) {
+            return Collections.emptyList();
+        }
+        List<Continuation> ret = new LinkedList<>();
+        for (String str : param.split(" ")) {
+            ret.add(Continuation.fromString(str));
+        }
+        return ret;
+    }
+
+    private TimeZone getTimeZone(String name) {
+        ZoneCache cache = zoneCache.get();
+        if (cache == null) {
+            cache = new ZoneCache();
+            zoneCache.set(cache);
+        }
+        TimeZone timeZone = cache.get(name);
+        if (timeZone == null) {
+            timeZone = TimeZone.getTimeZone(name);
+            cache.put(name, timeZone);
+        }
+        return timeZone;
+    }
+
+    @SuppressWarnings("serial")
+    private static class ZoneCache extends LinkedHashMap<String, TimeZone> {
+
+        ZoneCache() {
+            super(16, 0.75f, true);
+        }
+
+        @Override
+        protected boolean removeEldestEntry(Map.Entry<String, TimeZone> entry) {
+            return size() > 128; // large enough to cache common cases
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java
new file mode 100644
index 00000000000..8ace3ed72de
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java
@@ -0,0 +1,164 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping;
+
+import com.yahoo.net.URI;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.grouping.result.Group;
+import com.yahoo.search.grouping.result.RootGroup;
+import com.yahoo.search.result.Hit;
+
+import java.util.*;
+
+/**
+ * An instance of this class represents one of many grouping requests that are attached to a {@link Query}. Use the
+ * factory method {@link #newInstance(com.yahoo.search.Query)} to create a new instance of this, then create and set the
+ * {@link GroupingOperation} using {@link #setRootOperation(GroupingOperation)}. Once the search returns, access the
+ * result {@link Group} using the {@link #getResultGroup(Result)} method.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GroupingRequest {
+
+    private final static CompoundName PROP_REQUEST = new CompoundName(GroupingRequest.class.getName() + ".Request");
+    private final List<Continuation> continuations = new ArrayList<>();
+    private final int requestId;
+    private GroupingOperation root;
+    private TimeZone timeZone;
+    private URI resultId;
+
+    private GroupingRequest(int requestId) {
+        this.requestId = requestId;
+    }
+
+    /**
+     * Returns the id of this GroupingRequest. This id is injected into the {@link RootGroup} of the final result, and
+     * allows tracking of per-request meta data.
+     *
+     * @return The id of this.
+     */
+    public int getRequestId() {
+        return requestId;
+    }
+
+    /**
+     * Returns the root {@link GroupingOperation} that defines this request. As long as this remains unset, the request
+     * is void.
+     *
+     * @return The root operation.
+     */
+    public GroupingOperation getRootOperation() {
+        return root;
+    }
+
+    /**
+     * Sets the root {@link GroupingOperation} that defines this request. As long as this remains unset, the request is
+     * void.
+     *
+     * @param root The root operation to set.
+     * @return This, to allow chaining.
+     */
+    public GroupingRequest setRootOperation(GroupingOperation root) {
+        this.root = root;
+        return this;
+    }
+
+    /**
+     * Returns the {@link TimeZone} used when resolving time expressions such as {@link
+     * com.yahoo.search.grouping.request.DayOfMonthFunction} and {@link com.yahoo.search.grouping.request.HourOfDayFunction}.
+     *
+     * @return The time zone in use.
+     */
+    public TimeZone getTimeZone() {
+        return timeZone;
+    }
+
+    /**
+     * Sets the {@link TimeZone} used when resolving time expressions such as {@link
+     * com.yahoo.search.grouping.request.DayOfMonthFunction} and {@link com.yahoo.search.grouping.request.HourOfDayFunction}.
+     *
+     * @param timeZone The time zone to set.
+     * @return This, to allow chaining.
+     */
+    public GroupingRequest setTimeZone(TimeZone timeZone) {
+        this.timeZone = timeZone;
+        return this;
+    }
+
+    /**
+     * Returns the root result {@link RootGroup} that corresponds to this request. This is not available until the
+     * search returns. Because searchers are allowed to modify both {@link Result} and {@link Hit} objects freely, this
+     * method requires that you pass it the current {@link Result} object as argument.
+     *
+     * @param result The search result that contains the root group.
+     * @return The result {@link RootGroup} of this request, or null if not found.
+     */
+    public RootGroup getResultGroup(Result result) {
+        Hit root = result.hits().get(resultId, -1);
+        if (!(root instanceof RootGroup)) {
+            return null;
+        }
+        return (RootGroup)root;
+    }
+
+    /**
+     * Sets the result {@link RootGroup} of this request. This is used by the executing grouping searcher, and should
+     * not be called by a requesting searcher.
+     *
+     * @param group The result to set.
+     * @return This, to allow chaining.
+     */
+    public GroupingRequest setResultGroup(RootGroup group) {
+        this.resultId = group.getId();
+        return this;
+    }
+
+    /**
+     * Returns the list of {@link Continuation}s of this request. This is used by the executing grouping searcher to
+     * allow pagination of grouping results.
+     *
+     * @return The list of Continuations.
+     */
+    public List<Continuation> continuations() {
+        return continuations;
+    }
+
+    /**
+     * Creates and attaches a new instance of this class to the given {@link Query}. This is necessary to allow {@link
+     * #getRequests(Query)} to return all created requests.
+     *
+     * @param query The query to attach the request to.
+     * @return The created request.
+     */
+    public static GroupingRequest newInstance(Query query) {
+        List<GroupingRequest> lst = getRequests(query);
+        if (lst.isEmpty()) {
+            lst = new LinkedList<>();
+            query.properties().set(PROP_REQUEST, lst);
+        }
+        GroupingRequest ret = new GroupingRequest(lst.size());
+        lst.add(ret);
+        return ret;
+    }
+
+    /**
+     * Returns all instances of this class that have been attached to the given {@link Query}. If no requests have been
+     * attached to the {@link Query}, this method returns an empty list.
+     *
+     * @param query The query whose requests to return.
+     * @return The list of grouping requests.
+     */
+    @SuppressWarnings({ "unchecked" })
+    public static List<GroupingRequest> getRequests(Query query) {
+        Object lst = query.properties().get(PROP_REQUEST);
+        if (lst == null) {
+            return Collections.emptyList();
+        }
+        if (!(lst instanceof List)) {
+            throw new IllegalArgumentException("Expected " + GroupingRequest.class + ", got " + lst.getClass() + ".");
+        }
+        return (List<GroupingRequest>)lst;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
new file mode 100644
index 00000000000..1366fe1201b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping;
+
+import com.google.inject.Inject;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Before;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.config.ClusterConfig;
+import com.yahoo.search.grouping.request.AttributeValue;
+import com.yahoo.search.grouping.request.ExpressionVisitor;
+import com.yahoo.search.grouping.request.GroupingExpression;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.PhaseNames;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.yahoo.search.grouping.GroupingQueryParser.SELECT_PARAMETER_PARSING;
+
+/**
+ * This searcher ensure that all {@link GroupingRequest} objects attached to a {@link Query} makes sense to the search
+ * cluster for which this searcher has been deployed. This searcher uses exceptions to signal invalid grouping
+ * requests.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@Before(PhaseNames.BACKEND)
+@After(SELECT_PARAMETER_PARSING)
+@Provides(GroupingValidator.GROUPING_VALIDATED)
+public class GroupingValidator extends Searcher {
+
+    public static final String GROUPING_VALIDATED = "GroupingValidated";
+    public static final CompoundName PARAM_ENABLED = new CompoundName("validate_" + GroupingQueryParser.PARAM_REQUEST);
+    private final Set<String> attributeNames = new HashSet<>();
+    private final String clusterName;
+    private final boolean enabled;
+
+    /**
+     * Constructs a new instance of this searcher with the given component id and config.
+     *
+     * @param qrsConfig     The shared config for all searchers.
+     * @param clusterConfig The config for the cluster that this searcher is deployed for.
+     */
+    @Inject
+    public GroupingValidator(QrSearchersConfig qrsConfig, ClusterConfig clusterConfig,
+                             AttributesConfig attributesConfig) {
+        int clusterId = clusterConfig.clusterId();
+        QrSearchersConfig.Searchcluster.Indexingmode.Enum indexingMode = qrsConfig.searchcluster(clusterId).indexingmode();
+        enabled = (indexingMode != QrSearchersConfig.Searchcluster.Indexingmode.STREAMING);
+        clusterName = enabled ? qrsConfig.searchcluster(clusterId).name() : null;
+        for (AttributesConfig.Attribute attr : attributesConfig.attribute()) {
+            attributeNames.add(attr.name());
+        }
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        if (enabled && query.properties().getBoolean(PARAM_ENABLED, true)) {
+            ExpressionVisitor visitor = new MyVisitor();
+            for (GroupingRequest req : GroupingRequest.getRequests(query)) {
+                req.getRootOperation().visitExpressions(visitor);
+            }
+        }
+        return execution.search(query);
+    }
+
+    private class MyVisitor implements ExpressionVisitor {
+
+        @Override
+        public void visitExpression(GroupingExpression exp) {
+            if (exp instanceof AttributeValue) {
+                String name = ((AttributeValue)exp).getAttributeName();
+                if (!attributeNames.contains(name)) {
+                    throw new UnavailableAttributeException(clusterName, name);
+                }
+            }
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UnavailableAttributeException.java b/container-search/src/main/java/com/yahoo/search/grouping/UnavailableAttributeException.java
new file mode 100644
index 00000000000..7e147c88625
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/UnavailableAttributeException.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping;
+
+/**
+ * This exception is thrown by the {@link GroupingValidator} if it a {@link GroupingRequest} contains a reference to an
+ * unavailable attribute.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings("serial")
+public class UnavailableAttributeException extends RuntimeException {
+
+    private final String clusterName;
+    private final String attributeName;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param clusterName   The name of the cluster for which the request is illegal.
+     * @param attributeName The name of the attribute which is referenced but not available.
+     */
+    public UnavailableAttributeException(String clusterName, String attributeName) {
+        super("Grouping request references attribute '" + attributeName + "' which is not available " +
+              "in cluster '" + clusterName + "'.");
+        this.clusterName = clusterName;
+        this.attributeName = attributeName;
+    }
+
+    /**
+     * Returns the name of the cluster for which the request is illegal.
+     *
+     * @return The cluster name.
+     */
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    /**
+     * Returns the name of the attribute which is referenced but not available.
+     *
+     * @return The attribute name.
+     */
+    public String getAttributeName() {
+        return attributeName;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java
new file mode 100644
index 00000000000..f4145a31f33
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java
@@ -0,0 +1,279 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping;
+
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Before;
+import com.yahoo.log.LogLevel;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.grouping.request.AllOperation;
+import com.yahoo.search.grouping.request.AttributeValue;
+import com.yahoo.search.grouping.request.CountAggregator;
+import com.yahoo.search.grouping.request.EachOperation;
+import com.yahoo.search.grouping.request.GroupingExpression;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.grouping.request.MaxAggregator;
+import com.yahoo.search.grouping.request.MinAggregator;
+import com.yahoo.search.grouping.request.NegFunction;
+import com.yahoo.search.grouping.request.SummaryValue;
+import com.yahoo.search.grouping.result.Group;
+import com.yahoo.search.grouping.result.GroupList;
+import com.yahoo.search.query.Sorting;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitOrderer;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.PhaseNames;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * Implements 'unique' using a grouping expression.
+ *
+ * It doesn't work for multi-level sorting.
+ *
+ * @author andreer
+ */
+@After(PhaseNames.RAW_QUERY)
+@Before(PhaseNames.TRANSFORMED_QUERY)
+public class UniqueGroupingSearcher extends Searcher {
+
+    public static final CompoundName PARAM_UNIQUE = new CompoundName("unique");
+    private static final Logger log = Logger.getLogger(UniqueGroupingSearcher.class.getName());
+    private static final HitOrderer NOP_ORDERER = new HitOrderer() {
+
+        @Override
+        public void order(List<Hit> hits) {
+            // The order of hits is given by the grouping framework, and should not be re-ordered when we copy the hits
+            // from the groups to the base HitGroup in the result.
+        }
+    };
+    static final String LABEL_COUNT = "uniqueCount";
+    static final String LABEL_GROUPS = "uniqueGroups";
+    static final String LABEL_HITS = "uniqueHits";
+
+    /**
+     * Implements the deprecated "unique" api for deduplication by using grouping. We create a grouping expression on
+     * the field we wish to dedup on (which must be an attribute).
+     * Total hits is calculated using the new count unique groups functionality.
+     */
+    @Override
+    public Result search(Query query, Execution execution) {
+        // Determine if we should remove duplicates
+        String unique = query.properties().getString(PARAM_UNIQUE);
+        if (unique == null || unique.trim().isEmpty()) {
+            return execution.search(query);
+        }
+        query.trace("Performing deduping by attribute '" + unique + "'.", true, 3);
+        return dedupe(query, execution, unique);
+    }
+
+    /**
+     * Until we can use the grouping pagination features in 5.1, we'll have to support offset
+     * by simply requesting and discarding hit #0 up to hit #offset.
+     */
+    private static Result dedupe(Query query, Execution execution, String dedupField) {
+        Sorting sorting = query.getRanking().getSorting();
+        if (sorting != null && sorting.fieldOrders().size() > 1) {
+            query.trace("Can not use grouping for deduping with multi-level sorting.", 3);
+            // To support this we'd have to generate a grouping expression with as many levels
+            // as there are levels in the sort spec. This is probably too slow and costly that
+            // we'd ever want to actually use it (and a bit harder to implement as well).
+            return execution.search(query);
+        }
+
+        int hits = query.getHits();
+        int offset = query.getOffset();
+        int groupingHits = hits + offset;
+
+        GroupingRequest groupingRequest = GroupingRequest.newInstance(query);
+        groupingRequest.setRootOperation(
+                buildGroupingExpression(
+                        dedupField,
+                        groupingHits,
+                        query.getPresentation().getSummary(),
+                        sorting));
+
+        query.setHits(0);
+        query.setOffset(0);
+        Result result = execution.search(query);
+
+        query = result.getQuery(); // query could have changed further down in the chain
+        query.setHits(hits);
+        query.setOffset(offset);
+
+        Group root = groupingRequest.getResultGroup(result);
+        if (null == root) {
+            String msg = "Result group not found for deduping grouping request, returning empty result.";
+            query.trace(msg, 3);
+            log.log(LogLevel.WARNING, msg);
+            throw new IllegalStateException("Failed to produce deduped result set.");
+        }
+        result.hits().remove(root.getId().toString()); // hide our tracks
+
+        GroupList resultGroups = root.getGroupList(dedupField);
+        if (resultGroups == null) {
+            query.trace("Deduping grouping request returned no hits, returning empty result.", 3);
+            return result;
+        }
+
+        // Make sure that .addAll() doesn't re-order the hits we copy from the grouping
+        // framework. The groups are already in the order they should be.
+        result.hits().setOrderer(NOP_ORDERER);
+        result.hits().addAll(getRequestedHits(resultGroups, offset, hits));
+
+        Long countField = (Long) root.getField(LABEL_COUNT);
+        long count = countField != null ? countField : 0;
+        result.setTotalHitCount(count);
+
+        return result;
+    }
+
+    /**
+     * Create a hit ordering clause based on the sorting spec.
+     *
+     * @param sortingSpec A (single level!) sorting specification
+     * @return a grouping expression which produces a sortable value
+     */
+    private static List<GroupingExpression> createHitOrderingClause(Sorting sortingSpec) {
+        List<GroupingExpression> orderingClause = new ArrayList<>();
+        for (Sorting.FieldOrder fieldOrder : sortingSpec.fieldOrders()) {
+            Sorting.Order sortOrder = fieldOrder.getSortOrder();
+            switch (sortOrder) {
+            case ASCENDING:
+            case UNDEFINED:
+                // When we want ascending order, the hit with the smallest value should come first (and be surfaced).
+                orderingClause.add(new MinAggregator(new AttributeValue(fieldOrder.getFieldName())));
+                break;
+            case DESCENDING:
+                // When we sort in descending order, the hit with the largest value should come first (and be surfaced).
+                orderingClause.add(new NegFunction(new MaxAggregator(new AttributeValue(fieldOrder.getFieldName()))));
+                break;
+            default:
+                throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + ".");
+            }
+        }
+        return orderingClause;
+    }
+
+    /**
+     * Create a hit ordering clause based on the sorting spec.
+     *
+     * @param sortingSpec A (single level!) sorting specification
+     * @return a grouping expression which produces a sortable value
+     */
+    private static GroupingExpression createGroupOrderingClause(Sorting sortingSpec) {
+        GroupingExpression groupingClause = null;
+        for (Sorting.FieldOrder fieldOrder : sortingSpec.fieldOrders()) {
+            Sorting.Order sortOrder = fieldOrder.getSortOrder();
+            switch (sortOrder) {
+            case ASCENDING:
+            case UNDEFINED:
+                groupingClause = new AttributeValue(fieldOrder.getFieldName());
+                break;
+            case DESCENDING:
+                // To sort descending, just take the negative. This is the most common case
+                groupingClause = new NegFunction(new AttributeValue(fieldOrder.getFieldName()));
+                break;
+            default:
+                throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + ".");
+            }
+        }
+        return groupingClause;
+    }
+
+    /**
+     * Retrieve the actually unique hits from the grouping results.
+     *
+     * @param resultGroups the results of the dedup grouping expression.
+     * @param offset       the requested offset. Hits before this are discarded.
+     * @param hits         the requested number of hits. Hits in excess of this are discarded.
+     * @return A list of the actually requested hits, sorted as by the grouping expression.
+     */
+    private static List<Hit> getRequestedHits(GroupList resultGroups, int offset, int hits) {
+        List<Hit> receivedHits = getAllHitsFromGroupingResult(resultGroups);
+        if (receivedHits.size() <= offset) {
+            return Collections.emptyList(); // There weren't any hits as far out as requested.
+        }
+        int lastRequestedHit = Math.min(offset + hits, receivedHits.size());
+        return receivedHits.subList(offset, lastRequestedHit);
+    }
+
+    /**
+     * Get all the hits returned by the grouping request. This might be more or less than the user requested.
+     * This method handles the results from two different types of grouping expression, depending on whether
+     * sorting was used for the query or not.
+     *
+     * @param resultGroups The result group of the dedup grouping request
+     * @return A (correctly sorted) list of all the hits returned by the grouping expression.
+     */
+    private static List<Hit> getAllHitsFromGroupingResult(GroupList resultGroups) {
+        List<Hit> hits = new ArrayList<>(resultGroups.size());
+        for (Hit groupHit : resultGroups) {
+            Group group = (Group)groupHit;
+            GroupList sorted = group.getGroupList(LABEL_GROUPS);
+            if (sorted != null) {
+                group = (Group)sorted.iterator().next();
+            }
+            for (Hit hit : group.getHitList(LABEL_HITS)) {
+                hits.add(hit);
+            }
+        }
+        return hits;
+    }
+
+    static GroupingOperation buildGroupingExpression(String dedupField, int groupingHits, String summaryClass,
+                                                     Sorting sortSpec) {
+        if (sortSpec != null) {
+            return buildGroupingExpressionWithSorting(dedupField, groupingHits, summaryClass, sortSpec);
+        } else {
+            return buildGroupingExpressionWithRanking(dedupField, groupingHits, summaryClass);
+        }
+    }
+
+    /**
+     * Create the grouping expression when ranking is used for ordering
+     * (which is the default for grouping expressions, so ranking is not explicitly mentioned).
+     * See unit test for examples
+     */
+    private static GroupingOperation buildGroupingExpressionWithRanking(String dedupField, int groupingHits,
+                                                                        String summaryClass) {
+        return new AllOperation()
+            .setGroupBy(new AttributeValue(dedupField))
+            .addOutput(new CountAggregator().setLabel(LABEL_COUNT))
+            .setMax(groupingHits)
+            .addChild(new EachOperation()
+                .setMax(1)
+                .addChild(new EachOperation()
+                    .setLabel(LABEL_HITS)
+                    .addOutput(summaryClass == null ? new SummaryValue() : new SummaryValue(summaryClass))));
+    }
+
+    /**
+     * Create the grouping expression when sorting is used for ordering
+     * This grouping expression is more complicated and probably quite a bit heavier to execute.
+     * See unit test for examples
+     */
+    private static GroupingOperation buildGroupingExpressionWithSorting(String dedupField, int groupingHits,
+                                                                        String summaryClass, Sorting sortSpec) {
+        return new AllOperation()
+            .setGroupBy(new AttributeValue(dedupField))
+            .addOutput(new CountAggregator().setLabel(LABEL_COUNT))
+            .setMax(groupingHits)
+            .addOrderBy(createHitOrderingClause(sortSpec))
+            .addChild(new EachOperation()
+                .addChild(new AllOperation()
+                    .setGroupBy(createGroupOrderingClause(sortSpec))
+                    .addOrderBy(createHitOrderingClause(sortSpec))
+                    .setMax(1)
+                    .addChild(new EachOperation()
+                        .setLabel(LABEL_GROUPS)
+                        .addChild(new EachOperation()
+                            .setLabel(LABEL_HITS)
+                            .addOutput(summaryClass == null ? new SummaryValue() : new SummaryValue(summaryClass))))));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/package-info.java b/container-search/src/main/java/com/yahoo/search/grouping/package-info.java
new file mode 100644
index 00000000000..f569115008a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.grouping;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java
new file mode 100644
index 00000000000..2f321a5854d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AddFunction.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents an add-function in a {@link GroupingExpression}. It evaluates to a number that equals the
+ * result of adding the results of all arguments together in the order they were given to the constructor.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AddFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public AddFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private AddFunction(List<GroupingExpression> args) {
+        super("add", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static AddFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new AddFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AggregatorNode.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AggregatorNode.java
new file mode 100644
index 00000000000..0df204506c1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AggregatorNode.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an aggregated value in a {@link GroupingExpression}. Because it operates on a list of data, it
+ * can not be used as a document-level expression (i.e. level 0, see {@link GroupingExpression#resolveLevel(int)}). The
+ * contained expression is evaluated at the level of the aggregator minus 1.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class AggregatorNode extends GroupingExpression {
+
+    private final GroupingExpression exp;
+
+    protected AggregatorNode(String image) {
+        super(image + "()");
+        this.exp = null;
+    }
+
+    protected AggregatorNode(String image, GroupingExpression exp) {
+        super(image + "(" + exp.toString() + ")");
+        this.exp = exp;
+    }
+
+    /**
+     * Returns the expression that this node aggregates on.
+     *
+     * @return The expression.
+     */
+    public GroupingExpression getExpression() {
+        return exp;
+    }
+
+    @Override
+    public void resolveLevel(int level) {
+        super.resolveLevel(level);
+        if (level < 1) {
+            throw new IllegalArgumentException("Expression '" + this + "' not applicable for " +
+                                               GroupingOperation.getLevelDesc(level) + ".");
+        }
+        if (exp != null) {
+            exp.resolveLevel(level - 1);
+        }
+    }
+
+    @Override
+    public void visit(ExpressionVisitor visitor) {
+        super.visit(visitor);
+        if (exp != null) {
+            exp.visit(visitor);
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AllOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AllOperation.java
new file mode 100644
index 00000000000..e78be0c1c1a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AllOperation.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This is a grouping operation that processes the input list as a whole, as opposed to {@link EachOperation} which
+ * processes each element of that list separately.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AllOperation extends GroupingOperation {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public AllOperation() {
+        super("all");
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AndFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AndFunction.java
new file mode 100644
index 00000000000..3053153e5a3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AndFunction.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents an and-function in a {@link GroupingExpression}. It evaluates to a long that equals the result
+ * of and'ing the results of all arguments together in the order they were given to the constructor.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AndFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a long.
+     * @param arg2 The second compulsory argument, must evaluate to a long.
+     * @param argN The optional arguments, must evaluate to a long.
+     */
+    public AndFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private AndFunction(List<GroupingExpression> args) {
+        super("and", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static AndFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new AndFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ArrayAtLookup.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ArrayAtLookup.java
new file mode 100644
index 00000000000..1e613066bd4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ArrayAtLookup.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import com.google.common.annotations.Beta;
+
+/**
+ * Represents access of array element in a document attribute in a {@link GroupingExpression}.
+ *
+ * The first argument should be the name of an array attribute in the
+ * input {@link com.yahoo.search.result.Hit}, while the second
+ * argument is evaluated as an integer and used as the index in that array.
+ * If the index argument is less than 0 returns the first array element;
+ * if the index is greater than or equal to size(array) returns the last array element;
+ * if the array is empty returns 0 (or NaN?).
+ * @author arnej27959
+ */
+@Beta
+public class ArrayAtLookup extends DocumentValue {
+
+    private final String attributeName;
+    private final GroupingExpression arg2;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param attributeName The attribute name to assign to this.
+     */
+    public ArrayAtLookup(String attributeName, GroupingExpression indexArg) {
+        super("array.at(" + attributeName + ", " + indexArg + ")");
+        this.attributeName = attributeName;
+	this.arg2 = indexArg;
+    }
+
+    /**
+     * Returns the name of the attribute to retrieve from the input hit.
+     *
+     * @return The attribute name.
+     */
+    public String getAttributeName() {
+        return attributeName;
+    }
+
+    /**
+     * get the expression to evaluate before indexing
+     * @return grouping expression argument
+     */
+    public GroupingExpression getIndexArgument() {
+	return arg2;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeFunction.java
new file mode 100644
index 00000000000..c16903ddca8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeFunction.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a document attribute function in a {@link GroupingExpression}. It evaluates to the value of the
+ * named attribute in the input {@link com.yahoo.search.result.Hit}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AttributeFunction extends DocumentValue {
+
+    private final String name;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param attributeName The attribute name to assign to this.
+     */
+    public AttributeFunction(String attributeName) {
+        super("attribute(" + attributeName + ")");
+        name = attributeName;
+    }
+
+    /**
+     * Returns the name of the attribute to retrieve from the input hit.
+     *
+     * @return The attribute name.
+     */
+    public String getAttributeName() {
+        return name;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeValue.java
new file mode 100644
index 00000000000..135463bf108
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AttributeValue.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a document attribute value in a {@link GroupingExpression}. It evaluates to the value of the
+ * named attribute in the input {@link com.yahoo.search.result.Hit}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AttributeValue extends DocumentValue {
+
+    private final String name;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param attributeName The attribute name to assign to this.
+     */
+    public AttributeValue(String attributeName) {
+        super(attributeName);
+        name = attributeName;
+    }
+
+    /**
+     * Returns the name of the attribute to retrieve from the input hit.
+     *
+     * @return The attribute name.
+     */
+    public String getAttributeName() {
+        return name;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AvgAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgAggregator.java
new file mode 100644
index 00000000000..749b419488f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgAggregator.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an average-aggregator in a {@link GroupingExpression}. It evaluates to the average value that
+ * the contained expression evaluated to over all the inputs.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AvgAggregator extends AggregatorNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to aggregate on.
+     */
+    public AvgAggregator(GroupingExpression exp) {
+        super("avg", exp);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/AvgFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgFunction.java
new file mode 100644
index 00000000000..c0474064741
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/AvgFunction.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a min-function in a {@link GroupingExpression}. It evaluates to a number that equals the
+ * average of the results of all arguments.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class AvgFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public AvgFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private AvgFunction(List<GroupingExpression> args) {
+        super("avg", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static AvgFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new AvgFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/BooleanValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/BooleanValue.java
new file mode 100644
index 00000000000..c41cfa4c4f2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/BooleanValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a constant {@link Boolean} value in a {@link GroupingExpression}.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class BooleanValue extends ConstantValue<Boolean> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The immutable value to assign to this.
+     */
+    public BooleanValue(Boolean value) {
+        super(value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java
new file mode 100644
index 00000000000..735347cde87
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketResolver.java
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * This is a helper class for resolving buckets to a list of
+ * {@link GroupingExpression} objects. To resolve a list simply
+ * {@link #push(ConstantValue, boolean)} onto it, before calling
+ * {@link #resolve(GroupingExpression)} to retrieve the list of corresponding
+ * grouping expression object.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class BucketResolver {
+
+    private final List<BucketValue> buckets = new LinkedList<>();
+    private ConstantValue<?> prev = null;
+    private boolean previnclusive = false;
+    private int idx = 0;
+
+    /**
+     * Pushes the given expression onto this bucket resolver. Once all buckets have been pushed using this method, call
+     * {@link #resolve(GroupingExpression)} to retrieve to combined grouping expression.
+     *
+     * @param val The expression to push.
+     * @param inclusive Whether or not the value is inclusive or not.
+     * @throws IllegalArgumentException Thrown if the expression is incompatible.
+     */
+    public BucketResolver push(ConstantValue<?> val, boolean inclusive) {
+        if (prev == null) {
+            prev = val;
+        } else if (!(prev instanceof InfiniteValue || val instanceof InfiniteValue)
+                   && !prev.getClass().equals(val.getClass())) {
+            throw new IllegalArgumentException("Bucket type mismatch, expected '" + prev.getClass().getSimpleName() +
+                                               "' got '" + val.getClass().getSimpleName() + "'.");
+        } else if (prev instanceof InfiniteValue && val instanceof InfiniteValue) {
+            throw new IllegalArgumentException("Bucket type mismatch, cannot both be infinity.");
+        }
+        if ((++idx % 2) == 0) {
+            ConstantValue<?> begin = previnclusive ? prev : nextValue(prev);
+            ConstantValue<?> end = inclusive ? nextValue(val) : val;
+            if (begin instanceof DoubleValue || end instanceof DoubleValue) {
+                buckets.add(new DoubleBucket(begin, end));
+            } else if (begin instanceof LongValue || end instanceof LongValue) {
+                buckets.add(new LongBucket(begin, end));
+            } else if (begin instanceof StringValue || end instanceof StringValue) {
+                buckets.add(new StringBucket(begin, end));
+            } else if (begin instanceof RawValue || end instanceof RawValue) {
+                buckets.add(new RawBucket(begin, end));
+            } else {
+                throw new UnsupportedOperationException("Bucket type '" + val.getClass() + "' not supported.");
+            }
+        }
+        prev = val;
+        previnclusive = inclusive;
+        return this;
+    }
+
+    /**
+     * Resolves and returns the list of grouping expressions that correspond to the previously pushed buckets.
+     *
+     * @param exp The expression to assign to the function.
+     * @return The list corresponding to the pushed buckets.
+     */
+    public PredefinedFunction resolve(GroupingExpression exp) {
+        if ((idx % 2) == 1) {
+            throw new IllegalStateException("Missing to-limit of last bucket.");
+        }
+        int len = buckets.size();
+        if (len == 0) {
+            throw new IllegalStateException("Expected at least one bucket, got none.");
+        }
+        ConstantValue<?> begin = buckets.get(0).getFrom();
+        ConstantValue<?> end = buckets.get(0).getTo();
+        if (begin instanceof DoubleValue || end instanceof DoubleValue) {
+            if (len == 1) {
+                return new DoublePredefined(exp, (DoubleBucket)buckets.get(0));
+            } else {
+                return new DoublePredefined(exp, (DoubleBucket)buckets.get(0),
+                                            buckets.subList(1, len).toArray(new DoubleBucket[len - 1]));
+            }
+        } else if (begin instanceof LongValue || end instanceof LongValue) {
+            if (len == 1) {
+                return new LongPredefined(exp, (LongBucket)buckets.get(0));
+            } else {
+                return new LongPredefined(exp, (LongBucket)buckets.get(0),
+                                          buckets.subList(1, len).toArray(new LongBucket[len - 1]));
+            }
+        } else if (begin instanceof StringValue || end instanceof StringValue) {
+            if (len == 1) {
+                return new StringPredefined(exp, (StringBucket)buckets.get(0));
+            } else {
+                return new StringPredefined(exp, (StringBucket)buckets.get(0),
+                                            buckets.subList(1, len).toArray(new StringBucket[len - 1]));
+            }
+        } else if (begin instanceof RawValue || end instanceof RawValue) {
+            if (len == 1) {
+                return new RawPredefined(exp, (RawBucket)buckets.get(0));
+            } else {
+                return new RawPredefined(exp, (RawBucket)buckets.get(0),
+                                            buckets.subList(1, len).toArray(new RawBucket[len - 1]));
+            }
+        }
+        throw new UnsupportedOperationException("Bucket type '" + begin.getClass() + "' not supported.");
+    }
+
+    private ConstantValue<?> nextValue(ConstantValue<?> value) {
+        if (value instanceof LongValue) {
+            return LongBucket.nextValue((LongValue)value);
+        } else if (value instanceof DoubleValue) {
+            return DoubleBucket.nextValue((DoubleValue)value);
+        } else if (value instanceof StringValue) {
+            return StringBucket.nextValue((StringValue)value);
+        } else if (value instanceof RawValue) {
+            return RawBucket.nextValue((RawValue)value);
+        }
+        return value;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/BucketValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketValue.java
new file mode 100644
index 00000000000..858a44e2fe8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/BucketValue.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a bucket in a {@link PredefinedFunction}. The generic T is the data type of the range values
+ * 'from' and 'to'. The range is inclusive-from and exclusive-to. All supported data types are represented as subclasses
+ * of this.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class BucketValue extends GroupingExpression implements Comparable<BucketValue> {
+
+    private final ConstantValue<?> from;
+    private final ConstantValue<?> to;
+    private final ConstantValueComparator comparator = new ConstantValueComparator();
+
+    protected BucketValue(ConstantValue<?> inclusiveFrom, ConstantValue<?> exclusiveTo) {
+        super("bucket[" + asImage(inclusiveFrom) + ", " + asImage(exclusiveTo) + ">");
+        if (comparator.compare(exclusiveTo, inclusiveFrom) < 0) {
+            throw new IllegalArgumentException("Bucket to-value can not be less than from-value.");
+        }
+        from = inclusiveFrom;
+        to = exclusiveTo;
+    }
+
+    /**
+     * Returns the inclusive-from value of this bucket.
+     *
+     * @return The from-value.
+     */
+    public ConstantValue<?> getFrom() {
+        return from;
+    }
+
+    /**
+     * Returns the exclusive-to value of this bucket.
+     *
+     * @return The to-value.
+     */
+    public ConstantValue<?> getTo() {
+        return to;
+    }
+
+    @Override
+    public int compareTo(BucketValue rhs) {
+        if (comparator.compare(to, rhs.from) <= 0) {
+            return -1;
+        }
+        if (comparator.compare(from, rhs.to) >= 0) {
+            return 1;
+        }
+        return 0;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/CatFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/CatFunction.java
new file mode 100644
index 00000000000..9bc276bda92
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/CatFunction.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a cat-function in a {@link GroupingExpression}. It evaluates to a byte array that equals the
+ * concatenation of the binary result of all arguments in the order they were given to the constructor.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class CatFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument.
+     * @param arg2 The second compulsory argument.
+     * @param argN The optional arguments.
+     */
+    public CatFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private CatFunction(List<GroupingExpression> args) {
+        super("cat", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static CatFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new CatFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValue.java
new file mode 100644
index 00000000000..8b8d92b5ae8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValue.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a constant value in a {@link GroupingExpression}. Because it does not operate on any input,
+ * this expression type can be used at any input level (see {@link GroupingExpression#resolveLevel(int)}). All supported
+ * data types are represented as subclasses of this.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@SuppressWarnings("rawtypes")
+public abstract class ConstantValue<T extends Comparable> extends GroupingExpression {
+
+    private final T value;
+
+    protected ConstantValue(T value) {
+        super(asImage(value));
+        this.value = value;
+    }
+
+    /**
+     * Returns the constant value of this.
+     *
+     * @return The value.
+     */
+    public T getValue() {
+        return value;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValueComparator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValueComparator.java
new file mode 100644
index 00000000000..e8017bbb796
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ConstantValueComparator.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Comparator;
+
+/**
+ * This class compares two constant values, and takes into account that one of
+ * the arguments may be the very special infinity value.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+@SuppressWarnings("rawtypes")
+public class ConstantValueComparator implements Comparator<ConstantValue> {
+    @SuppressWarnings("unchecked")
+    @Override
+    public int compare(ConstantValue lhs, ConstantValue rhs) {
+        // Run infinite comparison method if one of the arguments are infinite.
+        if (rhs instanceof InfiniteValue) {
+            return (-1 * rhs.getValue().compareTo(lhs));
+        }
+        return (lhs.getValue().compareTo(rhs.getValue()));
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/CountAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/CountAggregator.java
new file mode 100644
index 00000000000..f54d92cdbf5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/CountAggregator.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an count-aggregator in a {@link GroupingExpression}. It evaluates to the number of elements
+ * there are in the input.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class CountAggregator extends AggregatorNode {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public CountAggregator() {
+        super("count");
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java
new file mode 100644
index 00000000000..3d416b31d95
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DateFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a timestamp-formatter function in a {@link GroupingExpression}. It evaluates to a string on the
+ * form "YYYY-MM-DD" of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DateFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public DateFunction(GroupingExpression exp) {
+        super("time.date", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java
new file mode 100644
index 00000000000..4ead68cc8f1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfMonthFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a day-of-month timestamp-function in a {@link GroupingExpression}. It evaluates to a long that
+ * equals the day of month (1-31) of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DayOfMonthFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public DayOfMonthFunction(GroupingExpression exp) {
+        super("time.dayofmonth", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java
new file mode 100644
index 00000000000..f91344e2e7b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfWeekFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a day-of-week timestamp-function in a {@link GroupingExpression}. It evaluates to a long that
+ * equals the day of week (0 - 6) of the result of the argument, Monday being 0.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DayOfWeekFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public DayOfWeekFunction(GroupingExpression exp) {
+        super("time.dayofweek", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java
new file mode 100644
index 00000000000..20313864493
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DayOfYearFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a day-of-year timestamp-function in a {@link GroupingExpression}. It evaluates to a long that
+ * equals the day of year (0-365) of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DayOfYearFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public DayOfYearFunction(GroupingExpression exp) {
+        super("time.dayofyear", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java
new file mode 100644
index 00000000000..c2f26e6b3b0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DebugWaitFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents debug_wait function in a {@link GroupingExpression}. For each hit evaluated,
+ * it waits for the time specified as the second argument. The third argument specifies if the wait
+ * should be a busy-wait or not. The first argument is then evaluated.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class DebugWaitFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, the expression to proxy.
+     * @param arg2 The second compulsory argument, must evaluate to a positive number.
+     * @param arg3 The third compulsory argument, specifying busy wait or not.
+     */
+    public DebugWaitFunction(GroupingExpression arg1, DoubleValue arg2, BooleanValue arg3) {
+        super("debugwait", Arrays.asList(arg1, arg2, arg3));
+    }
+
+    /**
+     * Returns the time to wait when evaluating this function.
+     *
+     * @return the number of seconds to wait.
+     */
+    public double getWaitTime() {
+        return ((DoubleValue)getArg(1)).getValue();
+    }
+
+    /**
+     * Returns whether or not the debug node should busy-wait.
+     *
+     * @return true if busy-wait, false if not.
+     */
+    public boolean getBusyWait() {
+        return ((BooleanValue)getArg(2)).getValue();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DivFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DivFunction.java
new file mode 100644
index 00000000000..9ed263362fa
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DivFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a div-function in a {@link GroupingExpression}. It evaluates to a number that equals the result
+ * of dividing the results of all arguments in the order they were given to the constructor (divide first argument by
+ * second, result by third, ...).
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DivFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public DivFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private DivFunction(List<GroupingExpression> args) {
+        super("div", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static DivFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new DivFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java
new file mode 100644
index 00000000000..02c8d66be5d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DocIdNsSpecificValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a document id specific value in a {@link GroupingExpression}. It evaluates to the namespace-
+ * specific value of the document id of the input {@link com.yahoo.search.result.Hit}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DocIdNsSpecificValue extends DocumentValue {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public DocIdNsSpecificValue() {
+        super("docidnsspecific()");
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DocumentValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DocumentValue.java
new file mode 100644
index 00000000000..98d5a6fe21f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DocumentValue.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a document value in a {@link GroupingExpression}. As such, the subclasses of this can only be
+ * used as document-level expressions (i.e. level 0, see {@link GroupingExpression#resolveLevel(int)}).
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class DocumentValue extends GroupingExpression {
+
+    protected DocumentValue(String image) {
+        super(image);
+    }
+
+    @Override
+    public void resolveLevel(int level) {
+        if (level != 0) {
+            throw new IllegalArgumentException("Expression '" + this + "' not applicable for " +
+                                               GroupingOperation.getLevelDesc(level) + ".");
+        }
+        super.resolveLevel(level);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleBucket.java
new file mode 100644
index 00000000000..4e12e96272e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleBucket.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+import java.text.ChoiceFormat;
+
+/**
+ * This class represents a {@link Double} bucket in a {@link PredefinedFunction}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DoubleBucket extends BucketValue {
+
+    /**
+     * Returns the next distinct value.
+     *
+     * @param value The base value.
+     * @return the next value.
+     */
+    public static DoubleValue nextValue(DoubleValue value) {
+        return (new DoubleValue(ChoiceFormat.nextDouble(value.getValue())));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    public DoubleBucket(double from, double to) {
+        super(new DoubleValue(from), new DoubleValue(to));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    public DoubleBucket(ConstantValue<?> from, ConstantValue<?> to) {
+        super(from, to);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DoublePredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DoublePredefined.java
new file mode 100644
index 00000000000..59265359715
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DoublePredefined.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a
+ * double.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DoublePredefined extends PredefinedFunction {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a double.
+     * @param arg1 The compulsory bucket.
+     * @param argN The optional buckets.
+     */
+    public DoublePredefined(GroupingExpression exp, DoubleBucket arg1, DoubleBucket... argN) {
+        this(exp, asList(arg1, argN));
+    }
+
+    private DoublePredefined(GroupingExpression exp, List<DoubleBucket> args) {
+        super(exp, args);
+    }
+
+    @Override
+    public DoubleBucket getBucket(int i) {
+        return (DoubleBucket)getArg(i + 1);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a double.
+     * @param args The buckets to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the list of buckets is empty.
+     */
+    public static DoublePredefined newInstance(GroupingExpression exp, List<DoubleBucket> args) {
+        if (args.isEmpty()) {
+            throw new IllegalArgumentException("Expected at least one bucket, got none.");
+        }
+        return new DoublePredefined(exp, args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleValue.java
new file mode 100644
index 00000000000..682102533ff
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/DoubleValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a constant {@link Double} value in a {@link GroupingExpression}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DoubleValue extends ConstantValue<Double> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The immutable value to assign to this.
+     */
+    public DoubleValue(double value) {
+        super(value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/EachOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/EachOperation.java
new file mode 100644
index 00000000000..12f6df1f497
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/EachOperation.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This is a grouping operation that processes each element of the input list separately, as opposed to {@link
+ * AllOperation} which processes that list as a whole.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class EachOperation extends GroupingOperation {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public EachOperation() {
+        super("each");
+    }
+
+    @Override
+    public void resolveLevel(int level) {
+        if (level == 0) {
+            throw new IllegalArgumentException("Operation '" + this + "' can not operate on " + getLevelDesc(level) + ".");
+        }
+        super.resolveLevel(level - 1);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ExpressionVisitor.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ExpressionVisitor.java
new file mode 100644
index 00000000000..ba411ac45ce
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ExpressionVisitor.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This interface defines the necessary callback to recursively visit all {@link GroupingExpression} objects in a {@link
+ * GroupingOperation}. It is used by the {@link com.yahoo.search.grouping.GroupingValidator} to ensure that all
+ * referenced attributes are valid for the cluster being queried.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface ExpressionVisitor {
+
+    /**
+     * This method is called for every {@link GroupingExpression} object in the targeted {@link GroupingOperation}.
+     *
+     * @param exp The expression being visited.
+     */
+    public void visitExpression(GroupingExpression exp);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java
new file mode 100644
index 00000000000..9ac3870718b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/FixedWidthFunction.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a fixed-width bucket-function in a {@link GroupingExpression}. It maps the input into the given
+ * number of buckets by the result of the argument expression.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class FixedWidthFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp        The expression to evaluate.
+     * @param width The width of each bucket.
+     */
+    public FixedWidthFunction(GroupingExpression exp, Number width) {
+        super("fixedwidth", Arrays.asList(exp, width instanceof Double ? new DoubleValue(width.doubleValue()) : new LongValue(width.longValue())));
+    }
+
+    /**
+     * Returns the number of buckets to divide the result into.
+     *
+     * @return The bucket count.
+     */
+    public Number getWidth() {
+        GroupingExpression w = getArg(1);
+        return (w instanceof LongValue) ? ((LongValue)w).getValue() : ((DoubleValue)w).getValue();
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java b/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java
new file mode 100644
index 00000000000..3003ce69abe
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/FunctionNode.java
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.*;
+
+/**
+ * This class represents a function in a {@link GroupingExpression}. Because it operate on other expressions (as opposed
+ * to {@link AggregatorNode} and {@link DocumentValue} that operate on inputs), this expression type can be used at any
+ * input level (see {@link GroupingExpression#resolveLevel(int)}).
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class FunctionNode extends GroupingExpression implements Iterable<GroupingExpression> {
+
+    private final List<GroupingExpression> args = new ArrayList<>();
+
+    protected FunctionNode(String image, List<GroupingExpression> args) {
+        super(image + "(" + asString(args) + ")");
+        this.args.addAll(args);
+    }
+
+    /**
+     * Returns the number of arguments that were given to this function at construction.
+     *
+     * @return The argument count.
+     */
+    public int getNumArgs() {
+        return args.size();
+    }
+
+    /**
+     * Returns the argument at the given index.
+     *
+     * @param i The index of the argument to return.
+     * @return The argument at the given index.
+     * @throws IndexOutOfBoundsException If the index is out of range.
+     */
+    public GroupingExpression getArg(int i) {
+        return args.get(i);
+    }
+
+    @Override
+    public Iterator<GroupingExpression> iterator() {
+        return Collections.unmodifiableList(args).iterator();
+    }
+
+    @Override
+    public void resolveLevel(int level) {
+        super.resolveLevel(level);
+        for (GroupingExpression arg : args) {
+            arg.resolveLevel(level);
+        }
+    }
+
+    @Override
+    public void visit(ExpressionVisitor visitor) {
+        super.visit(visitor);
+        for (GroupingExpression arg : args) {
+            arg.visit(visitor);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    protected static <T> List<T> asList(T arg1, T... argN) {
+        return asList(Arrays.asList(arg1), Arrays.asList(argN));
+    }
+
+    @SuppressWarnings("unchecked")
+    protected static <T> List<T> asList(T arg1, T arg2, T... argN) {
+        return asList(Arrays.asList(arg1, arg2), Arrays.asList(argN));
+    }
+
+    protected static <T> List<T> asList(List<T> foo, List<T> bar) {
+        List<T> ret = new LinkedList<>(foo);
+        ret.addAll(bar);
+        return ret;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingExpression.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingExpression.java
new file mode 100644
index 00000000000..6015557f81e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingExpression.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import com.yahoo.javacc.UnicodeUtilities;
+
+import java.util.List;
+
+/**
+ * This class represents an expression in a {@link GroupingOperation}. You may manually construct this expression, or
+ * you may use the {@link com.yahoo.search.grouping.request.parser.GroupingParser} to generate one from a query-string.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class GroupingExpression extends GroupingNode {
+
+    private Integer level = null;
+
+    protected GroupingExpression(String image) {
+        super(image);
+    }
+
+    /**
+     * Resolves the conceptual level of this expression. This level represents the type of data that is consumed by this
+     * expression, where level 0 is a single hit, level 1 is a group, level 2 is a list of groups, and so forth. This
+     * method verifies the input level against the expression type, and recursively resolves the level of all argument
+     * expressions.
+     *
+     * @param level The level of the input data.
+     * @throws IllegalArgumentException Thrown if the level of this expression could not be resolved.
+     * @throws IllegalStateException    Thrown if type failed to accept the number of arguments provided.
+     */
+    public void resolveLevel(int level) {
+        if (level < 0) {
+            throw new IllegalArgumentException("Expression '" + this + "' recurses through a single hit.");
+        }
+        this.level = level;
+    }
+
+    /**
+     * Returns the conceptual level of this expression.
+     *
+     * @return The level.
+     * @throws IllegalArgumentException Thrown if the level of this expression has not been resolved.
+     * @see #resolveLevel(int)
+     */
+    public int getLevel() {
+        if (level == null) {
+            throw new IllegalStateException("Level for expression '" + this + "' has not been resolved.");
+        }
+        return level;
+    }
+
+    /**
+     * Recursively calls {@link ExpressionVisitor#visitExpression(GroupingExpression)} for this expression and all of
+     * its argument expressions.
+     *
+     * @param visitor The visitor to call.
+     */
+    public void visit(ExpressionVisitor visitor) {
+        visitor.visitExpression(this);
+    }
+
+    /**
+     * Returns a string description of the given list of expressions. This is a comma-separated list of the expressions
+     * own {@link GroupingExpression#toString()} output.
+     *
+     * @param lst The list of expressions to output.
+     * @return The string description.
+     */
+    public static String asString(List<GroupingExpression> lst) {
+        StringBuilder ret = new StringBuilder();
+        for (int i = 0, len = lst.size(); i < len; ++i) {
+            ret.append(lst.get(i));
+            if (i < len - 1) {
+                ret.append(", ");
+            }
+        }
+        return ret.toString();
+    }
+
+    /**
+     * Returns a string representation of an object that can be used in the 'image' constructor argument of {@link
+     * GroupingNode}. This method ensures that strings are quoted, and that all complex characters are escaped.
+     *
+     * @param obj The object to output.
+     * @return The string representation.
+     */
+    public static String asImage(Object obj) {
+        if (!(obj instanceof String)) {
+            return obj.toString();
+        }
+        return UnicodeUtilities.quote((String)obj, '"');
+    }
+
+    @Override
+    public GroupingExpression setLabel(String label) {
+        super.setLabel(label);
+        return this;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingNode.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingNode.java
new file mode 100644
index 00000000000..b400dfe5737
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingNode.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This is the abstract super class of both {@link GroupingOperation} and {@link GroupingExpression}. All nodes can be
+ * assigned a {@link String} label which in turn can be used to identify the corresponding result objects.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class GroupingNode {
+
+    private final String image;
+    private String label = null;
+
+    protected GroupingNode(String image) {
+        this.image = image;
+    }
+
+    /**
+     * Returns the label assigned to this grouping expression.
+     *
+     * @return The label string.
+     */
+    public String getLabel() {
+        return label;
+    }
+
+    /**
+     * Assigns a label to this grouping expression. The label is applied to the results of this expression so that they
+     * can be identified by the caller when processing the output.
+     *
+     * @param str The label to assign to this.
+     * @return This, to allow chaining.
+     */
+    public GroupingNode setLabel(String str) {
+        label = str;
+        return this;
+    }
+
+    @Override
+    public String toString() {
+        return image;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java
new file mode 100644
index 00000000000..d49713ba9f2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/GroupingOperation.java
@@ -0,0 +1,582 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import com.yahoo.collections.LazyMap;
+import com.yahoo.collections.LazySet;
+import com.yahoo.search.grouping.request.parser.GroupingParser;
+import com.yahoo.search.grouping.request.parser.GroupingParserInput;
+import com.yahoo.search.grouping.request.parser.ParseException;
+import com.yahoo.search.grouping.request.parser.TokenMgrError;
+
+import java.util.*;
+
+/**
+ * This class represents a single node in a grouping operation tree. You may manually construct this tree, or you may
+ * use the {@link #fromString(String)} method to generate one from a query-string. To execute, assign it to a {@link
+ * com.yahoo.search.grouping.GroupingRequest} using the {@link com.yahoo.search.grouping.GroupingRequest#setRootOperation(GroupingOperation)}
+ * method.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class GroupingOperation extends GroupingNode {
+
+    private final List<GroupingExpression> orderBy = new ArrayList<>();
+    private final List<GroupingExpression> outputs = new ArrayList<>();
+    private final List<GroupingOperation> children = new ArrayList<>();
+    private final Map<String, GroupingExpression> alias = LazyMap.newHashMap();
+    private final Set<String> hints = LazySet.newHashSet();
+
+    private GroupingExpression groupBy = null;
+    private GroupingOperation parent = null;
+    private String where = null;
+    private boolean forceSinglePass = false;
+    private double accuracy = 0.95;
+    private int precision = 0;
+    private int level = -1;
+    private int max = -1;
+
+    protected GroupingOperation(String image) {
+        super(image);
+    }
+
+    /**
+     * Registers an alias with this operation. An alias is made available to expressions in both this node and all child
+     * nodes.
+     *
+     * @param id  The id of the alias to put.
+     * @param exp The expression to associate with the id.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation putAlias(String id, GroupingExpression exp) {
+        alias.put(id, exp);
+        return this;
+    }
+
+    /**
+     * Returns the alias associated with the given name. If no alias can be found in this node, this method queries its
+     * parent grouping node. If the alias still can not be found, this method returns null.
+     *
+     * @param id The id of the alias to return.
+     * @return The expression associated with the id.
+     */
+    public GroupingExpression getAlias(String id) {
+        if (alias.containsKey(id)) {
+            return alias.get(id);
+        } else if (parent != null) {
+            return parent.getAlias(id);
+        } else {
+            return null;
+        }
+    }
+
+    /**
+     * Adds a hint to this.
+     *
+     * @param hint The hint to add.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation addHint(String hint) {
+        hints.add(hint);
+        return this;
+    }
+
+    /**
+     * Returns whether or not the given hint has been added to this.
+     *
+     * @param hint The hint to check for.
+     * @return True if the hint has been added.
+     */
+    public boolean containsHint(String hint) {
+        return hints.contains(hint);
+    }
+
+    /**
+     * Returns an immutable view to the hint list of this node.
+     *
+     * @return The list.
+     */
+    public Set<String> getHints() {
+        return Collections.unmodifiableSet(hints);
+    }
+
+    /**
+     * Adds a child grouping node to this. This will also set the parent of the child so that it points to this node.
+     *
+     * @param op The child node to add.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation addChild(GroupingOperation op) {
+        op.parent = this;
+        children.add(op);
+        return this;
+    }
+
+    /**
+     * Convenience method to call {@link #addChild(GroupingOperation)} for each element in the given list.
+     *
+     * @param lst The list of operations to add.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation addChildren(List<GroupingOperation> lst) {
+        for (GroupingOperation op : lst) {
+            addChild(op);
+        }
+        return this;
+    }
+
+    /**
+     * Returns the number of child operations of this.
+     *
+     * @return The child count.
+     */
+    public int getNumChildren() {
+        return children.size();
+    }
+
+    /**
+     * Returns the child operation at the given index.
+     *
+     * @param i The index of the child to return.
+     * @return The child at the given index.
+     * @throws IndexOutOfBoundsException If the index is out of range.
+     */
+    public GroupingOperation getChild(int i) {
+        return children.get(i);
+    }
+
+    /**
+     * Returns an immutable view to the child list of this node.
+     *
+     * @return The list.
+     */
+    public List<GroupingOperation> getChildren() {
+        return Collections.unmodifiableList(children);
+    }
+
+    /**
+     * Assigns an expressions as the group-by clause of this operation.
+     *
+     * @param exp The expression to assign to this.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation setGroupBy(GroupingExpression exp) {
+        groupBy = exp;
+        return this;
+    }
+
+    /**
+     * Returns the expression assigned as the group-by clause of this.
+     *
+     * @return The expression.
+     */
+    public GroupingExpression getGroupBy() {
+        return groupBy;
+    }
+
+    /**
+     * Returns the conceptual level of this node.
+     *
+     * @return The level, or -1 if not resolved.
+     * @see #resolveLevel(int)
+     */
+    public int getLevel() {
+        return level;
+    }
+
+    /**
+     * Resolves the conceptual level of this operation. This level represents the type of data that is consumed by this
+     * operation, where level 0 is a single hit, level 1 is a group, level 2 is a list of groups, and so forth. This
+     * method verifies the input level against the operation type, and recursively resolves the level of all argument
+     * expressions.
+     *
+     * @param level The level of the input data.
+     * @throws IllegalArgumentException Thrown if a contained expression is invalid for the given level.
+     */
+    public void resolveLevel(int level) {
+        if (groupBy != null) {
+            if (level == 0) {
+                throw new IllegalArgumentException(
+                        "Operation '" + this + "' can not group " + getLevelDesc(level) + ".");
+            }
+            groupBy.resolveLevel(level - 1);
+            ++level;
+        }
+        if (hasMax()) {
+            if (level == 0) {
+                throw new IllegalArgumentException(
+                        "Operation '" + this + "' can not apply max to " + getLevelDesc(level) + ".");
+            }
+        }
+        this.level = level;
+        for (GroupingExpression exp : outputs) {
+            exp.resolveLevel(level);
+        }
+        if (!orderBy.isEmpty()) {
+            if (level == 0) {
+                throw new IllegalArgumentException(
+                        "Operation '" + this + "' can not order " + getLevelDesc(level) + ".");
+            }
+            for (GroupingExpression exp : orderBy) {
+                exp.resolveLevel(level - 1);
+            }
+        }
+        for (GroupingOperation child : children) {
+            child.resolveLevel(level);
+        }
+    }
+
+    public GroupingOperation setForceSinglePass(boolean forceSinglePass) {
+        this.forceSinglePass = forceSinglePass;
+        return this;
+    }
+
+    public boolean getForceSinglePass() {
+        return forceSinglePass;
+    }
+
+    /**
+     * Assigns the max clause of this. This is the maximum number of groups to return for this operation.
+     *
+     * @param max The expression to assign to this.
+     * @return This, to allow chaining.
+     * @see #setPrecision(int)
+     */
+    public GroupingOperation setMax(int max) {
+        this.max = max;
+        return this;
+    }
+
+    /**
+     * Returns the max clause of this.
+     *
+     * @return The expression.
+     * @see #setMax(int)
+     */
+    public int getMax() {
+        return max;
+    }
+
+    /**
+     * Indicates if the 'max' value has been set.
+     *
+     * @return true if max value is set.
+     */
+    public boolean hasMax() { return max >= 0; }
+
+    /**
+     * Assigns an accuracy value for this. This is a number between 0 and 1 describing the accuracy of the result, which
+     * again determines the speed of the grouping request. A low value will make sure the grouping operation runs fast,
+     * at the sacrifice if a (possible) imprecise result.
+     *
+     * @param accuracy The accuracy to assign to this.
+     * @return This, to allow chaining.
+     * @throws IllegalArgumentException If the accuracy is outside the allowed value range.
+     */
+    public GroupingOperation setAccuracy(double accuracy) {
+        if (accuracy > 1.0 || accuracy < 0.0) {
+            throw new IllegalArgumentException("Illegal accuracy '" + accuracy + "'. Must be between 0 and 1.");
+        }
+        this.accuracy = accuracy;
+        return this;
+    }
+
+    /**
+     * Return the accuracy of this.
+     *
+     * @return The accuracy value.
+     * @see #setAccuracy(double)
+     */
+    public double getAccuracy() {
+        return accuracy;
+    }
+
+    /**
+     * Adds an expression to the order-by clause of this operation.
+     *
+     * @param exp The expressions to add to this.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation addOrderBy(GroupingExpression exp) {
+        orderBy.add(exp);
+        return this;
+    }
+
+    /**
+     * Convenience method to call {@link #addOrderBy(GroupingExpression)} for each element in the given list.
+     *
+     * @param lst The list of expressions to add.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation addOrderBy(List<GroupingExpression> lst) {
+        for (GroupingExpression exp : lst) {
+            addOrderBy(exp);
+        }
+        return this;
+    }
+
+    /**
+     * Returns the number of expressions in the order-by clause of this.
+     *
+     * @return The expression count.
+     */
+    public int getNumOrderBy() {
+        return orderBy.size();
+    }
+
+    /**
+     * Returns the group-by expression at the given index.
+     *
+     * @param i The index of the expression to return.
+     * @return The expression at the given index.
+     * @throws IndexOutOfBoundsException If the index is out of range.
+     */
+    public GroupingExpression getOrderBy(int i) {
+        return orderBy.get(i);
+    }
+
+    /**
+     * Returns an immutable view to the order-by clause of this.
+     *
+     * @return The expression list.
+     */
+    public List<GroupingExpression> getOrderBy() {
+        return Collections.unmodifiableList(orderBy);
+    }
+
+    /**
+     * Adds an expression to the output clause of this operation.
+     *
+     * @param exp The expressions to add to this.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation addOutput(GroupingExpression exp) {
+        outputs.add(exp);
+        return this;
+    }
+
+    /**
+     * Convenience method to call {@link #addOutput(GroupingExpression)} for each element in the given list.
+     *
+     * @param lst The list of expressions to add.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation addOutputs(List<GroupingExpression> lst) {
+        for (GroupingExpression exp : lst) {
+            addOutput(exp);
+        }
+        return this;
+    }
+
+    /**
+     * Returns the number of expressions in the output clause of this.
+     *
+     * @return The expression count.
+     */
+    public int getNumOutputs() {
+        return outputs.size();
+    }
+
+    /**
+     * Returns the output expression at the given index.
+     *
+     * @param i The index of the expression to return.
+     * @return The expression at the given index.
+     * @throws IndexOutOfBoundsException If the index is out of range.
+     */
+    public GroupingExpression getOutput(int i) {
+        return outputs.get(i);
+    }
+
+    /**
+     * Returns an immutable view to the output clause of this.
+     *
+     * @return The expression list.
+     */
+    public List<GroupingExpression> getOutputs() {
+        return Collections.unmodifiableList(outputs);
+    }
+
+    /**
+     * Assigns the precision clause of this. This is the number of intermediate groups returned from each search-node
+     * during expression evaluation to give the dispatch-node more data to consider when selecting the N groups that are
+     * to be evaluated further.
+     *
+     * @param precision The precision to set.
+     * @return This, to allow chaining.
+     * @see #setMax(int)
+     */
+    public GroupingOperation setPrecision(int precision) {
+        this.precision = precision;
+        return this;
+    }
+
+    /**
+     * Returns the precision clause of this.
+     *
+     * @return The precision.
+     */
+    public int getPrecision() {
+        return precision;
+    }
+
+    /**
+     * Assigns a string as the where clause of this operation.
+     *
+     * @param str The string to assign to this.
+     * @return This, to allow chaining.
+     */
+    public GroupingOperation setWhere(String str) {
+        where = str;
+        return this;
+    }
+
+    /**
+     * Returns the where clause assigned to this operation.
+     *
+     * @return The where clause.
+     */
+    public String getWhere() {
+        return where;
+    }
+
+    /**
+     * Recursively calls {@link GroupingExpression#visit(ExpressionVisitor)} on all {@link GroupingExpression} objects
+     * in this operation and in all of its child operations.
+     *
+     * @param visitor The visitor to call.
+     */
+    public void visitExpressions(ExpressionVisitor visitor) {
+        for (GroupingExpression exp : alias.values()) {
+            exp.visit(visitor);
+        }
+        for (GroupingExpression exp : outputs) {
+            exp.visit(visitor);
+        }
+        for (GroupingExpression exp : orderBy) {
+            exp.visit(visitor);
+        }
+        if (groupBy != null) {
+            groupBy.visit(visitor);
+        }
+        for (GroupingOperation op : children) {
+            op.visitExpressions(visitor);
+        }
+    }
+
+    @Override
+    public GroupingOperation setLabel(String label) {
+        super.setLabel(label);
+        return this;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append(super.toString()).append("(");
+        if (groupBy != null) {
+            ret.append("group(").append(groupBy).append(") ");
+        }
+        for (String hint : hints) {
+            ret.append("hint(").append(hint).append(") ");
+        }
+        if (hasMax()) {
+            ret.append("max(").append(max).append(") ");
+        }
+        if (!orderBy.isEmpty()) {
+            ret.append("order(");
+            ret.append(GroupingExpression.asString(orderBy));
+            ret.append(") ");
+        }
+        if (!outputs.isEmpty()) {
+            ret.append("output(");
+            for (int i = 0, len = outputs.size(); i < len; ++i) {
+                GroupingExpression exp = outputs.get(i);
+                ret.append(exp);
+                String label = exp.getLabel();
+                if (label != null) {
+                    ret.append(" as(").append(label).append(")");
+                }
+                if (i < len - 1) {
+                    ret.append(", ");
+                }
+            }
+            ret.append(") ");
+        }
+        if (precision != 0) {
+            ret.append("precision(").append(precision).append(") ");
+        }
+        if (where != null) {
+            ret.append("where(").append(where).append(") ");
+        }
+        for (GroupingOperation child : children) {
+            ret.append(child).append(" ");
+        }
+        int len = ret.length();
+        if (ret.charAt(len - 1) == ' ') {
+            ret.setLength(len - 1);
+        }
+        ret.append(")");
+        String label = getLabel();
+        if (label != null) {
+            ret.append(" as(").append(label).append(")");
+        }
+        return ret.toString();
+    }
+
+    /**
+     * Returns a description of the given level. This allows for more descriptive errors being passed back to the user.
+     *
+     * @param level The level to describe.
+     * @return A description of the given level.
+     */
+    public static String getLevelDesc(int level) {
+        if (level <= 0) {
+            return "single hit";
+        } else if (level == 1) {
+            return "single group";
+        } else {
+            StringBuilder ret = new StringBuilder();
+            for (int i = 1; i < level; ++i) {
+                ret.append("list of ");
+            }
+            ret.append("groups");
+            return ret.toString();
+        }
+    }
+
+    /**
+     * Convenience method to call {@link #fromStringAsList(String)} and assert that the list contains exactly one
+     * grouping operation.
+     *
+     * @param str The string to parse.
+     * @return A grouping operation that corresponds to the string.
+     * @throws IllegalArgumentException Thrown if the string could not be parsed as a single operation.
+     */
+    public static GroupingOperation fromString(String str) {
+        List<GroupingOperation> lst = fromStringAsList(str);
+        if (lst.size() != 1) {
+            throw new IllegalArgumentException("Expected 1 operation, got " + lst.size() + ".");
+        }
+        return lst.get(0);
+    }
+
+    /**
+     * Parses the given string as a list of grouping operations. This method never returns null, it either returns a
+     * list of valid grouping requests or it throws an exception.
+     *
+     * @param str The string to parse.
+     * @return A list of grouping operations that corresponds to the string.
+     * @throws IllegalArgumentException Thrown if the string could not be parsed.
+     */
+    public static List<GroupingOperation> fromStringAsList(String str) {
+        if (str == null || str.trim().length() == 0) {
+            return Collections.emptyList();
+        }
+        GroupingParserInput input = new GroupingParserInput(str);
+        try {
+            return new GroupingParser(input).requestList();
+        } catch (ParseException | TokenMgrError e) {
+            throw new IllegalArgumentException(input.formatException(e.getMessage()), e);
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java
new file mode 100644
index 00000000000..5410ada6cf5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/HourOfDayFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents an hour-of-day timestamp-function in a {@link GroupingExpression}. It evaluates to a long that
+ * equals the hour of day (0-23) of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HourOfDayFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public HourOfDayFunction(GroupingExpression exp) {
+        super("time.hourofday", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/Infinite.java b/container-search/src/main/java/com/yahoo/search/grouping/request/Infinite.java
new file mode 100644
index 00000000000..dfee7d0e48a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/Infinite.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an Infinite value that may be used as a bucket
+ * size specifier.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+@SuppressWarnings("rawtypes")
+public class Infinite implements Comparable {
+    private final boolean negative;
+
+    /**
+     * Create an Infinite object with positive or negative sign.
+     * @param negative the signedness.
+     */
+    public Infinite(boolean negative) {
+        this.negative = negative;
+    }
+
+    /**
+     * Override the toString method in order to be re-parseable.
+     */
+    @Override
+    public String toString() {
+        return (negative ? "-inf" : "inf");
+    }
+
+    /**
+     * An infinity value is always less than or greater than.
+     */
+    @Override
+    public int compareTo(Object rhs) {
+        return (negative ? -1 : 1);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/InfiniteValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/InfiniteValue.java
new file mode 100644
index 00000000000..d20a9eb63f8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/InfiniteValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an infinite value in a {@link GroupingExpression}.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class InfiniteValue extends ConstantValue<Infinite> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The immutable value to assign to this.
+     */
+    public InfiniteValue(Infinite value) {
+        super(value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/InterpolatedLookup.java b/container-search/src/main/java/com/yahoo/search/grouping/request/InterpolatedLookup.java
new file mode 100644
index 00000000000..a49ccdddbbc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/InterpolatedLookup.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import com.google.common.annotations.Beta;
+
+/**
+ * This class represents a lookup in a multivalue document
+ * attribute in a {@link GroupingExpression}.  It takes the
+ * attribute (assumed to contain a sorted array) from the input
+ * {@link com.yahoo.search.result.Hit} and finds the index that
+ * the second (lookup) argument expression would have, with linear
+ * interpolation when the lookup argument is between two array
+ * element values.
+ *
+ * @author arnej27959
+ */
+@Beta
+public class InterpolatedLookup extends DocumentValue {
+
+    private final String attributeName;
+    private final GroupingExpression arg2;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param attributeName The attribute name the lookup should happen in
+     * @param lookupArg Expression giving a floating-point value for the lookup argument
+     */
+    public InterpolatedLookup(String attributeName, GroupingExpression lookupArg) {
+        super("interpolatedlookup(" + attributeName + ", " + lookupArg + ")");
+        this.attributeName = attributeName;
+	this.arg2 = lookupArg;
+    }
+
+    /**
+     * Get the name of the attribute to be retrieved from the input hit.
+     * @return The attribute name.
+     */
+    public String getAttributeName() {
+        return attributeName;
+    }
+
+    /**
+     * Get the expression that will be evaluated before lookup.
+     * @return grouping expression argument
+     */
+    public GroupingExpression getLookupArgument() {
+	return arg2;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongBucket.java
new file mode 100644
index 00000000000..566ca31cb2e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongBucket.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a {@link Long} bucket in a {@link PredefinedFunction}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LongBucket extends BucketValue {
+
+    /**
+     * Gives the next distinct long value.
+     *
+     * @param value the base value.
+     * @return the nextt value.
+     */
+    public static LongValue nextValue(LongValue value) {
+        long v = value.getValue();
+        return new LongValue(v < Long.MAX_VALUE ? v + 1 : v);
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    public LongBucket(long from, long to) {
+        super(new LongValue(from), new LongValue(to));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    @SuppressWarnings("rawtypes")
+    public LongBucket(ConstantValue from, ConstantValue to) {
+        super(from, to);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongPredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongPredefined.java
new file mode 100644
index 00000000000..486c8a9ddde
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongPredefined.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a
+ * long.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LongPredefined extends PredefinedFunction {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a long.
+     * @param arg1 The compulsory bucket.
+     * @param argN The optional buckets.
+     */
+    public LongPredefined(GroupingExpression exp, LongBucket arg1, LongBucket... argN) {
+        this(exp, asList(arg1, argN));
+    }
+
+    private LongPredefined(GroupingExpression exp, List<LongBucket> args) {
+        super(exp, args);
+    }
+
+    @Override
+    public LongBucket getBucket(int i) {
+        return (LongBucket)getArg(i + 1);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a long.
+     * @param args The buckets to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the list of buckets is empty.
+     */
+    public static LongPredefined newInstance(GroupingExpression exp, List<LongBucket> args) {
+        if (args.isEmpty()) {
+            throw new IllegalArgumentException("Expected at least one bucket, got none.");
+        }
+        return new LongPredefined(exp, args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java
new file mode 100644
index 00000000000..62a0cb01f08
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/LongValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a constant {@link Long} value in a {@link GroupingExpression}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LongValue extends ConstantValue<Long> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The immutable value to assign to this.
+     */
+    public LongValue(long value) {
+        super(value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java
new file mode 100644
index 00000000000..637e0fdf57e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathACosFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathACosFunction(GroupingExpression exp) {
+        super("math.acos", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java
new file mode 100644
index 00000000000..aa5677d90d4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathACosHFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathACosHFunction extends FunctionNode {
+/**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathACosHFunction(GroupingExpression exp) {
+        super("math.acosh", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java
new file mode 100644
index 00000000000..c4b9c7a62d6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathASinFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathASinFunction(GroupingExpression exp) {
+        super("math.asin", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java
new file mode 100644
index 00000000000..f368aefe88a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathASinHFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathASinHFunction extends FunctionNode {
+/**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathASinHFunction(GroupingExpression exp) {
+        super("math.asinh", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java
new file mode 100644
index 00000000000..ed9349c86e6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathATanFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathATanFunction(GroupingExpression exp) {
+        super("math.atan", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java
new file mode 100644
index 00000000000..ebcfd1895fa
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathATanHFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathATanHFunction extends FunctionNode {
+/**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathATanHFunction(GroupingExpression exp) {
+        super("math.atanh", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java
new file mode 100644
index 00000000000..78e2c3c9aa5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCbrtFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathCbrtFunction extends FunctionNode {
+/**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathCbrtFunction(GroupingExpression exp) {
+        super("math.cbrt", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java
new file mode 100644
index 00000000000..0ab35653607
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathCosFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathCosFunction(GroupingExpression exp) {
+        super("math.cos", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java
new file mode 100644
index 00000000000..f4137c302e8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathCosHFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathCosHFunction extends FunctionNode {
+/**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathCosHFunction(GroupingExpression exp) {
+        super("math.cosh", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java
new file mode 100644
index 00000000000..4be93d77c41
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathExpFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathExpFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathExpFunction(GroupingExpression exp) {
+        super("math.exp", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java
new file mode 100644
index 00000000000..f105332e352
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFloorFunction.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/** represents the math.floor(expression) function */
+public class MathFloorFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathFloorFunction(GroupingExpression exp) {
+        super("math.floor", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java
new file mode 100644
index 00000000000..5fe5a971be9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathFunctions.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public abstract class MathFunctions {
+    /**
+     * Defines the different types of math functions that are available.
+     */
+    public enum Function {
+        EXP,     //  0
+        POW,     //  1
+        LOG,     //  2
+        LOG1P,   //  3
+        LOG10,   //  4
+        SIN,     //  5
+        ASIN,    //  6
+        COS,     //  7
+        ACOS,    //  8
+        TAN,     //  9
+        ATAN,    // 10
+        SQRT,    // 11
+        SINH,    // 12
+        ASINH,   // 13
+        COSH,    // 14
+        ACOSH,   // 15
+        TANH,    // 16
+        ATANH,   // 17
+        CBRT,    // 18
+        HYPOT,   // 19
+        FLOOR;   // 20
+
+        static Function create(int tid) {
+            for(Function p : values()) {
+                if (tid == p.ordinal()) {
+                    return p;
+                }
+            }
+            return null;
+        }
+    }
+    public static FunctionNode newInstance(Function type, GroupingExpression x, GroupingExpression y) {
+        switch (type) {
+            case EXP: return new MathExpFunction(x);
+            case POW: return new MathPowFunction(x, y);
+            case LOG: return new MathLogFunction(x);
+            case LOG1P: return new MathLog1pFunction(x);
+            case LOG10: return new MathLog10Function(x);
+            case SIN: return new MathSinFunction(x);
+            case ASIN: return new MathASinFunction(x);
+            case COS: return new MathCosFunction(x);
+            case ACOS: return new MathACosFunction(x);
+            case TAN: return new MathTanFunction(x);
+            case ATAN: return new MathATanFunction(x);
+            case SQRT: return new MathSqrtFunction(x);
+            case SINH: return new MathSinHFunction(x);
+            case ASINH: return new MathASinHFunction(x);
+            case COSH: return new MathCosHFunction(x);
+            case ACOSH: return new MathACosHFunction(x);
+            case TANH: return new MathTanHFunction(x);
+            case ATANH: return new MathATanHFunction(x);
+            case CBRT: return new MathCbrtFunction(x);
+            case HYPOT: return new MathHypotFunction(x, y);
+            case FLOOR: return new MathFloorFunction(x);
+        }
+        return null;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java
new file mode 100644
index 00000000000..777a94f9107
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathHypotFunction.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathHypotFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param x The expression to evaluate for x, double value will be requested.
+     * @param y The expression to evaluate for y exponent, double value will be requested.
+     */
+    public MathHypotFunction(GroupingExpression x, GroupingExpression y) {
+        super("math.hypot", Arrays.asList(x, y));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java
new file mode 100644
index 00000000000..444ea7a7349
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog10Function.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathLog10Function extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathLog10Function(GroupingExpression exp) {
+        super("math.log10", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java
new file mode 100644
index 00000000000..3be6c799bf2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLog1pFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathLog1pFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathLog1pFunction(GroupingExpression exp) {
+        super("math.log1p", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java
new file mode 100644
index 00000000000..4d3b43d45b0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathLogFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathLogFunction extends FunctionNode {
+        /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathLogFunction(GroupingExpression exp) {
+        super("math.log", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java
new file mode 100644
index 00000000000..09a9a28cbb0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathPowFunction.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathPowFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param x The expression to evaluate for base, double value will be requested.
+     * @param y The expression to evaluate for the exponent, double value will be requested.
+     */
+    public MathPowFunction(GroupingExpression x, GroupingExpression y) {
+        super("math.pow", Arrays.asList(x,y));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathResolver.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathResolver.java
new file mode 100644
index 00000000000..9410c6ea347
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathResolver.java
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Stack;
+
+/**
+ * This is a helper class for resolving arithmetic operations over {@link GroupingExpression} objects. To resolve an
+ * operation simply push operator-expression pairs onto it, before calling {@link #resolve()} to retrieve the single
+ * corresponding grouping expression object.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MathResolver {
+
+    public enum Type {
+
+        ADD(0, "+"),
+        SUB(1, "-"),
+        DIV(2, "/"),
+        MOD(3, "%"),
+        MUL(4, "*");
+
+        private final int pre;
+        private final String image;
+
+        private Type(int pre, String image) {
+            this.pre = pre;
+            this.image = image;
+        }
+    }
+
+    private final List<Item> items = new LinkedList<>();
+
+    /**
+     * Pushes the given operator-expression pair onto this math resolver. Once all pairs have been pushed using this
+     * method, call {@link #resolve()} to retrieve to combined grouping expression.
+     *
+     * @param type The operator that appears before the expression being pushed.
+     * @param exp  The expression to push.
+     */
+    public void push(Type type, GroupingExpression exp) {
+        if (items.isEmpty() && type != Type.ADD) {
+            throw new IllegalArgumentException("First item in an arithmetic operation must be an addition.");
+        }
+        items.add(new Item(type, exp));
+    }
+
+    /**
+     * Converts the internal list of operator-expression pairs into a corresponding combined grouping expression. When
+     * this method returns there is no residue of the conversion, and this object can be reused.
+     *
+     * @return The grouping expression corresponding to the pushed arithmetic operations.
+     */
+    public GroupingExpression resolve() {
+        if (items.size() == 1) {
+            return items.remove(0).exp; // optimize common case
+        }
+        Stack<Item> stack = new Stack<>();
+        stack.push(items.remove(0));
+        while (!items.isEmpty()) {
+            Item item = items.remove(0);
+            while (stack.size() > 1 && stack.peek().type.pre >= item.type.pre) {
+                pop(stack);
+            }
+            stack.push(item);
+        }
+        while (stack.size() > 1) {
+            pop(stack);
+        }
+        return stack.remove(0).exp;
+    }
+
+    private void pop(Stack<Item> stack) {
+        Item rhs = stack.pop();
+        Item lhs = stack.peek();
+        switch (rhs.type) {
+        case ADD:
+            lhs.exp = new AddFunction(lhs.exp, rhs.exp);
+            break;
+        case DIV:
+            lhs.exp = new DivFunction(lhs.exp, rhs.exp);
+            break;
+        case MOD:
+            lhs.exp = new ModFunction(lhs.exp, rhs.exp);
+            break;
+        case MUL:
+            lhs.exp = new MulFunction(lhs.exp, rhs.exp);
+            break;
+        case SUB:
+            lhs.exp = new SubFunction(lhs.exp, rhs.exp);
+            break;
+        default:
+            throw new UnsupportedOperationException("Operator " + rhs.type + " not supported.");
+        }
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        for (int i = 0, len = items.size(); i < len; ++i) {
+            Item item = items.get(i);
+            if (i != 0) {
+                ret.append(" ").append(item.type.image).append(" ");
+            }
+            ret.append(item.exp.toString());
+        }
+        return ret.toString();
+    }
+
+    private static class Item {
+        final Type type;
+        GroupingExpression exp;
+
+        Item(Type type, GroupingExpression exp) {
+            this.type = type;
+            this.exp = exp;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java
new file mode 100644
index 00000000000..66612e9d80a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathSinFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathSinFunction(GroupingExpression exp) {
+        super("math.sin", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java
new file mode 100644
index 00000000000..79d260f51a0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSinHFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathSinHFunction extends FunctionNode {
+/**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathSinHFunction(GroupingExpression exp) {
+        super("math.sinh", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java
new file mode 100644
index 00000000000..18c9396dd12
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathSqrtFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathSqrtFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathSqrtFunction(GroupingExpression exp) {
+        super("math.sqrt", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java
new file mode 100644
index 00000000000..67db7a9d834
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathTanFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathTanFunction(GroupingExpression exp) {
+        super("math.tan", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java
new file mode 100644
index 00000000000..e111c1199d7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MathTanHFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author balder
+ */
+public class MathTanHFunction extends FunctionNode {
+/**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, double value will be requested.
+     */
+    public MathTanHFunction(GroupingExpression exp) {
+        super("math.tanh", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MaxAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxAggregator.java
new file mode 100644
index 00000000000..93f9e3c068e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxAggregator.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an maximum-aggregator in a {@link GroupingExpression}. It evaluates to the maximum value that
+ * the contained expression evaluated to over all the inputs.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MaxAggregator extends AggregatorNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to aggregate on.
+     */
+    public MaxAggregator(GroupingExpression exp) {
+        super("max", exp);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MaxFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxFunction.java
new file mode 100644
index 00000000000..da80a627c27
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MaxFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a max-function in a {@link GroupingExpression}. It evaluates to a number that equals the
+ * largest of the results of all arguments.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MaxFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public MaxFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private MaxFunction(List<GroupingExpression> args) {
+        super("max", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static MaxFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new MaxFunction(args);
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java b/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java
new file mode 100644
index 00000000000..b2bd503c52f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/Md5Function.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents an md5-function in a {@link GroupingExpression}. It evaluates to a long that equals the md5 of
+ * the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Md5Function extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp     The expression to evaluate.
+     * @param numBits The number of bits of the md5 to include.
+     */
+    public Md5Function(GroupingExpression exp, int numBits) {
+        super("md5", Arrays.asList(exp, new LongValue(numBits)));
+    }
+
+    /**
+     * Returns the number of bits of the md5 to include in the evaluated result.
+     *
+     * @return The bit count.
+     */
+    public int getNumBits() {
+        return ((LongValue)getArg(1)).getValue().intValue();
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinAggregator.java
new file mode 100644
index 00000000000..5bb2f6675c8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinAggregator.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an minimum-aggregator in a {@link GroupingExpression}. It evaluates to the minimum value that
+ * the contained expression evaluated to over all the inputs.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MinAggregator extends AggregatorNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to aggregate on.
+     */
+    public MinAggregator(GroupingExpression exp) {
+        super("min", exp);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java
new file mode 100644
index 00000000000..f66e23b87c0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a min-function in a {@link GroupingExpression}. It evaluates to a number that equals the
+ * smallest of the results of all arguments.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MinFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public MinFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private MinFunction(List<GroupingExpression> args) {
+        super("min", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static MinFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new MinFunction(args);
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java
new file mode 100644
index 00000000000..cb4b65f20b8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MinuteOfHourFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a minute-of-hour timestamp-function in a {@link GroupingExpression}. It evaluates to a long
+ * that equals the minute of hour (0-59) of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MinuteOfHourFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public MinuteOfHourFunction(GroupingExpression exp) {
+        super("time.minuteofhour", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ModFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ModFunction.java
new file mode 100644
index 00000000000..d3d2502b714
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ModFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a mod-function in a {@link GroupingExpression}. It evaluates to a number that equals the result
+ * of mod'ing the results of all arguments in the order they were given to the constructor (modulo first argument by
+ * second, result by third, ...).
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ModFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public ModFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private ModFunction(List<GroupingExpression> args) {
+        super("mod", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static ModFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new ModFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java
new file mode 100644
index 00000000000..25f39892ee1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MonthOfYearFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a month-of-year timestamp-function in a {@link GroupingExpression}. It evaluates to a long that
+ * equals the month of year (1-12) of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MonthOfYearFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public MonthOfYearFunction(GroupingExpression exp) {
+        super("time.monthofyear", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/MulFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/MulFunction.java
new file mode 100644
index 00000000000..d66361888b0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/MulFunction.java
@@ -0,0 +1,42 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a mul-function in a {@link GroupingExpression}. It evaluates to a number that equals the result
+ * of multiplying the results of all arguments together in the order they were given to the constructor.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class MulFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public MulFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private MulFunction(List<GroupingExpression> args) {
+        super("mul", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static MulFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new MulFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java
new file mode 100644
index 00000000000..7ea2b3a788b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NegFunction.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a negate-function in a {@link GroupingExpression}. It evaluates to a number that equals the
+ * negative of the results of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NegFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a number.
+     */
+    public NegFunction(GroupingExpression exp) {
+        super("neg", Arrays.asList(exp));
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java
new file mode 100644
index 00000000000..1eaad713383
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NormalizeSubjectFunction.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ */
+public class NormalizeSubjectFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a string.
+     */
+    public NormalizeSubjectFunction(GroupingExpression exp) {
+        super("normalizesubject", Arrays.asList(exp));
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java
new file mode 100644
index 00000000000..f876ee9a1df
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/NowFunction.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Collections;
+
+/**
+ * This class represents a now-function in a {@link GroupingExpression}. It evaluates to a long that equals the number
+ * of seconds since midnight, January 1, 1970 UTC.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NowFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public NowFunction() {
+        super("now", Collections.<GroupingExpression>emptyList());
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/OrFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/OrFunction.java
new file mode 100644
index 00000000000..0a7ec7ecc06
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/OrFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents an or-function in a {@link GroupingExpression}. It evaluates to a long that equals the result
+ * of or'ing the results of all arguments together in the order they were given to the constructor.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class OrFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a long.
+     * @param arg2 The second compulsory argument, must evaluate to a long.
+     * @param argN The optional arguments, must evaluate to a long.
+     */
+    public OrFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private OrFunction(List<GroupingExpression> args) {
+        super("or", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static OrFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new OrFunction(args);
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/PredefinedFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/PredefinedFunction.java
new file mode 100644
index 00000000000..b00ee97452c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/PredefinedFunction.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * This class represents a predefined bucket-function in a {@link GroupingExpression}. It maps the input into one of the
+ * given buckets by the result of the argument expression.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class PredefinedFunction extends FunctionNode {
+
+    protected PredefinedFunction(GroupingExpression exp, List<? extends BucketValue> args) {
+        super("predefined", asList(exp, args));
+        Iterator<? extends BucketValue> it = args.iterator();
+        BucketValue prev = it.next();
+        while (it.hasNext()) {
+            BucketValue arg = it.next();
+            if (prev.compareTo(arg) >= 0) {
+                throw new IllegalArgumentException("Buckets must be monotonically increasing, got " + prev +
+                                                   " before " + arg + ".");
+            }
+            prev = arg;
+        }
+    }
+
+    /**
+     * Returns the number of buckets to divide the result into.
+     *
+     * @return The bucket count.
+     */
+    public int getNumBuckets() {
+        return getNumArgs() - 1;
+    }
+
+    /**
+     * Returns the bucket at the given index.
+     *
+     * @param i The index of the bucket to return.
+     * @return The bucket at the given index.
+     * @throws IndexOutOfBoundsException If the index is out of range.
+     */
+    public BucketValue getBucket(int i) {
+        return (BucketValue)getArg(i + 1);
+    }
+
+    private static
+    List<GroupingExpression> asList(GroupingExpression exp, List<? extends BucketValue> args) {
+        List<GroupingExpression> ret = new LinkedList<>();
+        ret.add(exp);
+        ret.addAll(args);
+        return ret;
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBucket.java
new file mode 100644
index 00000000000..d13b8b6ca67
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBucket.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a {@link RawValue} bucket in a {@link PredefinedFunction}.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class RawBucket extends BucketValue {
+
+    /**
+     * Get the next distinct value.
+     *
+     * @param value The base value.
+     * @return the next value.
+     */
+    public static RawValue nextValue(RawValue value) {
+        return new RawValue(value.getValue().clone().put((byte)0));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    public RawBucket(RawBuffer from, RawBuffer to) {
+        super(new RawValue(from), new RawValue(to));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    public RawBucket(ConstantValue<?> from, ConstantValue<?> to) {
+        super(from, to);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawBuffer.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBuffer.java
new file mode 100644
index 00000000000..00b9c899263
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawBuffer.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.ArrayList;
+
+/**
+ * This class represents a buffer of byte values to be used as a backing buffer
+ * for raw buckets.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class RawBuffer implements Comparable<RawBuffer>, Cloneable {
+    private final ArrayList<Byte> buffer;
+
+    /**
+     * Create an empty buffer.
+     */
+    public RawBuffer() {
+        this.buffer = new ArrayList<>();
+    }
+
+    /**
+     * Create a buffer with initial content.
+     *
+     * @param buffer A buffer of values to be assigned this buffer.
+     */
+    public RawBuffer(ArrayList<Byte> buffer) {
+        this.buffer = buffer;
+    }
+
+    /**
+     * Create a buffer with initial content.
+     *
+     * @param bytes A buffer of bytes to be assigned this buffer.
+     */
+    public RawBuffer(byte[] bytes) {
+        buffer = new ArrayList<>();
+        put(bytes);
+    }
+
+    /**
+     * Insert a byte value into this buffer.
+     *
+     * @param value The value to add to the buffer.
+     * @return Reference to this.
+     */
+    public RawBuffer put(byte value) {
+        buffer.add(value);
+        return this;
+    }
+
+    /**
+     * Insert an array of byte values into this buffer.
+     *
+     * @param values The array to add to the buffer.
+     * @return Reference to this.
+     */
+    public RawBuffer put(byte[] values) {
+        for (int i = 0; i < values.length; i++) {
+            buffer.add(values[i]);
+        }
+        return this;
+    }
+
+    /**
+     * Create a copy of data in the internal buffer.
+     *
+     * @return A copy of the data.
+     */
+    public byte[] getBytes() {
+        byte[] ret = new byte[buffer.size()];
+        for (int i = 0; i < ret.length; i++) {
+            ret[i] = buffer.get(i);
+        }
+        return ret;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder s = new StringBuilder();
+        s.append("{");
+        for (int i = 0; i < buffer.size(); i++) {
+            s.append(buffer.get(i));
+            if (i < buffer.size() - 1) {
+                s.append(",");
+            }
+        }
+        s.append("}");
+        return s.toString();
+    }
+
+    @Override
+    public RawBuffer clone() {
+        return new RawBuffer(new ArrayList<>(buffer));
+    }
+
+    @Override
+    public int compareTo(RawBuffer rhs) {
+        Byte[] my = buffer.toArray(new Byte[0]);
+        Byte[] their = rhs.buffer.toArray(new Byte[0]);
+        for (int i = 0; i < my.length && i < their.length; i++) {
+            if (my[i] < their[i]) {
+                return -1;
+            } else if (my[i] > their[i]) {
+                return 1;
+            }
+        }
+        return (my.length < their.length ? -1 : (my.length > their.length ? 1 : 0));
+    }
+
+    @Override
+    public int hashCode() {
+        return buffer.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object rhs) {
+        if (rhs instanceof RawBuffer) {
+            return (compareTo((RawBuffer)rhs) == 0);
+        }
+        return false;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawPredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawPredefined.java
new file mode 100644
index 00000000000..c2650346231
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawPredefined.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a
+ * raw.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class RawPredefined extends PredefinedFunction {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a string.
+     * @param arg1 The compulsory bucket.
+     * @param argN The optional buckets.
+     */
+    public RawPredefined(GroupingExpression exp, RawBucket arg1, RawBucket... argN) {
+        this(exp, asList(arg1, argN));
+    }
+
+    private RawPredefined(GroupingExpression exp, List<RawBucket> args) {
+        super(exp, args);
+    }
+
+    @Override
+    public RawBucket getBucket(int i) {
+        return (RawBucket)getArg(i + 1);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a string.
+     * @param args The buckets to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the list of buckets is empty.
+     */
+    public static RawPredefined newInstance(GroupingExpression exp, List<RawBucket> args) {
+        if (args.isEmpty()) {
+            throw new IllegalArgumentException("Expected at least one bucket, got none.");
+        }
+        return new RawPredefined(exp, args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RawValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RawValue.java
new file mode 100644
index 00000000000..a04944d7897
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RawValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a raw value in a {@link GroupingExpression}.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class RawValue extends ConstantValue<RawBuffer> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The immutable value to assign to this.
+     */
+    public RawValue(RawBuffer value) {
+        super(value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/RelevanceValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/RelevanceValue.java
new file mode 100644
index 00000000000..8a5d4dc75d1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/RelevanceValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a document relevance score in a {@link GroupingExpression}. It evaluates to the relevance of
+ * the input {@link com.yahoo.search.result.Hit}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RelevanceValue extends DocumentValue {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public RelevanceValue() {
+        super("relevance()");
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java
new file mode 100644
index 00000000000..274bb20c9f7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ReverseFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a reverse-function in a {@link GroupingExpression}. It evaluates to a list that equals the list
+ * result of the argument, sorted in descending order.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class ReverseFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a list.
+     */
+    public ReverseFunction(GroupingExpression exp) {
+        super("reverse", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java
new file mode 100644
index 00000000000..9443f862a16
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SecondOfMinuteFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a second-of-minute timestamp-function in a {@link GroupingExpression}. It evaluates to a long
+ * that equals the second of minute (0-59) of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SecondOfMinuteFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public SecondOfMinuteFunction(GroupingExpression exp) {
+        super("time.secondofminute", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java
new file mode 100644
index 00000000000..d445007a039
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SizeFunction.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a size-function in a {@link GroupingExpression}. It evaluates to a number that equals the
+ * number of elements in the result of the argument (e.g. the number of elements in an array).
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SizeFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate.
+     */
+    public SizeFunction(GroupingExpression exp) {
+        super("size", Arrays.asList(exp));
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java
new file mode 100644
index 00000000000..2a8845f9847
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SortFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a sort-function in a {@link GroupingExpression}. It evaluates to a list that equals the list
+ * result of the argument, sorted in ascending order.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class SortFunction  extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a list.
+     */
+    public SortFunction(GroupingExpression exp) {
+        super("sort", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StrCatFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StrCatFunction.java
new file mode 100644
index 00000000000..455f9dee917
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StrCatFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a strcat-function in a {@link GroupingExpression}. It evaluates to a string that equals the
+ * contatenation of the string results of all arguments in the order they were given to the constructor.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StrCatFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a string.
+     * @param arg2 The second compulsory argument, must evaluate to a string.
+     * @param argN The optional arguments, must evaluate to a string.
+     */
+    public StrCatFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private StrCatFunction(List<GroupingExpression> args) {
+        super("strcat", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static StrCatFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new StrCatFunction(args);
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java
new file mode 100644
index 00000000000..2ef53f53bf2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StrLenFunction.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a strcat-function in a {@link GroupingExpression}. It evaluates to a long that equals the
+ * number of bytes in the string result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StrLenFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a string.
+     */
+    public StrLenFunction(GroupingExpression exp) {
+        super("strlen", Arrays.asList(exp));
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StringBucket.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StringBucket.java
new file mode 100644
index 00000000000..34c7b9f526a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StringBucket.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a {@link String} bucket in a {@link PredefinedFunction}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StringBucket extends BucketValue {
+
+    /**
+     * Get the next distinct value.
+     *
+     * @param value The base value.
+     * @return the next value.
+     */
+    public static StringValue nextValue(StringValue value) {
+        return new StringValue(value.getValue() + " ");
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    public StringBucket(String from, String to) {
+        super(new StringValue(from), new StringValue(to));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from          The from-value to assign to this.
+     * @param to            The to-value to assign to this.
+     */
+    public StringBucket(ConstantValue<?> from, ConstantValue<?> to) {
+        super(from, to);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StringPredefined.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StringPredefined.java
new file mode 100644
index 00000000000..d3a469fdd7e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StringPredefined.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a predefined bucket-function in a {@link GroupingExpression} for expressions that evaluate to a
+ * string.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StringPredefined extends PredefinedFunction {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a string.
+     * @param arg1 The compulsory bucket.
+     * @param argN The optional buckets.
+     */
+    public StringPredefined(GroupingExpression exp, StringBucket arg1, StringBucket... argN) {
+        this(exp, asList(arg1, argN));
+    }
+
+    private StringPredefined(GroupingExpression exp, List<StringBucket> args) {
+        super(exp, args);
+    }
+
+    @Override
+    public StringBucket getBucket(int i) {
+        return (StringBucket)getArg(i + 1);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param exp  The expression to evaluate, must evaluate to a string.
+     * @param args The buckets to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the list of buckets is empty.
+     */
+    public static StringPredefined newInstance(GroupingExpression exp, List<StringBucket> args) {
+        if (args.isEmpty()) {
+            throw new IllegalArgumentException("Expected at least one bucket, got none.");
+        }
+        return new StringPredefined(exp, args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/StringValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/StringValue.java
new file mode 100644
index 00000000000..87e818368d6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/StringValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a constant {@link String} value in a {@link GroupingExpression}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StringValue extends ConstantValue<String> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The immutable value to assign to this.
+     */
+    public StringValue(String value) {
+        super(value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SubFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SubFunction.java
new file mode 100644
index 00000000000..15e05c50f63
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SubFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents a div-function in a {@link GroupingExpression}. It evaluates to a number that equals the result
+ * of subtracting the results of all arguments in the order they were given to the constructor (subtract second argument
+ * from first, third from result, ...).
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SubFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a number.
+     * @param arg2 The second compulsory argument, must evaluate to a number.
+     * @param argN The optional arguments, must evaluate to a number.
+     */
+    public SubFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private SubFunction(List<GroupingExpression> args) {
+        super("sub", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static SubFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new SubFunction(args);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SumAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SumAggregator.java
new file mode 100644
index 00000000000..1ace1cfbba2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SumAggregator.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an sum-aggregator in a {@link GroupingExpression}. It evaluates to the sum of the values that
+ * the contained expression evaluated to over all the inputs.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SumAggregator extends AggregatorNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to aggregate on.
+     */
+    public SumAggregator(GroupingExpression exp) {
+        super("sum", exp);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/SummaryValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/SummaryValue.java
new file mode 100644
index 00000000000..72e4c6662d3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/SummaryValue.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a document summary in a {@link GroupingExpression}. It evaluates to the summary of the input
+ * {@link com.yahoo.search.result.Hit} that corresponds to the named summary class.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class SummaryValue extends DocumentValue {
+
+    private final String name;
+
+    /**
+     * Constructs a new instance of this class, using the default summary class.
+     */
+    public SummaryValue() {
+        super("summary()");
+        name = null;
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param summaryName The name of the summary class to assign to this.
+     */
+    public SummaryValue(String summaryName) {
+        super("summary(" + summaryName + ")");
+        name = summaryName;
+    }
+
+    /**
+     * Returns the name of the summary class used to retrieve the hit from the search node.
+     *
+     * @return The summary name.
+     */
+    public String getSummaryName() {
+        return name;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/TimeFunctions.java b/container-search/src/main/java/com/yahoo/search/grouping/request/TimeFunctions.java
new file mode 100644
index 00000000000..bde1c5831b5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/TimeFunctions.java
@@ -0,0 +1,148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This abstract class is a factory for timestamp functions in a {@link GroupingExpression}. Apart from offering
+ * per-function factory methods, this class also contains a {@link #newInstance(com.yahoo.search.grouping.request.TimeFunctions.Type,
+ * GroupingExpression)} method which is useful for runtime construction of grouping requests.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class TimeFunctions {
+
+    /**
+     * Defines the different types of timestamps-functions that are available.
+     */
+    public enum Type {
+        DATE,
+        DAY_OF_MONTH,
+        DAY_OF_WEEK,
+        DAY_OF_YEAR,
+        HOUR_OF_DAY,
+        MINUTE_OF_HOUR,
+        MONTH_OF_YEAR,
+        SECOND_OF_MINUTE,
+        YEAR
+    }
+
+    /**
+     * Creates a new timestamp-function of the specified type for the given {@link GroupingExpression}.
+     *
+     * @param type The type of function to create.
+     * @param exp  The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static FunctionNode newInstance(Type type, GroupingExpression exp) {
+        switch (type) {
+        case DATE:
+            return newDate(exp);
+        case DAY_OF_MONTH:
+            return newDayOfMonth(exp);
+        case DAY_OF_WEEK:
+            return newDayOfWeek(exp);
+        case DAY_OF_YEAR:
+            return newDayOfYear(exp);
+        case HOUR_OF_DAY:
+            return newHourOfDay(exp);
+        case MINUTE_OF_HOUR:
+            return newMinuteOfHour(exp);
+        case MONTH_OF_YEAR:
+            return newMonthOfYear(exp);
+        case SECOND_OF_MINUTE:
+            return newSecondOfMinute(exp);
+        case YEAR:
+            return newYear(exp);
+        }
+        throw new UnsupportedOperationException("Time function '" + type + "' not supported.");
+    }
+
+    /**
+     * Creates a new instance of {@link DateFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static DateFunction newDate(GroupingExpression exp) {
+        return new DateFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link DayOfMonthFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static DayOfMonthFunction newDayOfMonth(GroupingExpression exp) {
+        return new DayOfMonthFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link DayOfWeekFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static DayOfWeekFunction newDayOfWeek(GroupingExpression exp) {
+        return new DayOfWeekFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link DayOfYearFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static DayOfYearFunction newDayOfYear(GroupingExpression exp) {
+        return new DayOfYearFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link HourOfDayFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static HourOfDayFunction newHourOfDay(GroupingExpression exp) {
+        return new HourOfDayFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link MinuteOfHourFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static MinuteOfHourFunction newMinuteOfHour(GroupingExpression exp) {
+        return new MinuteOfHourFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link MonthOfYearFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static MonthOfYearFunction newMonthOfYear(GroupingExpression exp) {
+        return new MonthOfYearFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link SecondOfMinuteFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static SecondOfMinuteFunction newSecondOfMinute(GroupingExpression exp) {
+        return new SecondOfMinuteFunction(exp);
+    }
+
+    /**
+     * Creates a new instance of {@link YearFunction} for the given {@link GroupingExpression}.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     * @return The created function node.
+     */
+    public static YearFunction newYear(GroupingExpression exp) {
+        return new YearFunction(exp);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java
new file mode 100644
index 00000000000..8eab2af8691
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToDoubleFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a todouble-function in a {@link GroupingExpression}. It converts the result of the argument to
+ * a double. If the argument can not be converted, this function returns 0.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class ToDoubleFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate.
+     */
+    public ToDoubleFunction(GroupingExpression exp) {
+        super("todouble", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java
new file mode 100644
index 00000000000..c47a043eea0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToLongFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a tolong-function in a {@link GroupingExpression}. It converts the result of the argument to a
+ * long. If the argument can not be converted, this function returns 0.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class ToLongFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate.
+     */
+    public ToLongFunction(GroupingExpression exp) {
+        super("tolong", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java
new file mode 100644
index 00000000000..d1ba3afa28c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToRawFunction.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a toraw-function in a {@link GroupingExpression}. It
+ * converts the result of the argument to a raw type. If the argument can not
+ * be converted, this function returns null.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class ToRawFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate.
+     */
+    public ToRawFunction(GroupingExpression exp) {
+        super("toraw", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java
new file mode 100644
index 00000000000..364d9e5064d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ToStringFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a tolong-function in a {@link GroupingExpression}. It converts the result of the argument to a
+ * long. If the argument can not be converted, this function returns 0.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class ToStringFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate.
+     */
+    public ToStringFunction(GroupingExpression exp) {
+        super("tostring", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java
new file mode 100644
index 00000000000..2e23f41f139
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/UcaFunction.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents an uca-function in a {@link GroupingExpression}.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class UcaFunction extends FunctionNode {
+
+    private final String locale;
+    private final String strength;
+
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp     The expression to evaluate.
+     * @param locale  The locale to used for sorting.
+     */
+    public UcaFunction(GroupingExpression exp, String locale) {
+        super("uca", Arrays.asList(exp, new StringValue(locale)));
+        this.locale = locale;
+        this.strength = "TERTIARY";
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp     The expression to evaluate.
+     * @param locale  The locale to used for sorting.
+     * @param strength The strength level to use.
+     */
+    public UcaFunction(GroupingExpression exp, String locale, String strength) {
+        super("uca", Arrays.asList(exp, new StringValue(locale), new StringValue(strength)));
+        if (!validStrength(strength)) {
+            throw new IllegalArgumentException("Not a valid UCA strength: " + strength);
+        }
+        this.locale = locale;
+        this.strength = strength;
+    }
+
+    private boolean validStrength(String strength) {
+        return (strength.equals("PRIMARY") ||
+                strength.equals("SECONDARY") ||
+                strength.equals("TERTIARY") ||
+                strength.equals("QUATERNARY") ||
+                strength.equals("IDENTICAL"));
+    }
+
+    public String getLocale() {
+        return locale;
+    }
+
+    public String getStrength() {
+        return strength;
+    }
+}
+
+
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/XorAggregator.java b/container-search/src/main/java/com/yahoo/search/grouping/request/XorAggregator.java
new file mode 100644
index 00000000000..be0f092b929
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/XorAggregator.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents an xor-aggregator in a {@link GroupingExpression}. It evaluates to the xor of the values that
+ * the contained expression evaluated to over all the inputs.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class XorAggregator extends AggregatorNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to aggregate on.
+     */
+    public XorAggregator(GroupingExpression exp) {
+        super("xor", exp);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java
new file mode 100644
index 00000000000..304917bf905
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/XorBitFunction.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents an xor-function in a {@link GroupingExpression}. It evaluates to a long that equals the xor of
+ * 'width' bits over the binary representation of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class XorBitFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp     The expression to evaluate.
+     * @param numBits The number of bits of the expression value to xor.
+     */
+    public XorBitFunction(GroupingExpression exp, int numBits) {
+        super("xorbit", Arrays.asList(exp, new LongValue(numBits)));
+    }
+
+    /**
+     * Returns the number of bits of the expression value to xor.
+     *
+     * @return The bit count.
+     */
+    public int getNumBits() {
+        return ((LongValue)getArg(1)).getValue().intValue();
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/XorFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/XorFunction.java
new file mode 100644
index 00000000000..dc47926ea51
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/XorFunction.java
@@ -0,0 +1,43 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.List;
+
+/**
+ * This class represents an xor-function in a {@link GroupingExpression}. It evaluates to a long that equals the result
+ * of and'ing the results of all arguments together in the order they were given to the constructor.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class XorFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param arg1 The first compulsory argument, must evaluate to a long.
+     * @param arg2 The second compulsory argument, must evaluate to a long.
+     * @param argN The optional arguments, must evaluate to a long.
+     */
+    public XorFunction(GroupingExpression arg1, GroupingExpression arg2, GroupingExpression... argN) {
+        this(asList(arg1, arg2, argN));
+    }
+
+    private XorFunction(List<GroupingExpression> args) {
+        super("xor", args);
+    }
+
+    /**
+     * Constructs a new instance of this class from a list of arguments.
+     *
+     * @param args The arguments to pass to the constructor.
+     * @return The created instance.
+     * @throws IllegalArgumentException Thrown if the number of arguments is less than 2.
+     */
+    public static XorFunction newInstance(List<GroupingExpression> args) {
+        if (args.size() < 2) {
+            throw new IllegalArgumentException("Expected 2 or more arguments, got " + args.size() + ".");
+        }
+        return new XorFunction(args);
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java
new file mode 100644
index 00000000000..2115d99140d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/YearFunction.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * This class represents a year timestamp-function in a {@link GroupingExpression}. It evaluates to a long that equals
+ * the full year (e.g. 2010) of the result of the argument.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class YearFunction extends FunctionNode {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long.
+     */
+    public YearFunction(GroupingExpression exp) {
+        super("time.year", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java
new file mode 100644
index 00000000000..5754edd8155
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+/**
+ * This class represents a document checksum in a {@link GroupingExpression}. It evaluates to the YMUM checksum of the
+ * input {@link com.yahoo.search.result.Hit}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class YmumValue extends DocumentValue {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public YmumValue() {
+        super("ymum()");
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java
new file mode 100644
index 00000000000..b4790b912e7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveXFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class ZCurveXFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long or long[].
+     */
+    public ZCurveXFunction(GroupingExpression exp) {
+        super("zcurve.x", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java
new file mode 100644
index 00000000000..e9a011f2193
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/ZCurveYFunction.java
@@ -0,0 +1,18 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request;
+
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class ZCurveYFunction extends FunctionNode {
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param exp The expression to evaluate, must evaluate to a long or long[].
+     */
+    public ZCurveYFunction(GroupingExpression exp) {
+        super("zcurve.y", Arrays.asList(exp));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/package-info.java b/container-search/src/main/java/com/yahoo/search/grouping/request/package-info.java
new file mode 100644
index 00000000000..ff30ef2b939
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.grouping.request;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/parser/GroupingParserInput.java b/container-search/src/main/java/com/yahoo/search/grouping/request/parser/GroupingParserInput.java
new file mode 100644
index 00000000000..e87291fba18
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/request/parser/GroupingParserInput.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.request.parser;
+
+import com.yahoo.javacc.FastCharStream;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GroupingParserInput extends FastCharStream implements CharStream {
+
+    public GroupingParserInput(String input) {
+        super(input);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/AbstractList.java b/container-search/src/main/java/com/yahoo/search/grouping/result/AbstractList.java
new file mode 100644
index 00000000000..058c68470c4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/AbstractList.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import com.yahoo.collections.LazyMap;
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.result.HitGroup;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class AbstractList extends HitGroup {
+
+    private final Map<String, Continuation> continuations = LazyMap.newHashMap();
+    private final String label;
+
+    /**
+     * <p>Constructs a new instance of this class.</p>
+     *
+     * @param type  The type of this list.
+     * @param label The label of this list.
+     */
+    public AbstractList(String type, String label) {
+        super(type + ":" + label);
+        this.label = label;
+    }
+
+    /**
+     * <p>Returns the label of this list.</p>
+     *
+     * @return The label.
+     */
+    public String getLabel() {
+        return label;
+    }
+
+    /**
+     * <p>Returns the map of all possible {@link Continuation}s of this list.</p>
+     *
+     * @return The list of Continuations.
+     */
+    public Map<String, Continuation> continuations() {
+        return continuations;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java
new file mode 100644
index 00000000000..1d6dcc6762c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java
@@ -0,0 +1,60 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * This abstract class is used in {@link Group} instances where the identifying expression evaluated to a {@link
+ * com.yahoo.search.grouping.request.BucketValue}. The range is inclusive-from and exclusive-to.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class BucketGroupId<T> extends GroupId {
+
+    private final T from;
+    private final T to;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param type The type of this id's value.
+     * @param from The inclusive-from of the range.
+     * @param to   The exclusive-to of the range.
+     */
+    public BucketGroupId(String type, T from, T to) {
+        this(type, from, String.valueOf(from), to, String.valueOf(to));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param type      The type of this id's value.
+     * @param from      The inclusive-from of the range.
+     * @param fromImage The String representation of the <tt>from</tt> argument.
+     * @param to        The exclusive-to of the range.
+     * @param toImage   The String representation of the <tt>to</tt> argument.
+     */
+    public BucketGroupId(String type, T from, String fromImage, T to, String toImage) {
+        super(type, fromImage, toImage);
+        this.from = from;
+        this.to = to;
+    }
+
+    /**
+     * Returns the inclusive-from of the value range.
+     *
+     * @return The from-value.
+     */
+    public T getFrom() {
+        return from;
+    }
+
+    /**
+     * Returns the exclusive-to of the value range.
+     *
+     * @return The to-value.
+     */
+    public T getTo() {
+        return to;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/DoubleBucketId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/DoubleBucketId.java
new file mode 100644
index 00000000000..e9f7ffc04c0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/DoubleBucketId.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link
+ * com.yahoo.search.grouping.request.DoubleBucket}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DoubleBucketId extends BucketGroupId<Double> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from The identifying inclusive-from double.
+     * @param to   The identifying exclusive-to double.
+     */
+    public DoubleBucketId(Double from, Double to) {
+        super("double_bucket", from, to);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/DoubleId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/DoubleId.java
new file mode 100644
index 00000000000..c6f0b15feb2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/DoubleId.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link Double}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class DoubleId extends ValueGroupId<Double> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The identifying double.
+     */
+    public DoubleId(Double value) {
+        super("double", value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java b/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java
new file mode 100644
index 00000000000..ddf8fe6140d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java
@@ -0,0 +1,83 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.search.result.Relevance;
+
+/**
+ * This class represents a single group in the grouping result model. A group may contain any number of results (stored
+ * as fields, use {@link #getField(String)} to access), {@link GroupList} and {@link HitList}. Use the {@link
+ * com.yahoo.search.grouping.GroupingRequest#getResultGroup(com.yahoo.search.Result)} to retrieve an instance of this.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class Group extends HitGroup {
+
+    private static final long serialVersionUID = 2122928012157537800L;
+    private final GroupId groupId;
+
+    /**
+     * Creates a new instance of this class.
+     *
+     * @param groupId The id to assign to this group.
+     * @param rel     The relevance of this group.
+     */
+    public Group(GroupId groupId, Relevance rel) {
+        super(groupId.toString(), rel);
+        this.groupId = groupId;
+    }
+
+    /**
+     * Returns the id of this group. This is a model of the otherwise flattened {@link #getId() hit id}.
+     *
+     * @return The group id.
+     */
+    public GroupId getGroupId() {
+        return groupId;
+    }
+
+    /**
+     * Returns the {@link HitList} with the given label. The label is the one given to the {@link
+     * com.yahoo.search.grouping.request.EachOperation} that generated the list. This method returns null if no such
+     * list was found.
+     *
+     * @param label The label of the list to return.
+     * @return The requested list, or null.
+     */
+    public HitList getHitList(String label) {
+        for (Hit hit : this) {
+            if (!(hit instanceof HitList)) {
+                continue;
+            }
+            HitList lst = (HitList)hit;
+            if (!label.equals(lst.getLabel())) {
+                continue;
+            }
+            return lst;
+        }
+        return null;
+    }
+
+    /**
+     * Returns the {@link GroupList} with the given label. The label is the one given to the {@link
+     * com.yahoo.search.grouping.request.EachOperation} that generated the list. This method returns null if no such
+     * list was found.
+     *
+     * @param label The label of the list to return.
+     * @return The requested list, or null.
+     */
+    public GroupList getGroupList(String label) {
+        for (Hit hit : this) {
+            if (!(hit instanceof GroupList)) {
+                continue;
+            }
+            GroupList lst = (GroupList)hit;
+            if (!label.equals(lst.getLabel())) {
+                continue;
+            }
+            return lst;
+        }
+        return null;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/GroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/GroupId.java
new file mode 100644
index 00000000000..a9f5102caea
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/GroupId.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This abstract class represents the id of a single group in the grouping result model. A subclass corresponding to the
+ * evaluation result of generating {@link com.yahoo.search.grouping.request.GroupingExpression} is contained in all
+ * {@link Group} objects. It is used by {@link com.yahoo.search.grouping.GroupingRequest} to identify its root result
+ * group, and by all client code for identifying groups.
+ * <p>
+ * The {@link #toString()} method of this class generates a URI-compatible string on the form
+ * "group:&lt;typeName&gt;:&lt;subclassSpecific&gt;".
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class GroupId {
+
+    private final String type;
+    private final String image;
+
+    protected GroupId(String type, Object... args) {
+        this.type = type;
+
+        StringBuilder image = new StringBuilder("group:");
+        image.append(type);
+        for (Object arg : args) {
+            image.append(":").append(arg);
+        }
+        this.image = image.toString();
+    }
+
+    /**
+     * Returns the type name of this group id. This is the second part of the {@link #toString()} value of this.
+     *
+     * @return The type name.
+     */
+    public String getTypeName() {
+        return type;
+    }
+
+    @Override
+    public String toString() {
+        return image;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/GroupList.java b/container-search/src/main/java/com/yahoo/search/grouping/result/GroupList.java
new file mode 100644
index 00000000000..ee8d7c33fa7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/GroupList.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import com.yahoo.search.Result;
+import com.yahoo.search.grouping.GroupingRequest;
+
+/**
+ * This class represents a labeled group list in the grouping result model. It is contained in {@link Group}, and
+ * contains one or more {@link Group groups} itself, allowing for a hierarchy of grouping results. Use the {@link
+ * GroupingRequest#getResultGroup(Result)} to retrieve grouping results.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class GroupList extends AbstractList {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param label The label to assign to this.
+     */
+    public GroupList(String label) {
+        super("grouplist", label);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java b/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java
new file mode 100644
index 00000000000..abc87a92ab1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/HitList.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import com.yahoo.search.Result;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.result.Hit;
+
+/**
+ * <p>This class represents a labeled hit list in the grouping result model. It is contained in {@link Group}, and
+ * contains one or more {@link Hit hits} itself, making this the parent of leaf nodes in the hierarchy of grouping
+ * results. Use the {@link GroupingRequest#getResultGroup(Result)} to retrieve grouping results.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class HitList extends AbstractList {
+
+    /**
+     * <p>Constructs a new instance of this class.</p>
+     *
+     * @param label The label to assign to this.
+     */
+    public HitList(String label) {
+        super("hitlist", label);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java b/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java
new file mode 100644
index 00000000000..7558af5acb5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java
@@ -0,0 +1,99 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.text.Utf8String;
+import com.yahoo.text.XMLWriter;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * This is a helper class for rendering grouping results.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class HitRenderer {
+
+    private static final Utf8String ATR_LABEL = new Utf8String("label");
+    private static final Utf8String ATR_RELEVANCE = new Utf8String("relevance");
+    private static final Utf8String ATR_TYPE = new Utf8String("type");
+    private static final Utf8String TAG_BUCKET_FROM = new Utf8String("from");
+    private static final Utf8String TAG_BUCKET_TO = new Utf8String("to");
+    private static final Utf8String TAG_CONTINUATION = new Utf8String("continuation");
+    private static final Utf8String TAG_CONTINUATION_ID = new Utf8String("id");
+    private static final Utf8String TAG_GROUP_LIST = new Utf8String("grouplist");
+    private static final Utf8String TAG_GROUP = new Utf8String("group");
+    private static final Utf8String TAG_GROUP_ID = new Utf8String("id");
+    private static final Utf8String TAG_HIT_LIST = new Utf8String("hitlist");
+    private static final Utf8String TAG_OUTPUT = new Utf8String("output");
+
+    /**
+     * Renders the header for the given grouping hit. If the hit is not a grouping hit, this method does nothing and
+     * returns false.
+     * <p>Post-condition if this is a grouping hit: The hit tag is open.
+     *
+     * @param hit    The hit whose header to render.
+     * @param writer The writer to render to.
+     * @return True if the hit was rendered.
+     * @throws IOException Thrown if there was a problem writing.
+     */
+    public static boolean renderHeader(HitGroup hit, XMLWriter writer) throws IOException {
+        if (hit instanceof GroupList) {
+            writer.openTag(TAG_GROUP_LIST).attribute(ATR_LABEL, ((GroupList)hit).getLabel());
+            renderContinuations(((GroupList)hit).continuations(), writer);
+        } else if (hit instanceof Group) {
+            writer.openTag(TAG_GROUP).attribute(ATR_RELEVANCE, hit.getRelevance().toString());
+            renderGroupId(((Group)hit).getGroupId(), writer);
+            if (hit instanceof RootGroup) {
+                renderContinuation(Continuation.THIS_PAGE, ((RootGroup)hit).continuation(), writer);
+            }
+            for (String label : hit.fieldKeys()) {
+                writer.openTag(TAG_OUTPUT).attribute(ATR_LABEL, label).content(hit.getField(label), false).closeTag();
+            }
+        } else if (hit instanceof HitList) {
+            writer.openTag(TAG_HIT_LIST).attribute(ATR_LABEL, ((HitList)hit).getLabel());
+            renderContinuations(((HitList)hit).continuations(), writer);
+        } else {
+            return false;
+        }
+        writer.closeStartTag();
+        return true;
+    }
+
+    private static void renderGroupId(GroupId id, XMLWriter writer) {
+        writer.openTag(TAG_GROUP_ID).attribute(ATR_TYPE, id.getTypeName());
+        if (id instanceof ValueGroupId) {
+            writer.content(getIdValue((ValueGroupId)id), false);
+        } else if (id instanceof BucketGroupId) {
+            BucketGroupId bucketId = (BucketGroupId)id;
+            writer.openTag(TAG_BUCKET_FROM).content(getBucketFrom(bucketId), false).closeTag();
+            writer.openTag(TAG_BUCKET_TO).content(getBucketTo(bucketId), false).closeTag();
+        }
+        writer.closeTag();
+    }
+
+    private static Object getIdValue(ValueGroupId id) {
+        return id instanceof RawId ? Arrays.toString(((RawId)id).getValue()) : id.getValue();
+    }
+
+    private static Object getBucketFrom(BucketGroupId id) {
+        return id instanceof RawBucketId ? Arrays.toString(((RawBucketId)id).getFrom()) : id.getFrom();
+    }
+
+    private static Object getBucketTo(BucketGroupId id) {
+        return id instanceof RawBucketId ? Arrays.toString(((RawBucketId)id).getTo()) : id.getTo();
+    }
+
+    private static void renderContinuations(Map<String, Continuation> continuations, XMLWriter writer) {
+        for (Map.Entry<String, Continuation> entry : continuations.entrySet()) {
+            renderContinuation(entry.getKey(), entry.getValue(), writer);
+        }
+    }
+
+    private static void renderContinuation(String id, Continuation continuation, XMLWriter writer) {
+        writer.openTag(TAG_CONTINUATION).attribute(TAG_CONTINUATION_ID, id).content(continuation, false).closeTag();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/LongBucketId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/LongBucketId.java
new file mode 100644
index 00000000000..14ced353b67
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/LongBucketId.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link
+ * com.yahoo.search.grouping.request.LongBucket}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LongBucketId extends BucketGroupId<Long> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from The identifying inclusive-from long.
+     * @param to   The identifying exclusive-to long.
+     */
+    public LongBucketId(Long from, Long to) {
+        super("long_bucket", from, to);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/LongId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/LongId.java
new file mode 100644
index 00000000000..18d2098a5a1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/LongId.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link Long}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class LongId extends ValueGroupId<Long> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The identifying long.
+     */
+    public LongId(Long value) {
+        super("long", value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/NullId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/NullId.java
new file mode 100644
index 00000000000..a6473837c76
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/NullId.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is in {@link Group} instances where the identifying expression evaluated to null. For example, hits that
+ * fall outside the buckets of a {@link com.yahoo.search.grouping.request.PredefinedFunction} are added to an
+ * auto-generated group with this id.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class NullId extends GroupId {
+
+    /**
+     * Constructs a new instance of this class.
+     */
+    public NullId() {
+        super("null");
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/RawBucketId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/RawBucketId.java
new file mode 100644
index 00000000000..bb0dae9d6b8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/RawBucketId.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import java.util.Arrays;
+
+/**
+ * This class is used in {@link Group} instances where the identifying
+ * expression evaluated to a {@link com.yahoo.search.grouping.request.RawBucket}.
+ *
+ * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a>
+ */
+public class RawBucketId extends BucketGroupId<byte[]> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from The identifying inclusive-from raw buffer.
+     * @param to   The identifying exclusive-to raw buffer.
+     */
+    public RawBucketId(byte[] from, byte[] to) {
+        super("raw_bucket", from, Arrays.toString(from), to, Arrays.toString(to));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/RawId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/RawId.java
new file mode 100644
index 00000000000..48e9c6e4523
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/RawId.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import java.util.Arrays;
+
+/**
+ * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link Byte} array.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RawId extends ValueGroupId<byte[]> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The identifying byte array.
+     */
+    public RawId(byte[] value) {
+        super("raw", value, Arrays.toString(value));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/RootGroup.java b/container-search/src/main/java/com/yahoo/search/grouping/result/RootGroup.java
new file mode 100644
index 00000000000..238f9ec68f3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/RootGroup.java
@@ -0,0 +1,25 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.result.Relevance;
+
+/**
+ * This class represents the root {@link Group} in the grouping result model. This class adds a {@link Continuation}
+ * object that can be used to paginate the result.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RootGroup extends Group {
+
+    private final Continuation continuation;
+
+    public RootGroup(int id, Continuation continuation) {
+        super(new RootId(id), new Relevance(1.0));
+        this.continuation = continuation;
+    }
+
+    public Continuation continuation() {
+        return continuation;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/RootId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/RootId.java
new file mode 100644
index 00000000000..ebf3152646a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/RootId.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is used in {@link RootGroup} instances.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class RootId extends GroupId {
+
+    public RootId(int id) {
+        super("root", id);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/StringBucketId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/StringBucketId.java
new file mode 100644
index 00000000000..0b4459aa4b6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/StringBucketId.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link
+ * com.yahoo.search.grouping.request.StringBucket}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StringBucketId extends BucketGroupId<String> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param from The identifying inclusive-from string.
+     * @param to   The identifying exclusive-to string.
+     */
+    public StringBucketId(String from, String to) {
+        super("string_bucket", from, to);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/StringId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/StringId.java
new file mode 100644
index 00000000000..0a82b98af44
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/StringId.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+/**
+ * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link String}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class StringId extends ValueGroupId<String> {
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param value The identifying string.
+     */
+    public StringId(String value) {
+        super("string", value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java
new file mode 100644
index 00000000000..f6e815b231c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.result;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * This abstract class is used in {@link Group} instances where the identifying expression evaluated to a singe value.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public abstract class ValueGroupId<T> extends GroupId {
+
+    private final T value;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param type  The type of this id's value.
+     * @param value The identifying value.
+     */
+    public ValueGroupId(String type, T value) {
+        this(type, value, String.valueOf(value.toString()));
+    }
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param type       The type of this id's value.
+     * @param value      The identifying value.
+     * @param valueImage The String representation of the <tt>value</tt> argument.
+     */
+    public ValueGroupId(String type, T value, String valueImage) {
+        super(type, valueImage);
+        this.value = value;
+    }
+
+    /**
+     * Returns the identifying value.
+     *
+     * @return The value.
+     */
+    public T getValue() {
+        return value;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/package-info.java b/container-search/src/main/java/com/yahoo/search/grouping/result/package-info.java
new file mode 100644
index 00000000000..6c70f67971d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/result/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.grouping.result;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/CompositeContinuation.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/CompositeContinuation.java
new file mode 100644
index 00000000000..e8efce2d0bc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/CompositeContinuation.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.search.grouping.Continuation;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class CompositeContinuation extends EncodableContinuation implements Iterable<EncodableContinuation> {
+
+    private final List<EncodableContinuation> children = new ArrayList<>();
+
+    public CompositeContinuation add(EncodableContinuation child) {
+        children.add(child);
+        return this;
+    }
+
+    @Override
+    public Iterator<EncodableContinuation> iterator() {
+        return children.iterator();
+    }
+
+    @Override
+    public int hashCode() {
+        return children.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof CompositeContinuation && children.equals(((CompositeContinuation)obj).children);
+    }
+
+    @Override
+    public void encode(IntegerEncoder out) {
+        for (EncodableContinuation child : children) {
+            child.encode(out);
+        }
+    }
+
+    public static CompositeContinuation decode(IntegerDecoder from) {
+        CompositeContinuation ret = new CompositeContinuation();
+        while (from.hasNext()) {
+            ret.add(OffsetContinuation.decode(from));
+        }
+        return ret;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ContinuationDecoder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ContinuationDecoder.java
new file mode 100644
index 00000000000..a8779be09c2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ContinuationDecoder.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.search.grouping.Continuation;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public class ContinuationDecoder {
+
+    public static Continuation decode(String str) {
+        return CompositeContinuation.decode(new IntegerDecoder(str));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/EncodableContinuation.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/EncodableContinuation.java
new file mode 100644
index 00000000000..ca059cbe1fe
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/EncodableContinuation.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.search.grouping.Continuation;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+abstract class EncodableContinuation extends Continuation {
+
+    public abstract void encode(IntegerEncoder out);
+
+    @Override
+    public final String toString() {
+        IntegerEncoder encoder = new IntegerEncoder();
+        encode(encoder);
+        return encoder.toString();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java
new file mode 100644
index 00000000000..9de1c902be1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java
@@ -0,0 +1,598 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.search.grouping.request.AddFunction;
+import com.yahoo.search.grouping.request.AggregatorNode;
+import com.yahoo.search.grouping.request.AndFunction;
+import com.yahoo.search.grouping.request.ArrayAtLookup;
+import com.yahoo.search.grouping.request.AttributeFunction;
+import com.yahoo.search.grouping.request.AttributeValue;
+import com.yahoo.search.grouping.request.AvgAggregator;
+import com.yahoo.search.grouping.request.BucketValue;
+import com.yahoo.search.grouping.request.CatFunction;
+import com.yahoo.search.grouping.request.ConstantValue;
+import com.yahoo.search.grouping.request.CountAggregator;
+import com.yahoo.search.grouping.request.DateFunction;
+import com.yahoo.search.grouping.request.DayOfMonthFunction;
+import com.yahoo.search.grouping.request.DayOfWeekFunction;
+import com.yahoo.search.grouping.request.DayOfYearFunction;
+import com.yahoo.search.grouping.request.DebugWaitFunction;
+import com.yahoo.search.grouping.request.DivFunction;
+import com.yahoo.search.grouping.request.DocIdNsSpecificValue;
+import com.yahoo.search.grouping.request.DoubleValue;
+import com.yahoo.search.grouping.request.FixedWidthFunction;
+import com.yahoo.search.grouping.request.GroupingExpression;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.grouping.request.HourOfDayFunction;
+import com.yahoo.search.grouping.request.InfiniteValue;
+import com.yahoo.search.grouping.request.InterpolatedLookup;
+import com.yahoo.search.grouping.request.LongValue;
+import com.yahoo.search.grouping.request.MathACosFunction;
+import com.yahoo.search.grouping.request.MathACosHFunction;
+import com.yahoo.search.grouping.request.MathASinFunction;
+import com.yahoo.search.grouping.request.MathASinHFunction;
+import com.yahoo.search.grouping.request.MathATanFunction;
+import com.yahoo.search.grouping.request.MathATanHFunction;
+import com.yahoo.search.grouping.request.MathCbrtFunction;
+import com.yahoo.search.grouping.request.MathCosFunction;
+import com.yahoo.search.grouping.request.MathCosHFunction;
+import com.yahoo.search.grouping.request.MathExpFunction;
+import com.yahoo.search.grouping.request.MathFloorFunction;
+import com.yahoo.search.grouping.request.MathHypotFunction;
+import com.yahoo.search.grouping.request.MathLog10Function;
+import com.yahoo.search.grouping.request.MathLog1pFunction;
+import com.yahoo.search.grouping.request.MathLogFunction;
+import com.yahoo.search.grouping.request.MathPowFunction;
+import com.yahoo.search.grouping.request.MathSinFunction;
+import com.yahoo.search.grouping.request.MathSinHFunction;
+import com.yahoo.search.grouping.request.MathSqrtFunction;
+import com.yahoo.search.grouping.request.MathTanFunction;
+import com.yahoo.search.grouping.request.MathTanHFunction;
+import com.yahoo.search.grouping.request.MaxAggregator;
+import com.yahoo.search.grouping.request.MaxFunction;
+import com.yahoo.search.grouping.request.Md5Function;
+import com.yahoo.search.grouping.request.MinAggregator;
+import com.yahoo.search.grouping.request.MinFunction;
+import com.yahoo.search.grouping.request.MinuteOfHourFunction;
+import com.yahoo.search.grouping.request.ModFunction;
+import com.yahoo.search.grouping.request.MonthOfYearFunction;
+import com.yahoo.search.grouping.request.MulFunction;
+import com.yahoo.search.grouping.request.NegFunction;
+import com.yahoo.search.grouping.request.NormalizeSubjectFunction;
+import com.yahoo.search.grouping.request.NowFunction;
+import com.yahoo.search.grouping.request.OrFunction;
+import com.yahoo.search.grouping.request.PredefinedFunction;
+import com.yahoo.search.grouping.request.RawValue;
+import com.yahoo.search.grouping.request.RelevanceValue;
+import com.yahoo.search.grouping.request.ReverseFunction;
+import com.yahoo.search.grouping.request.SecondOfMinuteFunction;
+import com.yahoo.search.grouping.request.SizeFunction;
+import com.yahoo.search.grouping.request.SortFunction;
+import com.yahoo.search.grouping.request.StrCatFunction;
+import com.yahoo.search.grouping.request.StrLenFunction;
+import com.yahoo.search.grouping.request.StringValue;
+import com.yahoo.search.grouping.request.SubFunction;
+import com.yahoo.search.grouping.request.SumAggregator;
+import com.yahoo.search.grouping.request.SummaryValue;
+import com.yahoo.search.grouping.request.ToDoubleFunction;
+import com.yahoo.search.grouping.request.ToLongFunction;
+import com.yahoo.search.grouping.request.ToRawFunction;
+import com.yahoo.search.grouping.request.ToStringFunction;
+import com.yahoo.search.grouping.request.UcaFunction;
+import com.yahoo.search.grouping.request.XorAggregator;
+import com.yahoo.search.grouping.request.XorBitFunction;
+import com.yahoo.search.grouping.request.XorFunction;
+import com.yahoo.search.grouping.request.YearFunction;
+import com.yahoo.search.grouping.request.YmumValue;
+import com.yahoo.search.grouping.request.ZCurveXFunction;
+import com.yahoo.search.grouping.request.ZCurveYFunction;
+
+import com.yahoo.searchlib.aggregation.AggregationResult;
+import com.yahoo.searchlib.aggregation.AverageAggregationResult;
+import com.yahoo.searchlib.aggregation.CountAggregationResult;
+import com.yahoo.searchlib.aggregation.ExpressionCountAggregationResult;
+import com.yahoo.searchlib.aggregation.HitsAggregationResult;
+import com.yahoo.searchlib.aggregation.MaxAggregationResult;
+import com.yahoo.searchlib.aggregation.MinAggregationResult;
+import com.yahoo.searchlib.aggregation.SumAggregationResult;
+import com.yahoo.searchlib.aggregation.XorAggregationResult;
+
+import com.yahoo.searchlib.expression.AddFunctionNode;
+import com.yahoo.searchlib.expression.AggregationRefNode;
+import com.yahoo.searchlib.expression.AndFunctionNode;
+import com.yahoo.searchlib.expression.ArrayAtLookupNode;
+import com.yahoo.searchlib.expression.AttributeNode;
+import com.yahoo.searchlib.expression.BucketResultNode;
+import com.yahoo.searchlib.expression.CatFunctionNode;
+import com.yahoo.searchlib.expression.ConstantNode;
+import com.yahoo.searchlib.expression.DebugWaitFunctionNode;
+import com.yahoo.searchlib.expression.DivideFunctionNode;
+import com.yahoo.searchlib.expression.ExpressionNode;
+import com.yahoo.searchlib.expression.FixedWidthBucketFunctionNode;
+import com.yahoo.searchlib.expression.FloatBucketResultNode;
+import com.yahoo.searchlib.expression.FloatBucketResultNodeVector;
+import com.yahoo.searchlib.expression.FloatResultNode;
+import com.yahoo.searchlib.expression.GetDocIdNamespaceSpecificFunctionNode;
+import com.yahoo.searchlib.expression.GetYMUMChecksumFunctionNode;
+import com.yahoo.searchlib.expression.IntegerBucketResultNode;
+import com.yahoo.searchlib.expression.IntegerBucketResultNodeVector;
+import com.yahoo.searchlib.expression.IntegerResultNode;
+import com.yahoo.searchlib.expression.InterpolatedLookupNode;
+import com.yahoo.searchlib.expression.MD5BitFunctionNode;
+import com.yahoo.searchlib.expression.MathFunctionNode;
+import com.yahoo.searchlib.expression.MaxFunctionNode;
+import com.yahoo.searchlib.expression.MinFunctionNode;
+import com.yahoo.searchlib.expression.ModuloFunctionNode;
+import com.yahoo.searchlib.expression.MultiArgFunctionNode;
+import com.yahoo.searchlib.expression.MultiplyFunctionNode;
+import com.yahoo.searchlib.expression.NegateFunctionNode;
+import com.yahoo.searchlib.expression.NormalizeSubjectFunctionNode;
+import com.yahoo.searchlib.expression.NumElemFunctionNode;
+import com.yahoo.searchlib.expression.OrFunctionNode;
+import com.yahoo.searchlib.expression.RangeBucketPreDefFunctionNode;
+import com.yahoo.searchlib.expression.RawBucketResultNode;
+import com.yahoo.searchlib.expression.RawBucketResultNodeVector;
+import com.yahoo.searchlib.expression.RawResultNode;
+import com.yahoo.searchlib.expression.RelevanceNode;
+import com.yahoo.searchlib.expression.ResultNodeVector;
+import com.yahoo.searchlib.expression.ReverseFunctionNode;
+import com.yahoo.searchlib.expression.SortFunctionNode;
+import com.yahoo.searchlib.expression.StrCatFunctionNode;
+import com.yahoo.searchlib.expression.StrLenFunctionNode;
+import com.yahoo.searchlib.expression.StringBucketResultNode;
+import com.yahoo.searchlib.expression.StringBucketResultNodeVector;
+import com.yahoo.searchlib.expression.StringResultNode;
+import com.yahoo.searchlib.expression.TimeStampFunctionNode;
+import com.yahoo.searchlib.expression.ToFloatFunctionNode;
+import com.yahoo.searchlib.expression.ToIntFunctionNode;
+import com.yahoo.searchlib.expression.ToRawFunctionNode;
+import com.yahoo.searchlib.expression.ToStringFunctionNode;
+import com.yahoo.searchlib.expression.UcaFunctionNode;
+import com.yahoo.searchlib.expression.XorBitFunctionNode;
+import com.yahoo.searchlib.expression.XorFunctionNode;
+import com.yahoo.searchlib.expression.ZCurveFunctionNode;
+
+/**
+ * This is a helper class for {@link RequestBuilder} that offloads the code to convert {@link GroupingExpression} type
+ * objects to back-end specific expressions. This is a straightforward one-to-one conversion.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class ExpressionConverter {
+
+    public static final String DEFAULT_SUMMARY_NAME = "";
+    public static final int DEFAULT_TIME_OFFSET = 0;
+    private String defaultSummaryName = DEFAULT_SUMMARY_NAME;
+    private int timeOffset = DEFAULT_TIME_OFFSET;
+
+    /**
+     * Sets the summary name to use when converting {@link SummaryValue} that was created without an explicit name.
+     *
+     * @param summaryName The default summary name to use.
+     * @return This, to allow chaining.
+     */
+    public ExpressionConverter setDefaultSummaryName(String summaryName) {
+        defaultSummaryName = summaryName;
+        return this;
+    }
+
+    /**
+     * Sets an offset to use for all time-based grouping expressions.
+     *
+     * @param millis The offset in milliseconds.
+     * @return This, to allow chaining.
+     */
+    public ExpressionConverter setTimeOffset(int millis) {
+        this.timeOffset = millis / 1000;
+        return this;
+    }
+
+    /**
+     * Converts the given ast type grouping expression to a corresponding back-end type aggregation result.
+     *
+     * @param exp The expression to convert.
+     * @return The corresponding back-end result.
+     * @throws UnsupportedOperationException Thrown if the given expression could not be converted.
+     */
+    public AggregationResult toAggregationResult(GroupingExpression exp) {
+        int level = exp.getLevel();
+        // Is aggregating on list of groups?
+        if (level > 1) {
+            /*
+            * The below aggregator operates on lists of groups in the query language world.
+            * Internally, it operates on hits (by evaluating the group-by expression for each hit).
+            * The group-by expression is passed to the aggregator by RequestBuilder.
+            */
+            if (exp instanceof CountAggregator) {
+                return new ExpressionCountAggregationResult();
+            }
+            throw new UnsupportedOperationException(
+                    "Can not aggregate on " + GroupingOperation.getLevelDesc(level) + ".");
+        }
+        if (exp instanceof AvgAggregator) {
+            return new AverageAggregationResult()
+                    .setExpression(toExpressionNode(((AvgAggregator)exp).getExpression()));
+        }
+        if (exp instanceof CountAggregator) {
+            return new CountAggregationResult()
+                    .setExpression(new ConstantNode(new IntegerResultNode(0)));
+        }
+        if (exp instanceof MaxAggregator) {
+            return new MaxAggregationResult()
+                    .setExpression(toExpressionNode(((MaxAggregator)exp).getExpression()));
+        }
+        if (exp instanceof MinAggregator) {
+            return new MinAggregationResult()
+                    .setExpression(toExpressionNode(((MinAggregator)exp).getExpression()));
+        }
+        if (exp instanceof SumAggregator) {
+            return new SumAggregationResult()
+                    .setExpression(toExpressionNode(((SumAggregator)exp).getExpression()));
+        }
+        if (exp instanceof SummaryValue) {
+            String summaryName = ((SummaryValue)exp).getSummaryName();
+            return new HitsAggregationResult()
+                    .setSummaryClass(summaryName != null ? summaryName : defaultSummaryName)
+                    .setExpression(new ConstantNode(new IntegerResultNode(0)));
+        }
+        if (exp instanceof XorAggregator) {
+            return new XorAggregationResult()
+                    .setExpression(toExpressionNode(((XorAggregator)exp).getExpression()));
+        }
+        throw new UnsupportedOperationException("Can not convert '" + exp + "' to an aggregator.");
+    }
+
+    /**
+     * Converts the given ast type grouping expression to a corresponding back-end type expression.
+     *
+     * @param exp The expression to convert.
+     * @return The corresponding back-end expression.
+     * @throws UnsupportedOperationException Thrown if the given expression could not be converted.
+     */
+    public ExpressionNode toExpressionNode(GroupingExpression exp) {
+        if (exp instanceof AddFunction) {
+            return addArguments(new AddFunctionNode(), (AddFunction)exp);
+        }
+        if (exp instanceof AggregatorNode) {
+            return new AggregationRefNode(toAggregationResult(exp));
+        }
+        if (exp instanceof AndFunction) {
+            return addArguments(new AndFunctionNode(), (AndFunction)exp);
+        }
+        if (exp instanceof AttributeValue) {
+            return new AttributeNode(((AttributeValue)exp).getAttributeName());
+        }
+        if (exp instanceof AttributeFunction) {
+            return new AttributeNode(((AttributeFunction)exp).getAttributeName());
+        }
+        if (exp instanceof CatFunction) {
+            return addArguments(new CatFunctionNode(), (CatFunction)exp);
+        }
+        if (exp instanceof DebugWaitFunction) {
+            return new DebugWaitFunctionNode(toExpressionNode(((DebugWaitFunction)exp).getArg(0)),
+                                             ((DebugWaitFunction)exp).getWaitTime(),
+                                             ((DebugWaitFunction)exp).getBusyWait());
+        }
+        if (exp instanceof DocIdNsSpecificValue) {
+            return new GetDocIdNamespaceSpecificFunctionNode();
+        }
+        if (exp instanceof DoubleValue) {
+            return new ConstantNode(new FloatResultNode(((DoubleValue)exp).getValue()));
+        }
+        if (exp instanceof DivFunction) {
+            return addArguments(new DivideFunctionNode(), (DivFunction)exp);
+        }
+        if (exp instanceof FixedWidthFunction) {
+            Number w = ((FixedWidthFunction)exp).getWidth();
+            return new FixedWidthBucketFunctionNode(
+                    w instanceof Double ? new FloatResultNode(w.doubleValue()) : new IntegerResultNode(w.longValue()),
+                    toExpressionNode(((FixedWidthFunction)exp).getArg(0)));
+        }
+        if (exp instanceof LongValue) {
+            return new ConstantNode(new IntegerResultNode(((LongValue)exp).getValue()));
+        }
+        if (exp instanceof MaxFunction) {
+            return addArguments(new MaxFunctionNode(), (MaxFunction)exp);
+        }
+        if (exp instanceof Md5Function) {
+            return new MD5BitFunctionNode().setNumBits(((Md5Function)exp).getNumBits())
+                                           .addArg(toExpressionNode(((Md5Function)exp).getArg(0)));
+        }
+        if (exp instanceof UcaFunction) {
+            UcaFunction uca = (UcaFunction)exp;
+            return new UcaFunctionNode(toExpressionNode(uca.getArg(0)), uca.getLocale(), uca.getStrength());
+        }
+        if (exp instanceof MinFunction) {
+            return addArguments(new MinFunctionNode(), (MinFunction)exp);
+        }
+        if (exp instanceof ModFunction) {
+            return addArguments(new ModuloFunctionNode(), (ModFunction)exp);
+        }
+        if (exp instanceof MulFunction) {
+            return addArguments(new MultiplyFunctionNode(), (MulFunction)exp);
+        }
+        if (exp instanceof NegFunction) {
+            return new NegateFunctionNode(toExpressionNode(((NegFunction)exp).getArg(0)));
+        }
+        if (exp instanceof NormalizeSubjectFunction) {
+            return new NormalizeSubjectFunctionNode(toExpressionNode(((NormalizeSubjectFunction)exp).getArg(0)));
+        }
+        if (exp instanceof NowFunction) {
+            return new ConstantNode(new IntegerResultNode(System.currentTimeMillis() / 1000));
+        }
+        if (exp instanceof OrFunction) {
+            return addArguments(new OrFunctionNode(), (OrFunction)exp);
+        }
+        if (exp instanceof PredefinedFunction) {
+            return new RangeBucketPreDefFunctionNode(toBucketList((PredefinedFunction)exp),
+                                                     toExpressionNode(((PredefinedFunction)exp).getArg(0)));
+        }
+        if (exp instanceof RelevanceValue) {
+            return new RelevanceNode();
+        }
+        if (exp instanceof ReverseFunction) {
+            return new ReverseFunctionNode(toExpressionNode(((ReverseFunction)exp).getArg(0)));
+        }
+        if (exp instanceof SizeFunction) {
+            return new NumElemFunctionNode(toExpressionNode(((SizeFunction)exp).getArg(0)));
+        }
+        if (exp instanceof SortFunction) {
+            return new SortFunctionNode(toExpressionNode(((SortFunction)exp).getArg(0)));
+        }
+        if (exp instanceof ArrayAtLookup) {
+            ArrayAtLookup aal = (ArrayAtLookup) exp;
+            return new ArrayAtLookupNode(aal.getAttributeName(), toExpressionNode(aal.getIndexArgument()));
+        }
+        if (exp instanceof InterpolatedLookup) {
+            InterpolatedLookup sarl = (InterpolatedLookup) exp;
+            return new InterpolatedLookupNode(sarl.getAttributeName(), toExpressionNode(sarl.getLookupArgument()));
+        }
+        if (exp instanceof StrCatFunction) {
+            return addArguments(new StrCatFunctionNode(), (StrCatFunction)exp);
+        }
+        if (exp instanceof StringValue) {
+            return new ConstantNode(new StringResultNode(((StringValue)exp).getValue()));
+        }
+        if (exp instanceof StrLenFunction) {
+            return new StrLenFunctionNode(toExpressionNode(((StrLenFunction)exp).getArg(0)));
+        }
+        if (exp instanceof SubFunction) {
+            return toSubNode((SubFunction)exp);
+        }
+        if (exp instanceof ToDoubleFunction) {
+            return new ToFloatFunctionNode(toExpressionNode(((ToDoubleFunction)exp).getArg(0)));
+        }
+        if (exp instanceof ToLongFunction) {
+            return new ToIntFunctionNode(toExpressionNode(((ToLongFunction)exp).getArg(0)));
+        }
+        if (exp instanceof ToRawFunction) {
+            return new ToRawFunctionNode(toExpressionNode(((ToRawFunction)exp).getArg(0)));
+        }
+        if (exp instanceof ToStringFunction) {
+            return new ToStringFunctionNode(toExpressionNode(((ToStringFunction)exp).getArg(0)));
+        }
+        if (exp instanceof DateFunction) {
+            StrCatFunctionNode ret = new StrCatFunctionNode();
+            GroupingExpression arg = ((DateFunction)exp).getArg(0);
+            ret.addArg(new ToStringFunctionNode(toTime(arg, TimeStampFunctionNode.TimePart.Year)));
+            ret.addArg(new ConstantNode(new StringResultNode("-")));
+            ret.addArg(new ToStringFunctionNode(toTime(arg, TimeStampFunctionNode.TimePart.Month)));
+            ret.addArg(new ConstantNode(new StringResultNode("-")));
+            ret.addArg(new ToStringFunctionNode(toTime(arg, TimeStampFunctionNode.TimePart.MonthDay)));
+            return ret;
+        }
+        if (exp instanceof MathSqrtFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathSqrtFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.SQRT);
+        }
+        if (exp instanceof MathCbrtFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathCbrtFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.CBRT);
+        }
+        if (exp instanceof MathLogFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathLogFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.LOG);
+        }
+        if (exp instanceof MathLog1pFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathLog1pFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.LOG1P);
+        }
+        if (exp instanceof MathLog10Function) {
+            return new MathFunctionNode(toExpressionNode(((MathLog10Function)exp).getArg(0)),
+                                        MathFunctionNode.Function.LOG10);
+        }
+        if (exp instanceof MathExpFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathExpFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.EXP);
+        }
+        if (exp instanceof MathPowFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathPowFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.POW)
+                    .addArg(toExpressionNode(((MathPowFunction)exp).getArg(1)));
+        }
+        if (exp instanceof MathHypotFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathHypotFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.HYPOT)
+                    .addArg(toExpressionNode(((MathHypotFunction)exp).getArg(1)));
+        }
+        if (exp instanceof MathSinFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathSinFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.SIN);
+        }
+        if (exp instanceof MathASinFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathASinFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.ASIN);
+        }
+        if (exp instanceof MathCosFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathCosFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.COS);
+        }
+        if (exp instanceof MathACosFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathACosFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.ACOS);
+        }
+        if (exp instanceof MathTanFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathTanFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.TAN);
+        }
+        if (exp instanceof MathATanFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathATanFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.ATAN);
+        }
+        if (exp instanceof MathSinHFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathSinHFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.SINH);
+        }
+        if (exp instanceof MathASinHFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathASinHFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.ASINH);
+        }
+        if (exp instanceof MathCosHFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathCosHFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.COSH);
+        }
+        if (exp instanceof MathACosHFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathACosHFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.ACOSH);
+        }
+        if (exp instanceof MathTanHFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathTanHFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.TANH);
+        }
+        if (exp instanceof MathATanHFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathATanHFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.ATANH);
+        }
+        if (exp instanceof MathFloorFunction) {
+            return new MathFunctionNode(toExpressionNode(((MathFloorFunction)exp).getArg(0)),
+                                        MathFunctionNode.Function.FLOOR);
+        }
+        if (exp instanceof ZCurveXFunction) {
+            return new ZCurveFunctionNode(toExpressionNode(((ZCurveXFunction)exp).getArg(0)),
+                                          ZCurveFunctionNode.Dimension.X);
+        }
+        if (exp instanceof ZCurveYFunction) {
+            return new ZCurveFunctionNode(toExpressionNode(((ZCurveYFunction)exp).getArg(0)),
+                                          ZCurveFunctionNode.Dimension.Y);
+        }
+        if (exp instanceof DayOfMonthFunction) {
+            return toTime(((DayOfMonthFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.MonthDay);
+        }
+        if (exp instanceof DayOfWeekFunction) {
+            return toTime(((DayOfWeekFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.WeekDay);
+        }
+        if (exp instanceof DayOfYearFunction) {
+            return toTime(((DayOfYearFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.YearDay);
+        }
+        if (exp instanceof HourOfDayFunction) {
+            return toTime(((HourOfDayFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.Hour);
+        }
+        if (exp instanceof MinuteOfHourFunction) {
+            return toTime(((MinuteOfHourFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.Minute);
+        }
+        if (exp instanceof MonthOfYearFunction) {
+            return toTime(((MonthOfYearFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.Month);
+        }
+        if (exp instanceof SecondOfMinuteFunction) {
+            return toTime(((SecondOfMinuteFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.Second);
+        }
+        if (exp instanceof YearFunction) {
+            return toTime(((YearFunction)exp).getArg(0), TimeStampFunctionNode.TimePart.Year);
+        }
+        if (exp instanceof XorFunction) {
+            return addArguments(new XorFunctionNode(), (XorFunction)exp);
+        }
+        if (exp instanceof XorBitFunction) {
+            return new XorBitFunctionNode().setNumBits(((XorBitFunction)exp).getNumBits())
+                                           .addArg(toExpressionNode(((XorBitFunction)exp).getArg(0)));
+        }
+        if (exp instanceof YmumValue) {
+            return new GetYMUMChecksumFunctionNode();
+        }
+        throw new UnsupportedOperationException("Can not convert '" + exp + "' of class " + exp.getClass().getName() +
+                                                " to an expression.");
+    }
+
+    private TimeStampFunctionNode toTime(GroupingExpression arg, TimeStampFunctionNode.TimePart timePart) {
+        if (timeOffset == 0) {
+            return new TimeStampFunctionNode(toExpressionNode(arg), timePart, true);
+        }
+        AddFunctionNode exp = new AddFunctionNode();
+        exp.addArg(toExpressionNode(arg));
+        exp.addArg(new ConstantNode(new IntegerResultNode(timeOffset)));
+        return new TimeStampFunctionNode(exp, timePart, true);
+    }
+
+    private MultiArgFunctionNode addArguments(MultiArgFunctionNode ret, Iterable<GroupingExpression> lst) {
+        for (GroupingExpression exp : lst) {
+            ret.addArg(toExpressionNode(exp));
+        }
+        return ret;
+    }
+
+    private MultiArgFunctionNode toSubNode(Iterable<GroupingExpression> lst) {
+        MultiArgFunctionNode ret = new AddFunctionNode();
+        int i = 0;
+        for (GroupingExpression exp : lst) {
+            ExpressionNode node = toExpressionNode(exp);
+            if (++i > 1) {
+                node = new NegateFunctionNode(node);
+            }
+            ret.addArg(node);
+        }
+        return ret;
+    }
+
+    private ResultNodeVector toBucketList(PredefinedFunction fnc) {
+        ResultNodeVector ret = null;
+        for (int i = 0, len = fnc.getNumBuckets(); i < len; ++i) {
+            BucketResultNode bucket = toBucket(fnc.getBucket(i));
+            if (ret == null) {
+                if (bucket instanceof FloatBucketResultNode) {
+                    ret = new FloatBucketResultNodeVector();
+                } else if (bucket instanceof IntegerBucketResultNode) {
+                    ret = new IntegerBucketResultNodeVector();
+                } else if (bucket instanceof RawBucketResultNode) {
+                    ret = new RawBucketResultNodeVector();
+                } else {
+                    ret = new StringBucketResultNodeVector();
+                }
+            }
+            ret.add(bucket);
+        }
+        return ret;
+    }
+
+    private BucketResultNode toBucket(GroupingExpression exp) {
+        if (!(exp instanceof BucketValue)) {
+            throw new UnsupportedOperationException("Can not convert '" + exp + "' to a bucket.");
+        }
+        ConstantValue<?> begin = ((BucketValue)exp).getFrom();
+        ConstantValue<?> end = ((BucketValue)exp).getTo();
+        if (begin instanceof DoubleValue || end instanceof DoubleValue) {
+            return new FloatBucketResultNode(
+                    begin instanceof InfiniteValue ? FloatResultNode.getNegativeInfinity().getFloat()
+                                                   : Double.valueOf(begin.toString()),
+                    end instanceof InfiniteValue ? FloatResultNode.getPositiveInfinity().getFloat()
+                                                 : Double.valueOf(end.toString()));
+        } else if (begin instanceof LongValue || end instanceof LongValue) {
+            return new IntegerBucketResultNode(
+                    begin instanceof InfiniteValue ? IntegerResultNode.getNegativeInfinity().getInteger()
+                                                   : Long.valueOf(begin.toString()),
+                    end instanceof InfiniteValue ? IntegerResultNode.getPositiveInfinity().getInteger()
+                                                 : Long.valueOf(end.toString()));
+        } else if (begin instanceof StringValue || end instanceof StringValue) {
+            return new StringBucketResultNode(
+                    begin instanceof InfiniteValue ? StringResultNode.getNegativeInfinity()
+                                                   : new StringResultNode((String)begin.getValue()),
+                    end instanceof InfiniteValue ? StringResultNode.getPositiveInfinity()
+                                                 : new StringResultNode((String)end.getValue()));
+        } else {
+            return new RawBucketResultNode(
+                    begin instanceof InfiniteValue ? RawResultNode.getNegativeInfinity()
+                                                   : new RawResultNode(((RawValue)begin).getValue().getBytes()),
+                    end instanceof InfiniteValue ? RawResultNode.getPositiveInfinity()
+                                                 : new RawResultNode(((RawValue)end).getValue().getBytes()));
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java
new file mode 100644
index 00000000000..e5e91f21f5f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java
@@ -0,0 +1,411 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.fastsearch.GroupingListHit;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.QueryCanonicalizer;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.grouping.GroupingValidator;
+import com.yahoo.search.grouping.result.Group;
+import com.yahoo.search.grouping.result.RootGroup;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.searchlib.aggregation.Grouping;
+import com.yahoo.vespa.objects.Identifiable;
+import com.yahoo.vespa.objects.ObjectOperation;
+import com.yahoo.vespa.objects.ObjectPredicate;
+
+/**
+ * Executes the {@link GroupingRequest grouping requests} set up by other searchers. This does the necessary
+ * transformation from the abstract request to Vespa grouping expressions (using {@link RequestBuilder}), and the
+ * corresponding transformation of results (using {@link ResultBuilder}).
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@After({ GroupingValidator.GROUPING_VALIDATED,
+         "com.yahoo.search.querytransform.WandSearcher",
+         "com.yahoo.search.querytransform.BooleanSearcher" })
+@Provides({ GroupingExecutor.COMPONENT_NAME, QueryCanonicalizer.queryCanonicalization } )
+public class GroupingExecutor extends Searcher {
+
+    public final static String COMPONENT_NAME = "GroupingExecutor";
+    private final static CompoundName PROP_GROUPINGLIST = newCompoundName("GroupingList");
+    private final static Logger log = Logger.getLogger(GroupingExecutor.class.getName());
+
+    /**
+     * Constructs a new instance of this searcher without configuration.
+     * This makes the searcher completely useless for searching purposes,
+     * and should only be used for testing its logic.
+     */
+    GroupingExecutor() {
+    }
+
+    /**
+     * Constructs a new instance of this searcher with the given component id.
+     *
+     * @param componentId The identifier to assign to this searcher.
+     */
+    public GroupingExecutor(ComponentId componentId) {
+        super(componentId);
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        String error = QueryCanonicalizer.canonicalize(query);
+        if (error != null) {
+            return new Result(query, ErrorMessage.createIllegalQuery(error));
+        }
+        query.prepare();
+
+        // Retrieve grouping requests from query.
+        List<GroupingRequest> reqList = GroupingRequest.getRequests(query);
+        if (reqList.isEmpty()) {
+            return execution.search(query);
+        }
+
+        // Convert requests to Vespa style grouping.
+        Map<Integer, Grouping> groupingMap = new HashMap<>();
+        List<RequestContext> ctxList = new LinkedList<>();
+        for (GroupingRequest grpRequest : reqList) {
+            ctxList.add(convertRequest(query, grpRequest, groupingMap));
+        }
+        if (groupingMap.isEmpty()) {
+            return execution.search(query);
+        }
+
+        // Perform the necessary passes to execute grouping.
+        Result result = performSearch(query, execution, groupingMap);
+
+        // Convert Vespa style results to hits.
+        HitConverter hitConverter = new HitConverter(this, query);
+        for (RequestContext ctx : ctxList) {
+            RootGroup grp = convertResult(ctx, groupingMap, hitConverter);
+            ctx.request.setResultGroup(grp);
+            result.hits().add(grp);
+        }
+        return result;
+    }
+
+    @Override
+    public void fill(Result result, String summaryClass, Execution execution) {
+        Map<String, Result> summaryMap = new HashMap<>();
+        for (Iterator<Hit> it = result.hits().unorderedDeepIterator(); it.hasNext(); ) {
+            Hit hit = it.next();
+            Object metaData = hit.getSearcherSpecificMetaData(this);
+            String hitSummary = (metaData instanceof String) ? (String)metaData : summaryClass;
+            Result summaryResult = summaryMap.get(hitSummary);
+            if (summaryResult == null) {
+                summaryResult = new Result(result.getQuery());
+                summaryMap.put(hitSummary, summaryResult);
+            }
+            summaryResult.hits().add(hit);
+        }
+        for (Map.Entry<String, Result> entry : summaryMap.entrySet()) {
+            Result res = entry.getValue();
+            execution.fill(res, entry.getKey());
+            ErrorMessage err = res.hits().getError();
+            if (err != null) {
+                result.hits().addError(err);
+            }
+        }
+        Result defaultResult = summaryMap.get(ExpressionConverter.DEFAULT_SUMMARY_NAME);
+        if (defaultResult != null) {
+            // the reason we need to do this fix is that the docsum packet protocol uses null summary class name to
+            // signal that the backend should use its configured default, whereas for grouping it uses the literal
+            // "default" to signal the same
+            for (Hit hit : defaultResult.hits()) {
+                hit.setFilled(null);
+            }
+        }
+    }
+
+    /**
+     * Converts the given {@link GroupingRequest} into a set of {@link Grouping} objects. The returned object holds the
+     * context that corresponds to the given request, whereas the created {@link Grouping} objects are written directly
+     * to the given map.
+     *
+     * @param query The query being executed.
+     * @param req   The request to convert.
+     * @param map   The grouping map to write to.
+     * @return The context required to identify the request results.
+     */
+    private RequestContext convertRequest(Query query, GroupingRequest req, Map<Integer, Grouping> map) {
+        RequestBuilder builder = new RequestBuilder(req.getRequestId());
+        builder.setRootOperation(req.getRootOperation());
+        builder.setDefaultSummaryName(query.getPresentation().getSummary());
+        builder.setTimeZone(req.getTimeZone());
+        builder.addContinuations(req.continuations());
+        builder.build();
+
+        RequestContext ctx = new RequestContext(req, builder.getTransform());
+        List<Grouping> grpList = builder.getRequestList();
+        for (Grouping grp : grpList) {
+            int grpId = map.size();
+            grp.setId(grpId);
+            map.put(grpId, grp);
+            ctx.idList.add(grpId);
+        }
+        return ctx;
+    }
+
+    /**
+     * Converts the results of the given request context into a single {@link Group}.
+     *
+     * @param requestCtx   The context that identifies the results to convert.
+     * @param groupingMap  The map of all {@link Grouping} objects available.
+     * @param hitConverter The converter to use for {@link Hit} conversion.
+     * @return The corresponding root RootGroup.
+     */
+    private RootGroup convertResult(RequestContext requestCtx, Map<Integer, Grouping> groupingMap,
+                                    HitConverter hitConverter) {
+        ResultBuilder builder = new ResultBuilder();
+        builder.setHitConverter(hitConverter);
+        builder.setTransform(requestCtx.transform);
+        builder.setRequestId(requestCtx.request.getRequestId());
+        for (Integer grpId : requestCtx.idList) {
+            builder.addGroupingResult(groupingMap.get(grpId));
+        }
+        builder.build();
+        return builder.getRoot();
+    }
+
+    /**
+     * Performs the actual search passes to complete all the given {@link Grouping} requests. This method uses the
+     * grouping map argument as both an input and an output variable, as the contained {@link Grouping} objects are
+     * updates as results arrive from the back end.
+     *
+     * @param query       The query to execute.
+     * @param execution   The execution context used to run the queries.
+     * @param groupingMap The map of grouping requests to perform.
+     * @return The search result to pass back from this searcher.
+     */
+    private Result performSearch(Query query, Execution execution, Map<Integer, Grouping> groupingMap) {
+        // Determine how many passes to perform.
+        int lastPass = 0;
+        for (Grouping grouping : groupingMap.values()) {
+            if ( ! grouping.useSinglePass()) {
+                lastPass = Math.max(lastPass, grouping.getLevels().size());
+            }
+        }
+
+        // Perform multi-pass query to complete all grouping requests.
+        Item origRoot = query.getModel().getQueryTree().getRoot();
+        int prePassErrors = query.errors().size();
+        Result ret = null;
+        Item baseRoot = origRoot;
+        if (lastPass > 0) {
+            baseRoot = origRoot.clone();
+        }
+        if (query.isTraceable(3) && query.getGroupingSessionCache()) {
+            query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId(true) + "'.", 3);
+        }
+        for (int pass = 0; pass <= lastPass; ++pass) {
+            boolean firstPass = (pass == 0);
+            List<Grouping> passList = getGroupingListForPassN(groupingMap, pass);
+            if (passList.isEmpty()) {
+                throw new RuntimeException("No grouping request for pass " + pass + ", bug!");
+            }
+            if (log.isLoggable(LogLevel.DEBUG)) {
+                for (Grouping grouping : passList) {
+                    log.log(LogLevel.DEBUG, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping);
+                }
+            }
+            Item passRoot;
+            if (firstPass) {
+                passRoot = origRoot;     // Use original query the first time.
+            } else if (pass == lastPass) {
+                passRoot = baseRoot; // Has already been cloned once, use this for last pass.
+            } else {
+                // noinspection ConstantConditions
+                passRoot = baseRoot.clone();
+            }
+            if (query.isTraceable(4) && query.getGroupingSessionCache()) {
+                query.trace("Grouping with session cache '" + query.getGroupingSessionCache() + "' enabled for pass #" + pass + ".", 4);
+            }
+            if (origRoot != passRoot) {
+                query.getModel().getQueryTree().setRoot(passRoot);
+            }
+            setGroupingList(query, passList);
+            Result passResult = execution.search(query);
+            if (passResult.hits().getError() != null) {
+                if (firstPass) {
+                    if (passResult.hits().getErrorHit().errors().size() > prePassErrors ||
+                        passResult.hits().getErrorHit().errors().size() == 0) {
+                        return passResult;
+                    }
+                } else {
+                    return passResult;
+                }
+            }
+            Map<Integer, Grouping> passGroupingMap = mergeGroupingResults(passResult);
+            mergeGroupingMaps(groupingMap, passGroupingMap);
+            if (firstPass) {
+                ret = passResult;
+            }
+        }
+        if (log.isLoggable(LogLevel.DEBUG)) {
+            for (Grouping grouping : groupingMap.values()) {
+                log.log(LogLevel.DEBUG, "Result Grouping(" + grouping.getId() + "): " + grouping);
+            }
+        }
+        return ret;
+    }
+
+    /**
+     * Merges the content of result into state. This needs to be done in order to conserve the context objects contained
+     * in the state as they are not part of the serialized object representation.
+     *
+     * @param state  the current state.
+     * @param result the results from the current pass.
+     */
+    private void mergeGroupingMaps(Map<Integer, Grouping> state, Map<Integer, Grouping> result) {
+        for (Grouping grouping : result.values()) {
+            Grouping old = state.get(grouping.getId());
+            if (old != null) {
+                old.merge(grouping);
+                // no need to invoke postMerge, as state is empty for
+                // current level
+            } else {
+                log.warning("Got grouping result with unknown id: " + grouping);
+            }
+        }
+    }
+
+    /**
+     * Returns a list of {@link Grouping} objects that are to be used for the given pass.
+     *
+     * @param groupingMap The map of all grouping objects.
+     * @param pass        The pass about to be performed.
+     * @return A list of grouping objects.
+     */
+    private List<Grouping> getGroupingListForPassN(Map<Integer, Grouping> groupingMap, int pass) {
+        List<Grouping> ret = new ArrayList<>();
+        for (Grouping grouping : groupingMap.values()) {
+            if (grouping.useSinglePass()) {
+                if (pass == 0) {
+                    grouping.setFirstLevel(0);
+                    grouping.setLastLevel(grouping.getLevels().size());
+                    ret.add(grouping); // more levels to go
+                }
+            } else {
+                if (pass <= grouping.getLevels().size()) {
+                    grouping.setFirstLevel(pass);
+                    grouping.setLastLevel(pass);
+                    ret.add(grouping); // more levels to go
+                }
+            }
+        }
+        return ret;
+    }
+
+    /**
+     * Merges the grouping content of the given result object. The first grouping hit found by iterating over the result
+     * content is kept, and all consecutive matching hits are merged into this.
+     *
+     * @param result The result to traverse.
+     * @return A map of merged grouping objects.
+     */
+    private Map<Integer, Grouping> mergeGroupingResults(Result result) {
+        Map<Integer, Grouping> ret = new HashMap<>();
+        for (Iterator<Hit> i = result.hits().unorderedIterator(); i.hasNext(); ) {
+            Hit hit = i.next();
+            if (hit instanceof GroupingListHit) {
+                ContextInjector injector = new ContextInjector(hit);
+                for (Grouping grp : ((GroupingListHit)hit).getGroupingList()) {
+                    grp.select(injector, injector);
+                    Grouping old = ret.get(grp.getId());
+                    if (old != null) {
+                        old.merge(grp);
+                    } else {
+                        ret.put(grp.getId(), grp);
+                    }
+                }
+                i.remove();
+            }
+        }
+        for (Grouping grouping : ret.values()) {
+            grouping.postMerge();
+        }
+        return ret;
+    }
+
+    /**
+     * Returns the list of {@link Grouping} objects assigned to the given query. If no list has been assigned, this
+     * method returns an empty list.
+     *
+     * @param query The query whose grouping list to return.
+     * @return The list of assigned grouping objects.
+     */
+    @SuppressWarnings({ "unchecked" })
+    public static List<Grouping> getGroupingList(Query query) {
+        Object obj = query.properties().get(PROP_GROUPINGLIST);
+        if (!(obj instanceof List)) {
+            return Collections.emptyList();
+        }
+        return (List<Grouping>)obj;
+    }
+
+    /**
+     * Sets the list of {@link Grouping} objects assigned to the given query. This method overwrites any grouping
+     * objects already assigned to the query.
+     *
+     * @param query The query whose grouping list to set.
+     * @param lst   The grouping list to set.
+     */
+    public static void setGroupingList(Query query, List<Grouping> lst) {
+        query.properties().set(PROP_GROUPINGLIST, lst);
+    }
+
+    private static CompoundName newCompoundName(String name) {
+        return new CompoundName(GroupingExecutor.class.getName() + "." + name);
+    }
+
+    private static class ContextInjector implements ObjectPredicate, ObjectOperation {
+
+        final Object context;
+
+        ContextInjector(Object context) {
+            this.context = context;
+        }
+
+        @Override
+        public boolean check(Object obj) {
+            return com.yahoo.searchlib.aggregation.Hit.class.isInstance(obj);
+        }
+
+        @Override
+        public void execute(Object obj) {
+            ((com.yahoo.searchlib.aggregation.Hit)obj).setContext(context);
+        }
+    }
+
+    private static class RequestContext {
+
+        final List<Integer> idList = new LinkedList<>();
+        final GroupingRequest request;
+        final GroupingTransform transform;
+
+        RequestContext(GroupingRequest request, GroupingTransform transform) {
+            this.request = request;
+            this.transform = transform;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingTransform.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingTransform.java
new file mode 100644
index 00000000000..928b0ebd22f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingTransform.java
@@ -0,0 +1,137 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.search.grouping.Continuation;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This class contains enough information about how a {@link com.yahoo.search.grouping.request.GroupingOperation} was
+ * transformed into a list {@link com.yahoo.searchlib.aggregation.Grouping} objects, so that the results of those
+ * queries can be transformed into something that corresponds to the original request.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class GroupingTransform {
+
+    private final Map<Integer, Set<Integer>> children = new HashMap<>();
+    private final Map<Integer, String> labels = new HashMap<>();
+    private final Map<Integer, Integer> maxes = new HashMap<>();
+    private final Map<Integer, Integer> offsetByTag = new HashMap<>();
+    private final Map<ResultId, Integer> offsetById = new HashMap<>();
+    private final Set<ResultId> unstable = new HashSet<>();
+    private final int requestId;
+
+    public GroupingTransform(int requestId) {
+        this.requestId = requestId;
+    }
+
+    public GroupingTransform addContinuation(Continuation cont) {
+        if (cont instanceof CompositeContinuation) {
+            for (Continuation item : ((CompositeContinuation)cont)) {
+                addContinuation(item);
+            }
+        } else if (cont instanceof OffsetContinuation) {
+            OffsetContinuation offsetCont = (OffsetContinuation)cont;
+            ResultId id = offsetCont.getResultId();
+            if (!id.startsWith(requestId)) {
+                return this;
+            }
+            if (offsetCont.testFlag(OffsetContinuation.FLAG_UNSTABLE)) {
+                unstable.add(id);
+            } else {
+                unstable.remove(id);
+            }
+            int tag = offsetCont.getTag();
+            int offset = offsetCont.getOffset();
+            if (getOffset(tag) < offset) {
+                offsetByTag.put(tag, offset);
+            }
+            offsetById.put(id, offset);
+        } else {
+            throw new UnsupportedOperationException(cont.getClass().getName());
+        }
+        return this;
+    }
+
+    public boolean isStable(ResultId resultId) {
+        return !unstable.contains(resultId);
+    }
+
+    public int getOffset(int tag) {
+        return toPosInt(offsetByTag.get(tag));
+    }
+
+    public int getOffset(ResultId resultId) {
+        return toPosInt(offsetById.get(resultId));
+    }
+
+    public GroupingTransform putMax(int tag, int max, String type) {
+        if (maxes.containsKey(tag)) {
+            throw new IllegalStateException("Can not set max of " + type + " " + tag + " to " + max +
+                                            " because it is already set to " + maxes.get(tag) + ".");
+        }
+        maxes.put(tag, max);
+        return this;
+    }
+
+    public int getMax(int tag) {
+        return toPosInt(maxes.get(tag));
+    }
+
+    public GroupingTransform putLabel(int parentTag, int tag, String label, String type) {
+        Set<Integer> siblings = children.get(parentTag);
+        if (siblings == null) {
+            siblings = new HashSet<>();
+            children.put(parentTag, siblings);
+        } else {
+            for (Integer sibling : siblings) {
+                if (label.equals(labels.get(sibling))) {
+                    throw new UnsupportedOperationException("Can not use " + type + " label '" + label +
+                                                            "' for multiple siblings.");
+                }
+            }
+        }
+        siblings.add(tag);
+        if (labels.containsKey(tag)) {
+            throw new IllegalStateException("Can not set label of " + type + " " + tag + " to '" + label +
+                                            "' because it is already set to '" + labels.get(tag) + "'.");
+        }
+        labels.put(tag, label);
+        return this;
+    }
+
+    public String getLabel(int tag) {
+        return labels.get(tag);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder ret = new StringBuilder();
+        ret.append("groupingTransform {\n");
+        ret.append("\tlabels {\n");
+        for (Map.Entry<Integer, String> entry : labels.entrySet()) {
+            ret.append("\t\t").append(entry.getKey()).append(" : ").append(entry.getValue()).append("\n");
+        }
+        ret.append("\t}\n");
+        ret.append("\toffsets {\n");
+        for (Map.Entry<Integer, Integer> entry : offsetByTag.entrySet()) {
+            ret.append("\t\t").append(entry.getKey()).append(" : ").append(entry.getValue()).append("\n");
+        }
+        ret.append("\t}\n");
+        ret.append("\tmaxes {\n");
+        for (Map.Entry<Integer, Integer> entry : maxes.entrySet()) {
+            ret.append("\t\t").append(entry.getKey()).append(" : ").append(entry.getValue()).append("\n");
+        }
+        ret.append("\t}\n");
+        ret.append("}");
+        return ret.toString();
+    }
+
+    private static int toPosInt(Integer val) {
+        return val == null ? 0 : Math.max(0, val.intValue());
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java
new file mode 100644
index 00000000000..81ae100b84f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/HitConverter.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.fs4.QueryPacketData;
+import com.yahoo.prelude.fastsearch.DocsumDefinitionSet;
+import com.yahoo.prelude.fastsearch.FastHit;
+import com.yahoo.prelude.fastsearch.GroupingListHit;
+import com.yahoo.search.Query;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.Hit;
+import com.yahoo.searchlib.aggregation.FS4Hit;
+import com.yahoo.searchlib.aggregation.VdsHit;
+
+/**
+ * Implementation of the {@link ResultBuilder.HitConverter} interface for {@link GroupingExecutor}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class HitConverter implements ResultBuilder.HitConverter {
+
+    private final Searcher searcher;
+    private final Query query;
+
+    /**
+     * Creates a new instance of this class.
+     *
+     * @param searcher The searcher that owns this converter.
+     * @param query    The query that returned the hits.
+     */
+    public HitConverter(Searcher searcher, Query query) {
+        this.searcher = searcher;
+        this.query = query;
+    }
+
+    @Override
+    public com.yahoo.search.result.Hit toSearchHit(String summaryClass, com.yahoo.searchlib.aggregation.Hit hit) {
+        if (hit instanceof FS4Hit) {
+            return convertFs4Hit(summaryClass, (FS4Hit)hit);
+        } else if (hit instanceof VdsHit) {
+            return convertVdsHit(summaryClass, (VdsHit)hit);
+        } else {
+            throw new UnsupportedOperationException("Hit type '" + hit.getClass().getName() + "' not supported.");
+        }
+    }
+
+    private Hit convertFs4Hit(String summaryClass, FS4Hit grpHit) {
+        FastHit ret = new FastHit();
+        ret.setRelevance(grpHit.getRank());
+        ret.setGlobalId(grpHit.getGlobalId());
+        ret.setPartId(grpHit.getPath(), 0);
+        ret.setDistributionKey(grpHit.getDistributionKey());
+        ret.setFillable();
+        ret.setSearcherSpecificMetaData(searcher, summaryClass);
+
+        Hit ctxHit = (Hit)grpHit.getContext();
+        if (ctxHit == null) {
+            throw new NullPointerException("Hit has no context.");
+        }
+        ret.setSource(ctxHit.getSource());
+        ret.setSourceNumber(ctxHit.getSourceNumber());
+        ret.setQuery(ctxHit.getQuery());
+
+        if (ctxHit instanceof GroupingListHit) {
+            // in a live system the ctxHit can only by GroupingListHit, but because the code used Hit prior to version
+            // 5.10 we need to check to avoid breaking existing unit tests -- both internally and with customers
+            QueryPacketData queryPacketData = ((GroupingListHit)ctxHit).getQueryPacketData();
+            if (queryPacketData != null) {
+                ret.setQueryPacketData(queryPacketData);
+            }
+        }
+        return ret;
+    }
+
+    private Hit convertVdsHit(String summaryClass, VdsHit grpHit) {
+        FastHit ret = new FastHit();
+        ret.setRelevance(grpHit.getRank());
+        if (grpHit.getSummary().getData().length > 0) {
+            GroupingListHit ctxHit = (GroupingListHit)grpHit.getContext();
+            if (ctxHit == null) {
+                throw new NullPointerException("Hit has no context.");
+            }
+            DocsumDefinitionSet defs = ctxHit.getDocsumDefinitionSet();
+            defs.lazyDecode(summaryClass, grpHit.getSummary().getData(), ret);
+            ret.setFilled(summaryClass);
+            ret.setFilled(query.getPresentation().getSummary());
+        }
+        return ret;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java
new file mode 100644
index 00000000000..c398fb41db2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class IntegerDecoder {
+
+    private static final int CHAR_MIN = IntegerEncoder.CHARS[0];
+    private static final int CHAR_MAX = IntegerEncoder.CHARS[IntegerEncoder.CHARS.length - 1];
+    private final String input;
+    private int pos = 0;
+
+    public IntegerDecoder(String input) {
+        this.input = input;
+    }
+
+    public boolean hasNext() {
+        return pos < input.length();
+    }
+
+    public int next() {
+        int val = 0;
+        int len = decodeChar(input.charAt(pos++));
+        for (int i = 0; i < len; i++) {
+            val = (val << 4) | decodeChar(input.charAt(pos + i));
+        }
+        pos += len;
+        return (val >>> 1) ^ (-(val & 0x1));
+    }
+
+    private static int decodeChar(char c) {
+        if (c >= CHAR_MIN && c <= CHAR_MAX) {
+            return (0xF & (c - CHAR_MIN));
+        } else {
+            throw new NumberFormatException(String.valueOf(c));
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerEncoder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerEncoder.java
new file mode 100644
index 00000000000..c710905a0c8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerEncoder.java
@@ -0,0 +1,32 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class IntegerEncoder {
+
+    public static final char[] CHARS = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+                                         'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P' };
+    private final StringBuilder out = new StringBuilder();
+
+    public void append(int val) {
+        val = ((val << 1) ^ (val >> 31));
+        int cnt = 8;
+        for (int i = 0; i < 8; ++i) {
+            if (((val >> (28 - 4 * i)) & 0xF) != 0) {
+                break;
+            }
+            --cnt;
+        }
+        out.append(CHARS[cnt]);
+        for (int i = 8 - cnt; i < 8; ++i) {
+            out.append(CHARS[(val >> (28 - 4 * i)) & 0xF]);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return out.toString();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/OffsetContinuation.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/OffsetContinuation.java
new file mode 100644
index 00000000000..789be271c5c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/OffsetContinuation.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class OffsetContinuation extends EncodableContinuation {
+
+    public static final int FLAG_UNSTABLE = 1;
+    private final ResultId resultId;
+    private final int tag;
+    private final int offset;
+    private final int flags;
+
+    public OffsetContinuation(ResultId resultId, int tag, int offset, int flags) {
+        resultId.getClass(); // throws NullPointerException
+        this.resultId = resultId;
+        this.tag = tag;
+        this.offset = offset;
+        this.flags = flags;
+    }
+
+    public ResultId getResultId() {
+        return resultId;
+    }
+
+    public int getTag() {
+        return tag;
+    }
+
+    public int getOffset() {
+        return offset;
+    }
+
+    public int getFlags() {
+        return flags;
+    }
+
+    public boolean testFlag(int flag) {
+        return (flags & flag) != 0;
+    }
+
+    @Override
+    public int hashCode() {
+        return resultId.hashCode() + offset + flags;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof OffsetContinuation)) {
+            return false;
+        }
+        OffsetContinuation rhs = (OffsetContinuation)obj;
+        if (!resultId.equals(rhs.resultId)) {
+            return false;
+        }
+        if (tag != rhs.tag) {
+            return false;
+        }
+        if (offset != rhs.offset) {
+            return false;
+        }
+        if (flags != rhs.flags) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void encode(IntegerEncoder out) {
+        resultId.encode(out);
+        out.append(tag);
+        out.append(offset);
+        out.append(flags);
+    }
+
+    public static OffsetContinuation decode(IntegerDecoder in) {
+        ResultId resultId = ResultId.decode(in);
+        int tag = in.next();
+        int offset = in.next();
+        int flags = in.next();
+        return new OffsetContinuation(resultId, tag, offset, flags);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java
new file mode 100644
index 00000000000..9d47464b1de
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java
@@ -0,0 +1,397 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.grouping.request.EachOperation;
+import com.yahoo.search.grouping.request.GroupingExpression;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.grouping.request.NegFunction;
+import com.yahoo.searchlib.aggregation.*;
+import com.yahoo.searchlib.expression.ExpressionNode;
+
+import java.util.*;
+
+/**
+ * This class implements the necessary logic to build a list of {@link Grouping} objects from an instance of {@link
+ * GroupingOperation}. It is used by the {@link GroupingExecutor}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class RequestBuilder {
+
+    private static final int LOOKAHEAD = 1;
+    private final ExpressionConverter converter = new ExpressionConverter();
+    private final List<Grouping> requestList = new LinkedList<>();
+    private final GroupingTransform transform;
+    private GroupingOperation root;
+    private int tag = 0;
+
+    /**
+     * Constructs a new instance of this class.
+     *
+     * @param requestId The id of the corresponding {@link GroupingRequest}.
+     */
+    public RequestBuilder(int requestId) {
+        this.transform = new GroupingTransform(requestId);
+    }
+
+    /**
+     * Sets the abstract syntax tree of the request whose back-end queries to create.
+     *
+     * @param root The grouping request to convert.
+     * @return This, to allow chaining.
+     */
+    public RequestBuilder setRootOperation(GroupingOperation root) {
+        root.getClass(); // throws NullPointerException
+        this.root = root;
+        return this;
+    }
+
+    /**
+     * Sets the time zone to build the request for. This information is propagated to the time-based grouping
+     * expressions so that the produced groups are reasonable for the given zone.
+     *
+     * @param timeZone The time zone to set.
+     * @return This, to allow chaining.
+     */
+    public RequestBuilder setTimeZone(TimeZone timeZone) {
+        converter.setTimeOffset(timeZone != null ? timeZone.getOffset(System.currentTimeMillis())
+                                                 : ExpressionConverter.DEFAULT_TIME_OFFSET);
+        return this;
+    }
+
+    /**
+     * Sets the name of the summary class to use if a {@link com.yahoo.search.grouping.request.SummaryValue} has none.
+     *
+     * @param summaryName The summary class name to set.
+     * @return This, to allow chaining.
+     */
+    public RequestBuilder setDefaultSummaryName(String summaryName) {
+        converter.setDefaultSummaryName(summaryName != null ? summaryName
+                                                            : ExpressionConverter.DEFAULT_SUMMARY_NAME);
+        return this;
+    }
+
+    /**
+     * Returns the transform that was created when {@link #build()} was called.
+     *
+     * @return The grouping transform that was built.
+     */
+    public GroupingTransform getTransform() {
+        return transform;
+    }
+
+    /**
+     * Returns the list of grouping objects that were created when {@link #build()} was called.
+     *
+     * @return The list of built grouping objects.
+     */
+    public List<Grouping> getRequestList() {
+        return requestList;
+    }
+
+    /**
+     * Constructs a set of Vespa specific grouping request that corresponds to the parameters given to this builder.
+     * This method might fail due to unsupported constructs in the request, in which case an exception is thrown.
+     *
+     * @throws IllegalStateException         If this method is called more than once.
+     * @throws UnsupportedOperationException If the grouping request contains unsupported constructs.
+     */
+    public void build() {
+        if (tag != 0) {
+            throw new IllegalStateException();
+        }
+        root.resolveLevel(1);
+
+        Grouping grouping = new Grouping();
+        grouping.getRoot().setTag(++tag);
+        grouping.setForceSinglePass(root.getForceSinglePass() || root.containsHint("singlepass"));
+        Stack<BuildFrame> stack = new Stack<>();
+        stack.push(new BuildFrame(grouping, new BuildState(), root));
+        while (!stack.isEmpty()) {
+            BuildFrame frame = stack.pop();
+            processRequestNode(frame);
+            List<GroupingOperation> children = frame.astNode.getChildren();
+            if (children.isEmpty()) {
+                requestList.add(frame.grouping);
+            } else {
+                for (int i = children.size(); --i >= 0; ) {
+                    Grouping childGrouping = (i == 0) ? frame.grouping : frame.grouping.clone();
+                    BuildState childState = (i == 0) ? frame.state : new BuildState(frame.state);
+                    BuildFrame child = new BuildFrame(childGrouping, childState, children.get(i));
+                    stack.push(child);
+                }
+            }
+        }
+        pruneRequests();
+    }
+
+    public RequestBuilder addContinuations(Iterable<Continuation> continuations) {
+        for (Continuation continuation : continuations) {
+            if (continuation == null) {
+                continue;
+            }
+            transform.addContinuation(continuation);
+        }
+        return this;
+    }
+
+    private void processRequestNode(BuildFrame frame) {
+        int level = frame.astNode.getLevel();
+        if (level > 2) {
+            throw new UnsupportedOperationException("Can not operate on " +
+                                                    GroupingOperation.getLevelDesc(level) + ".");
+        }
+        if (frame.astNode instanceof EachOperation) {
+            resolveEach(frame);
+        } else {
+            resolveOutput(frame);
+        }
+        resolveState(frame);
+        injectGroupByToExpressionCountAggregator(frame);
+    }
+
+    private void injectGroupByToExpressionCountAggregator(BuildFrame frame) {
+        Group group = getLeafGroup(frame);
+        // The ExpressionCountAggregationResult uses the group-by expression to simulate aggregation of list of groups.
+        group.getAggregationResults().stream()
+                .filter(aggr -> aggr instanceof ExpressionCountAggregationResult)
+                .forEach(aggr -> aggr.setExpression(frame.state.groupBy.clone()));
+    }
+
+    private void resolveEach(BuildFrame frame) {
+        int parentTag = getLeafGroup(frame).getTag();
+        if (frame.state.groupBy != null) {
+            GroupingLevel grpLevel = new GroupingLevel();
+            grpLevel.getGroupPrototype().setTag(++tag);
+            grpLevel.setExpression(frame.state.groupBy);
+            frame.state.groupBy = null;
+            int offset = transform.getOffset(tag);
+            if (frame.state.precision != null) {
+                grpLevel.setPrecision(frame.state.precision + offset);
+                frame.state.precision = null;
+            }
+            if (frame.state.max != null) {
+                transform.putMax(tag, frame.state.max, "group list");
+                grpLevel.setMaxGroups(LOOKAHEAD + frame.state.max + offset);
+                frame.state.max = null;
+            }
+            frame.grouping.getLevels().add(grpLevel);
+        }
+        String label = frame.astNode.getLabel();
+        if (label != null) {
+            frame.state.label = label;
+        }
+        if (frame.astNode.getLevel() > 0) {
+            transform.putLabel(parentTag, getLeafGroup(frame).getTag(), frame.state.label, "group list");
+        }
+        resolveOutput(frame);
+        if (!frame.state.orderByExp.isEmpty()) {
+            GroupingLevel grpLevel = getLeafGroupingLevel(frame);
+            for (int i = 0, len = frame.state.orderByExp.size(); i < len; ++i) {
+                grpLevel.getGroupPrototype().addOrderBy(frame.state.orderByExp.get(i),
+                                                        frame.state.orderByAsc.get(i));
+            }
+            frame.state.orderByExp.clear();
+            frame.state.orderByAsc.clear();
+        }
+    }
+
+    private void resolveState(BuildFrame frame) {
+        resolveGroupBy(frame);
+        resolveMax(frame);
+        resolveOrderBy(frame);
+        resolvePrecision(frame);
+        resolveWhere(frame);
+    }
+
+    private void resolveGroupBy(BuildFrame frame) {
+        GroupingExpression exp = frame.astNode.getGroupBy();
+        if (exp != null) {
+            if (frame.state.groupBy != null) {
+                throw new UnsupportedOperationException("Can not group list of groups.");
+            }
+            frame.state.groupBy = converter.toExpressionNode(exp);
+            frame.state.label = exp.toString(); // label for next each()
+
+        } else {
+            int level = frame.astNode.getLevel();
+            if (level == 0) {
+                // no next each()
+            } else if (level == 1) {
+                frame.state.label = "hits"; // next each() is hitlist
+            } else {
+                throw new UnsupportedOperationException("Can not create anonymous " +
+                                                        GroupingOperation.getLevelDesc(level) + ".");
+            }
+        }
+    }
+
+    private void resolveMax(BuildFrame frame) {
+
+        if (frame.astNode.hasMax()) {
+            int max = frame.astNode.getMax();
+            if (isRootOperation(frame)) {
+                frame.grouping.setTopN(max);
+            } else {
+                frame.state.max = max;
+            }
+        }
+    }
+
+    private void resolveOrderBy(BuildFrame frame) {
+        List<GroupingExpression> lst = frame.astNode.getOrderBy();
+        if (lst == null || lst.isEmpty()) {
+            return;
+        }
+        int reqLevel = frame.astNode.getLevel();
+        if (reqLevel != 2) {
+            throw new UnsupportedOperationException(
+                    "Can not order " + GroupingOperation.getLevelDesc(reqLevel) + " content.");
+        }
+        for (GroupingExpression exp : lst) {
+            boolean asc = true;
+            if (exp instanceof NegFunction) {
+                asc = false;
+                exp = ((NegFunction)exp).getArg(0);
+            }
+            frame.state.orderByExp.add(converter.toExpressionNode(exp));
+            frame.state.orderByAsc.add(asc);
+        }
+    }
+
+    private void resolveOutput(BuildFrame frame) {
+        List<GroupingExpression> lst = frame.astNode.getOutputs();
+        if (lst == null || lst.isEmpty()) {
+            return;
+        }
+        Group group = getLeafGroup(frame);
+        for (GroupingExpression exp : lst) {
+            group.addAggregationResult(toAggregationResult(exp, group, frame));
+        }
+    }
+
+    private AggregationResult toAggregationResult(GroupingExpression exp, Group group, BuildFrame frame) {
+        AggregationResult result = converter.toAggregationResult(exp);
+        result.setTag(++tag);
+
+        String label = exp.getLabel();
+        if (result instanceof HitsAggregationResult) {
+            if (label != null) {
+                throw new UnsupportedOperationException("Can not label expression '" + exp + "'.");
+            }
+            HitsAggregationResult hits = (HitsAggregationResult)result;
+            if (frame.state.max != null) {
+                transform.putMax(tag, frame.state.max, "hit list");
+                int offset = transform.getOffset(tag);
+                hits.setMaxHits(LOOKAHEAD + frame.state.max + offset);
+                frame.state.max = null;
+            }
+            transform.putLabel(group.getTag(), tag, frame.state.label, "hit list");
+        } else {
+            transform.putLabel(group.getTag(), tag, label != null ? label : exp.toString(), "output");
+        }
+        return result;
+    }
+
+    private void resolvePrecision(BuildFrame frame) {
+        int precision = frame.astNode.getPrecision();
+        if (precision > 0) {
+            frame.state.precision = precision;
+        }
+    }
+
+    private void resolveWhere(BuildFrame frame) {
+        String where = frame.astNode.getWhere();
+        if (where != null) {
+            if (!isRootOperation(frame)) {
+                throw new UnsupportedOperationException("Can not apply 'where' to non-root group.");
+            }
+            switch (where) {
+            case "true":
+                frame.grouping.setAll(true);
+                break;
+            case "$query":
+                // ignore
+                break;
+            default:
+                throw new UnsupportedOperationException("Operation 'where' does not support '" + where + "'.");
+            }
+        }
+    }
+
+    private boolean isRootOperation(BuildFrame frame) {
+        return frame.astNode == root && frame.state.groupBy == null;
+    }
+
+    private GroupingLevel getLeafGroupingLevel(BuildFrame frame) {
+        if (frame.grouping.getLevels().isEmpty()) {
+            return null;
+        }
+        return frame.grouping.getLevels().get(frame.grouping.getLevels().size() - 1);
+    }
+
+    private Group getLeafGroup(BuildFrame frame) {
+        if (frame.grouping.getLevels().isEmpty()) {
+            return frame.grouping.getRoot();
+        } else {
+            GroupingLevel grpLevel = getLeafGroupingLevel(frame);
+            return grpLevel != null ? grpLevel.getGroupPrototype() : null;
+        }
+    }
+
+    private void pruneRequests() {
+        for (int reqIdx = requestList.size(); --reqIdx >= 0; ) {
+            Grouping request = requestList.get(reqIdx);
+            List<GroupingLevel> lst = request.getLevels();
+            for (int lvlIdx = lst.size(); --lvlIdx >= 0; ) {
+                if (!lst.get(lvlIdx).getGroupPrototype().getAggregationResults().isEmpty()) {
+                    break;
+                }
+                lst.remove(lvlIdx);
+            }
+            if (lst.isEmpty() && request.getRoot().getAggregationResults().isEmpty()) {
+                requestList.remove(reqIdx);
+            }
+        }
+    }
+
+    private static class BuildFrame {
+
+        final Grouping grouping;
+        final BuildState state;
+        final GroupingOperation astNode;
+
+        BuildFrame(Grouping grouping, BuildState state, GroupingOperation astNode) {
+            this.grouping = grouping;
+            this.state = state;
+            this.astNode = astNode;
+        }
+    }
+
+    private static class BuildState {
+
+        final List<ExpressionNode> orderByExp = new ArrayList<>();
+        final List<Boolean> orderByAsc = new ArrayList<>();
+        ExpressionNode groupBy = null;
+        String label = null;
+        Integer max = null;
+        Integer precision = null;
+
+        BuildState() {
+            // empty
+        }
+
+        BuildState(BuildState obj) {
+            for (ExpressionNode e : obj.orderByExp) {
+                orderByExp.add(e.clone());
+            }
+            orderByAsc.addAll(obj.orderByAsc);
+            groupBy = obj.groupBy;
+            label = obj.label;
+            max = obj.max;
+            precision = obj.precision;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java
new file mode 100644
index 00000000000..590b531812a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java
@@ -0,0 +1,353 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.grouping.result.*;
+import com.yahoo.search.grouping.result.Group;
+import com.yahoo.search.result.Relevance;
+import com.yahoo.searchlib.aggregation.*;
+import com.yahoo.searchlib.expression.*;
+
+import java.util.*;
+
+/**
+ * This class implements the necessary logic to build a {@link RootGroup} from a list of {@link Grouping} objects. It is
+ * used by the {@link GroupingExecutor}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class ResultBuilder {
+
+    private final CompositeContinuation continuation = new CompositeContinuation();
+    private RootGroup root;
+    private GroupListBuilder rootBuilder;
+    private HitConverter hitConverter;
+    private GroupingTransform transform;
+
+    /**
+     * Sets the id of the {@link GroupingRequest} that this builder is creating the result for.
+     *
+     * @param requestId The id of the corresponding GroupingRequest.
+     * @return This, to allow chaining.
+     */
+    public ResultBuilder setRequestId(int requestId) {
+        root = new RootGroup(requestId, continuation);
+        rootBuilder = new GroupListBuilder(ResultId.valueOf(requestId), 0, true, true);
+        return this;
+    }
+
+    /**
+     * Sets the transform that details how the result should be built.
+     *
+     * @param transform The transform to set.
+     * @return This, to allow chaining.
+     */
+    public ResultBuilder setTransform(GroupingTransform transform) {
+        this.transform = transform;
+        return this;
+    }
+
+    /**
+     * Sets the converts that details how hits are converted.
+     *
+     * @param hitConverter The converter to set.
+     * @return This, to allow chaining.
+     */
+    public ResultBuilder setHitConverter(HitConverter hitConverter) {
+        this.hitConverter = hitConverter;
+        return this;
+    }
+
+    /**
+     * Adds a grouping result to this transform. This method will recurse through the given object and retrieve all the
+     * information it needs to produce the desired result when calling {@link #build()}.
+     *
+     * @param executionResult The grouping result to process.
+     */
+    public void addGroupingResult(Grouping executionResult) {
+        executionResult.unifyNull();
+        rootBuilder.addGroup(executionResult.getRoot());
+    }
+
+    /**
+     * Returns the root {@link RootGroup} that was created when {@link #build()} was called.
+     *
+     * @return The root that was built.
+     */
+    public RootGroup getRoot() {
+        return root;
+    }
+
+    /**
+     * Returns the {@link Continuation} that would recreate the exact same result as this. It is not complete until
+     * {@link #build()} has been called.
+     *
+     * @return The continuation of this result.
+     */
+    public Continuation getContinuation() {
+        return continuation;
+    }
+
+    /**
+     * Constructs the grouping result tree that corresponds to the parameters given to this builder. This method might
+     * fail due to unsupported constructs in the results, in which case an exception is thrown.
+     *
+     * @throws UnsupportedOperationException Thrown if the grouping result contains unsupported constructs.
+     */
+    public void build() {
+        int numChildren = rootBuilder.childGroups.size();
+        if (numChildren != 1) {
+            throw new UnsupportedOperationException("Expected 1 group, got " + numChildren + ".");
+        }
+        rootBuilder.childGroups.get(0).fill(root);
+    }
+
+    private class GroupBuilder {
+
+        boolean [] results = new boolean[8];
+        GroupListBuilder [] childLists = new GroupListBuilder[8];
+        int childCount = 0;
+        final ResultId resultId;
+        final com.yahoo.searchlib.aggregation.Group group;
+        final boolean stable;
+
+        GroupBuilder(ResultId resultId, com.yahoo.searchlib.aggregation.Group group, boolean stable) {
+            this.resultId = resultId;
+            this.group = group;
+            this.stable = stable;
+        }
+
+        Group build(double relevance) {
+            return fill(new Group(newGroupId(group), new Relevance(relevance)));
+        }
+
+        Group fill(Group group) {
+            for (AggregationResult res : this.group.getAggregationResults()) {
+                int tag = res.getTag();
+                if (res instanceof HitsAggregationResult) {
+                    group.add(newHitList(group.size(), tag, (HitsAggregationResult)res));
+                } else {
+                    String label = transform.getLabel(res.getTag());
+                    if (label != null) {
+                        group.setField(label, newResult(res, tag));
+                    }
+                }
+            }
+            for (GroupListBuilder child : childLists) {
+                if (child != null) {
+                    group.add(child.build());
+                }
+            }
+            return group;
+        }
+
+        GroupListBuilder getOrCreateChildList(int tag, boolean ranked) {
+            int index = tag + 1; // Add 1 to avoid the dreaded -1 default value.
+            if (index >= childLists.length) {
+                childLists = Arrays.copyOf(childLists, tag + 8);
+            }
+            GroupListBuilder ret = childLists[index];
+            if (ret == null) {
+                ret = new GroupListBuilder(resultId.newChildId(childCount), tag, stable, ranked);
+                childLists[index] = ret;
+                childCount++;
+            }
+            return ret;
+        }
+
+        void merge(com.yahoo.searchlib.aggregation.Group group) {
+            for (AggregationResult res : group.getAggregationResults()) {
+                int tag = res.getTag() + 1; // Add 1 due to dreaded -1 initialization as default.
+                if (tag >= results.length) {
+                    results = Arrays.copyOf(results, tag+8);
+                }
+                if ( ! results[tag] ) {
+                    this.group.getAggregationResults().add(res);
+                    results[tag] = true;
+                }
+            }
+        }
+
+        GroupId newGroupId(com.yahoo.searchlib.aggregation.Group execGroup) {
+            ResultNode res = execGroup.getId();
+            if (res instanceof FloatResultNode) {
+                return new DoubleId(res.getFloat());
+            } else if (res instanceof IntegerResultNode) {
+                return new LongId(res.getInteger());
+            } else if (res instanceof NullResultNode) {
+                return new NullId();
+            } else if (res instanceof RawResultNode) {
+                return new RawId(res.getRaw());
+            } else if (res instanceof StringResultNode) {
+                return new StringId(res.getString());
+            } else if (res instanceof FloatBucketResultNode) {
+                FloatBucketResultNode bucketId = (FloatBucketResultNode)res;
+                return new DoubleBucketId(bucketId.getFrom(), bucketId.getTo());
+            } else if (res instanceof IntegerBucketResultNode) {
+                IntegerBucketResultNode bucketId = (IntegerBucketResultNode)res;
+                return new LongBucketId(bucketId.getFrom(), bucketId.getTo());
+            } else if (res instanceof StringBucketResultNode) {
+                StringBucketResultNode bucketId = (StringBucketResultNode)res;
+                return new StringBucketId(bucketId.getFrom(), bucketId.getTo());
+            } else if (res instanceof RawBucketResultNode) {
+                RawBucketResultNode bucketId = (RawBucketResultNode)res;
+                return new RawBucketId(bucketId.getFrom(), bucketId.getTo());
+            } else {
+                throw new UnsupportedOperationException(res.getClass().getName());
+            }
+        }
+
+        Object newResult(ExpressionNode execResult, int tag) {
+            if (execResult instanceof AverageAggregationResult) {
+                return ((AverageAggregationResult)execResult).getAverage().getNumber();
+            } else if (execResult instanceof CountAggregationResult) {
+                return ((CountAggregationResult)execResult).getCount();
+            } else if (execResult instanceof ExpressionCountAggregationResult) {
+                long count = ((ExpressionCountAggregationResult)execResult).getEstimatedUniqueCount();
+                return correctExpressionCountEstimate(count, tag);
+            } else if (execResult instanceof MaxAggregationResult) {
+                return ((MaxAggregationResult)execResult).getMax().getValue();
+            } else if (execResult instanceof MinAggregationResult) {
+                return ((MinAggregationResult)execResult).getMin().getValue();
+            } else if (execResult instanceof SumAggregationResult) {
+                return ((SumAggregationResult)execResult).getSum().getValue();
+            } else if (execResult instanceof XorAggregationResult) {
+                return ((XorAggregationResult)execResult).getXor();
+            } else {
+                throw new UnsupportedOperationException(execResult.getClass().getName());
+            }
+        }
+
+        private long correctExpressionCountEstimate(long count, int tag) {
+            int actualGroupCount = group.getChildren().size();
+            // Use actual group count if estimate differ. If max is present, only use actual group count if less than max.
+            // NOTE: If the actual group count is 0, estimate is also 0.
+            if (actualGroupCount > 0 && count != actualGroupCount) {
+                if (transform.getMax(tag + 1) == 0 || transform.getMax(tag + 1) > actualGroupCount) {
+                    return actualGroupCount;
+                }
+            }
+            return count;
+        }
+
+
+        HitList newHitList(int listIdx, int tag, HitsAggregationResult execResult) {
+            HitList hitList = new HitList(transform.getLabel(tag));
+            List<Hit> hits = execResult.getHits();
+            PageInfo page = new PageInfo(resultId.newChildId(listIdx), tag, stable, hits.size());
+            for (int i = page.firstEntry; i < page.lastEntry; ++i) {
+                hitList.add(hitConverter.toSearchHit(execResult.getSummaryClass(), hits.get(i)));
+            }
+            page.putContinuations(hitList.continuations());
+            return hitList;
+        }
+    }
+
+    private class GroupListBuilder {
+
+        final Map<ResultNode, GroupBuilder> childResultGroups = new HashMap<>();
+        final List<GroupBuilder> childGroups = new ArrayList<>();
+        final ResultId resultId;
+        final int tag;
+        final boolean stable;
+        final boolean stableChildren;
+        final boolean ranked;
+
+        GroupListBuilder(ResultId resultId, int tag, boolean stable, boolean ranked) {
+            this.resultId = resultId;
+            this.tag = tag;
+            this.stable = stable;
+            this.stableChildren = stable && transform.isStable(resultId);
+            this.ranked = ranked;
+        }
+
+        GroupList build() {
+            PageInfo page = new PageInfo(resultId, tag, stable, childGroups.size());
+            GroupList groupList = new GroupList(transform.getLabel(tag));
+            for (int i = page.firstEntry; i < page.lastEntry; ++i) {
+                GroupBuilder child = childGroups.get(i);
+                groupList.add(child.build(ranked ? child.group.getRank() :
+                                          (double)(page.lastEntry - i) / (page.lastEntry - page.firstEntry)));
+            }
+            page.putContinuations(groupList.continuations());
+            return groupList;
+        }
+
+        void addGroup(com.yahoo.searchlib.aggregation.Group execGroup) {
+            GroupBuilder groupBuilder = getOrCreateGroup(execGroup);
+            if (!execGroup.getChildren().isEmpty()) {
+                boolean ranked = execGroup.getChildren().get(0).isRankedByRelevance();
+                execGroup.sortChildrenByRank();
+                for (com.yahoo.searchlib.aggregation.Group childGroup : execGroup.getChildren()) {
+                    GroupListBuilder childList = groupBuilder.getOrCreateChildList(childGroup.getTag(), ranked);
+                    childList.addGroup(childGroup);
+                }
+            }
+        }
+
+        GroupBuilder getOrCreateGroup(com.yahoo.searchlib.aggregation.Group execGroup) {
+            ResultNode res = execGroup.getId();
+            GroupBuilder ret = childResultGroups.get(res);
+            if (ret != null) {
+                ret.merge(execGroup);
+            } else {
+                ret = new GroupBuilder(resultId.newChildId(childResultGroups.size()), execGroup, stableChildren);
+                childResultGroups.put(res, ret);
+                childGroups.add(ret);
+            }
+            return ret;
+        }
+    }
+
+    private class PageInfo {
+
+        final ResultId resultId;
+        final int tag;
+        final int max;
+        final int numEntries;
+        final int firstEntry;
+        final int lastEntry;
+
+        PageInfo(ResultId resultId, int tag, boolean stable, int numEntries) {
+            this.resultId = resultId;
+            this.tag = tag;
+            this.numEntries = numEntries;
+            max = transform.getMax(tag);
+            if (max > 0) {
+                firstEntry = stable ? transform.getOffset(resultId) : 0;
+                lastEntry = Math.min(numEntries, firstEntry + max);
+            } else {
+                firstEntry = 0;
+                lastEntry = numEntries;
+            }
+        }
+
+        void putContinuations(Map<String, Continuation> out) {
+            if (max > 0) {
+                if (firstEntry > 0) {
+                    continuation.add(new OffsetContinuation(resultId, tag, firstEntry, 0));
+
+                    int prevPage = Math.max(0, Math.min(firstEntry, lastEntry) - max);
+                    out.put(Continuation.PREV_PAGE, new OffsetContinuation(resultId, tag, prevPage,
+                                                                           OffsetContinuation.FLAG_UNSTABLE));
+                }
+                if (lastEntry < numEntries) {
+                    out.put(Continuation.NEXT_PAGE, new OffsetContinuation(resultId, tag, lastEntry,
+                                                                           OffsetContinuation.FLAG_UNSTABLE));
+                }
+            }
+        }
+    }
+
+    /**
+     * Defines a helper interface to convert Vespa style grouping hits into corresponding instances of {@link Hit}. It
+     * is an interface to simplify testing.
+     *
+     * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+     */
+    public interface HitConverter {
+
+        public com.yahoo.search.result.Hit toSearchHit(String summaryClass, com.yahoo.searchlib.aggregation.Hit hit);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultId.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultId.java
new file mode 100644
index 00000000000..21026ac7e92
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultId.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.grouping.vespa;
+
+import java.util.Arrays;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+class ResultId {
+
+    private final int[] indexes;
+    private final int hashCode;
+
+    private ResultId(int[] indexes) {
+        this.indexes = indexes;
+        this.hashCode = Arrays.hashCode(indexes);
+    }
+
+    public boolean startsWith(int... prefix) {
+        if (prefix.length > indexes.length) {
+            return false;
+        }
+        for (int i = 0; i < prefix.length; ++i) {
+            if (prefix[i] != indexes[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public ResultId newChildId(int childIdx) {
+        int[] arr = Arrays.copyOf(indexes, indexes.length + 1);
+        arr[indexes.length] = childIdx;
+        return new ResultId(arr);
+    }
+
+    @Override
+    public int hashCode() {
+        return hashCode;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof ResultId && Arrays.equals(indexes, ((ResultId)obj).indexes);
+    }
+
+    @Override
+    public String toString() {
+        return Arrays.toString(indexes);
+    }
+
+    public void encode(IntegerEncoder out) {
+        out.append(indexes.length);
+        for (int i : indexes) {
+            out.append(i);
+        }
+    }
+
+    public static ResultId decode(IntegerDecoder in) {
+        int len = in.next();
+        int[] arr = new int[len];
+        for (int i = 0; i < len; ++i) {
+            arr[i] = in.next();
+        }
+        return new ResultId(arr);
+    }
+
+    public static ResultId valueOf(int... indexes) {
+        return new ResultId(Arrays.copyOf(indexes, indexes.length));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java
new file mode 100644
index 00000000000..f844b5dd940
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java
@@ -0,0 +1,173 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.handler;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.yahoo.collections.ListMap;
+import com.yahoo.container.jdisc.ExtendedResponse;
+import com.yahoo.container.handler.Coverage;
+import com.yahoo.container.handler.Timing;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.logging.AccessLogEntry;
+import com.yahoo.container.logging.HitCounts;
+import com.yahoo.jdisc.HeaderFields;
+import com.yahoo.jdisc.handler.CompletionHandler;
+import com.yahoo.jdisc.handler.ContentChannel;
+import com.yahoo.processing.execution.Execution.Trace.LogValue;
+import com.yahoo.processing.rendering.AsynchronousSectionedRenderer;
+import com.yahoo.processing.rendering.Renderer;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.query.context.QueryContext;
+
+/**
+ * Wrap the result of a query as an HTTP response.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class HttpSearchResponse extends ExtendedResponse {
+
+    private final Result result;
+    private final Query query;
+    private final Renderer<Result> rendererCopy;
+    private final Timing timing;
+    private final HitCounts hitCounts;
+
+    public HttpSearchResponse(int status, Result result, Query query, Renderer renderer) {
+        super(status);
+        this.query = query;
+        this.result = result;
+        this.rendererCopy = renderer;
+
+        this.timing = SearchResponse.createTiming(query, result);
+        this.hitCounts = SearchResponse.createHitCounts(query, result);
+        populateHeaders(headers(), result.getHeaders(false));
+    }
+
+    /**
+     * Copy custom HTTP headers from the search result over to the HTTP
+     * response.
+     *
+     * @param outputHeaders
+     *            the headers which will be sent to a client
+     * @param searchHeaders
+     *            the headers from the search result, or null
+     */
+    private static void populateHeaders(HeaderFields outputHeaders,
+            ListMap<String, String> searchHeaders) {
+        if (searchHeaders == null) {
+            return;
+        }
+        for (Map.Entry<String, List<String>> header : searchHeaders.entrySet()) {
+            for (String value : header.getValue()) {
+                outputHeaders.add(header.getKey(), value);
+            }
+        }
+    }
+
+    public ListenableFuture<Boolean> waitableRender(OutputStream stream) throws IOException {
+        return waitableRender(result, query, rendererCopy, stream);
+    }
+
+    public static ListenableFuture<Boolean> waitableRender(Result result,
+                                                           Query query,
+                                                           Renderer<Result> renderer,
+                                                           OutputStream stream) throws IOException {
+        SearchResponse.trimHits(result);
+        SearchResponse.removeEmptySummaryFeatureFields(result);
+        return renderer.render(stream, result, query.getModel().getExecution(), query);
+
+    }
+
+    @Override
+    public void render(OutputStream output, ContentChannel networkChannel, CompletionHandler handler) throws IOException {
+        if (rendererCopy instanceof AsynchronousSectionedRenderer) {
+            AsynchronousSectionedRenderer<Result> renderer = (AsynchronousSectionedRenderer<Result>) rendererCopy;
+            renderer.setNetworkWiring(networkChannel, handler);
+        }
+        try {
+            try {
+                waitableRender(output);
+            } finally {
+                if (!(rendererCopy instanceof AsynchronousSectionedRenderer)) {
+                    output.flush();
+                }
+            }
+        } finally {
+            if (networkChannel != null && !(rendererCopy instanceof AsynchronousSectionedRenderer)) {
+                networkChannel.close(handler);
+            }
+        }
+    }
+
+    @Override
+    public void populateAccessLogEntry(final AccessLogEntry accessLogEntry) {
+        super.populateAccessLogEntry(accessLogEntry);
+        populateAccessLogEntry(accessLogEntry, getHitCounts());
+    }
+
+    /* package-private */
+    static void populateAccessLogEntry(AccessLogEntry jdiscRequestAccessLogEntry, HitCounts hitCounts) {
+        // This entry will be logged at Jetty level. Here we just populate with tidbits from this context.
+
+        jdiscRequestAccessLogEntry.setHitCounts(hitCounts);
+    }
+
+    @Override
+    public String getParsedQuery() {
+        return query.toString();
+    }
+
+    @Override
+    public Timing getTiming() {
+        return timing;
+    }
+
+    @Override
+    public Coverage getCoverage() {
+        return result.getCoverage(false);
+    }
+
+    @Override
+    public HitCounts getHitCounts() {
+        return hitCounts;
+    }
+
+    /**
+     * Returns MIME type of this response
+     */
+    @Override
+    public String getContentType() {
+        return rendererCopy.getMimeType();
+    }
+
+    /**
+     * Returns expected character encoding of this response
+     */
+    @Override
+    public String getCharacterEncoding() {
+        String encoding = result.getQuery().getModel().getEncoding();
+        return (encoding != null) ? encoding : rendererCopy.getEncoding();
+    }
+
+    /** Returns the query wrapped by this */
+    public Query getQuery() { return query; }
+
+    /** Returns the result wrapped by this */
+    public Result getResult() { return result; }
+
+    @Override
+    public Iterable<LogValue> getLogValues() {
+        QueryContext context = query.getContext(false);
+        return context == null
+                ? Collections::emptyIterator
+                : context::logValueIterator;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
new file mode 100644
index 00000000000..c431fdac638
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
@@ -0,0 +1,532 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.handler;
+
+import com.google.inject.Inject;
+import com.yahoo.collections.Tuple2;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Chain;
+import com.yahoo.component.chain.ChainsConfigurer;
+import com.yahoo.component.chain.model.ChainsModel;
+import com.yahoo.component.chain.model.ChainsModelBuilder;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.container.Container;
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.core.ChainsConfig;
+import com.yahoo.container.core.QrTemplatesConfig;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.container.jdisc.VespaHeaders;
+import com.yahoo.container.logging.AccessLog;
+import com.yahoo.container.protect.FreezeDetector;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.language.Linguistics;
+import com.yahoo.log.LogLevel;
+import com.yahoo.net.UriTools;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.IndexModel;
+import com.yahoo.prelude.VespaSVersionRetriever;
+import com.yahoo.prelude.query.QueryException;
+import com.yahoo.prelude.query.parser.ParseException;
+import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.processing.rendering.Renderer;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.config.IndexInfoConfig;
+import com.yahoo.search.debug.DebugRpcAdaptor;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.config.QueryProfileConfigurer;
+import com.yahoo.search.query.profile.config.QueryProfilesConfig;
+import com.yahoo.search.query.properties.DefaultProperties;
+import com.yahoo.search.rendering.RendererRegistry;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.SearchChainRegistry;
+import com.yahoo.search.statistics.ElapsedTime;
+import com.yahoo.statistics.Callback;
+import com.yahoo.statistics.Handle;
+import com.yahoo.statistics.Statistics;
+import com.yahoo.statistics.Value;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Handles search request.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class SearchHandler extends LoggingRequestHandler {
+
+    private final AtomicInteger requestsInFlight = new AtomicInteger(0);
+
+    // max number of threads for the executor for this handler
+    private final int maxThreads;
+
+    private static final CompoundName DETAILED_TIMING_LOGGING = new CompoundName("trace.timingDetails");
+
+    /** Event name for number of connections to the search subsystem */
+    private static final String SEARCH_CONNECTIONS = "search_connections";
+
+    private static Logger log = Logger.getLogger(SearchHandler.class.getName());
+
+    private Value searchConnections;
+
+    private final SearchChainRegistry searchChainRegistry;
+
+    private final RendererRegistry rendererRegistry;
+
+    private final IndexFacts indexFacts;
+
+    private final SpecialTokenRegistry specialTokens;
+
+    public static final String defaultSearchChainName = "default";
+    private static final String fallbackSearchChain = "vespa";
+    private static final CompoundName FORCE_TIMESTAMPS = new CompoundName("trace.timestamps");;
+
+    // This is a hack to add the RPC adaptors for search only once
+    // TODO: Figure out the correct life cycle and init of RPC adaptors
+    static {
+        Container c = Container.get();
+        c.addOptionalRpcAdaptor(new DebugRpcAdaptor());
+    }
+
+    private final Linguistics linguistics;
+
+    private final CompiledQueryProfileRegistry queryProfileRegistry;
+
+    private final class MeanConnections implements Callback {
+        @Override
+        public void run(final Handle h, final boolean firstTime) {
+            if (firstTime) {
+                metric.set(SEARCH_CONNECTIONS, 0.0d, null);
+                return;
+            }
+            Value v = (Value) h;
+            metric.set(SEARCH_CONNECTIONS, v.getMean(), null);
+        }
+    }
+
+    @Inject
+    public SearchHandler(
+            final ChainsConfig chainsConfig,
+            final IndexInfoConfig indexInfo,
+            final QrSearchersConfig clusters,
+            final SpecialtokensConfig specialtokens,
+            final Statistics statistics,
+            final Linguistics linguistics,
+            final Metric metric,
+            final ComponentRegistry<Renderer> renderers,
+            final Executor executor,
+            final AccessLog accessLog,
+            final QueryProfilesConfig queryProfileConfig,
+            final ComponentRegistry<Searcher> searchers) {
+        super(executor, accessLog, metric, true);
+        log.log(LogLevel.DEBUG, "SearchHandler.init " + System.identityHashCode(this));
+        searchChainRegistry = new SearchChainRegistry(searchers);
+        setupSearchChainRegistry(searchers, chainsConfig);
+        indexFacts = new IndexFacts(new IndexModel(indexInfo, clusters));
+        indexFacts.freeze();
+        specialTokens = new SpecialTokenRegistry(specialtokens);
+        rendererRegistry = new RendererRegistry(renderers.allComponents());
+        QueryProfileRegistry queryProfileRegistry = QueryProfileConfigurer.createFromConfig(queryProfileConfig);
+        this.queryProfileRegistry = queryProfileRegistry.compile();
+
+        this.linguistics = linguistics;
+        this.maxThreads = examineExecutor(executor);
+
+        searchConnections = new Value(SEARCH_CONNECTIONS, statistics,
+                                      new Value.Parameters().setLogRaw(true).setLogMax(true)
+                                              .setLogMean(true).setLogMin(true)
+                                              .setNameExtension(true)
+                                              .setCallback(new MeanConnections()));
+    }
+
+    /** @deprecated use the constructor without deprecated parameters */
+    @Deprecated
+    public SearchHandler(
+            final ChainsConfig chainsConfig,
+            final IndexInfoConfig indexInfo,
+            final QrSearchersConfig clusters,
+            final SpecialtokensConfig specialTokens,
+            final QrTemplatesConfig ignored,
+            final FreezeDetector ignored2,
+            final Statistics statistics,
+            final Linguistics linguistics,
+            final Metric metric,
+            final ComponentRegistry<Renderer> renderers,
+            final Executor executor,
+            final AccessLog accessLog,
+            final QueryProfilesConfig queryProfileConfig,
+            final ComponentRegistry<Searcher> searchers) {
+        this(chainsConfig, indexInfo, clusters, specialTokens, statistics, linguistics, metric, renderers,
+             executor, accessLog, queryProfileConfig, searchers);
+    }
+
+    private void setupSearchChainRegistry(final ComponentRegistry<Searcher> searchers,
+                                          final ChainsConfig chainsConfig) {
+        final ChainsModel chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig);
+        ChainsConfigurer.prepareChainRegistry(searchChainRegistry, chainsModel, searchers);
+        searchChainRegistry.freeze();
+    }
+
+    private static int examineExecutor(Executor executor) {
+        if (executor instanceof ThreadPoolExecutor) {
+            return ((ThreadPoolExecutor) executor).getMaximumPoolSize();
+        }
+        return Integer.MAX_VALUE; // assume unbound
+    }
+
+    @Override
+    public final HttpResponse handle(com.yahoo.container.jdisc.HttpRequest request) {
+        requestsInFlight.incrementAndGet();
+        try {
+            try {
+                return handleBody(request);
+            } catch (final QueryException e) {
+                return (e.getCause() instanceof IllegalArgumentException)
+                        ? invalidParameterResponse(request, e)
+                        : illegalQueryResponse(request, e);
+            } catch (final RuntimeException e) { // Make sure we generate a valid
+                                                 // XML response even on unexpected
+                                                 // errors
+                log.log(Level.WARNING, "Failed handling " + request, e);
+                return internalServerErrorResponse(request, e);
+            }
+        } finally {
+            requestsInFlight.decrementAndGet();
+        }
+    }
+
+    private int getHttpResponseStatus(com.yahoo.container.jdisc.HttpRequest httpRequest, Result result) {
+        boolean benchmarkOutput = VespaHeaders.benchmarkOutput(httpRequest);
+        if (benchmarkOutput) {
+            return VespaHeaders.getEagerErrorStatus(result.hits().getError(),
+                                                    SearchResponse.getErrorIterator(result.hits().getErrorHit()));
+        } else {
+            return VespaHeaders.getStatus(SearchResponse.isSuccess(result),
+                                          result.hits().getError(),
+                                          SearchResponse.getErrorIterator(result.hits().getErrorHit()));
+        }
+
+    }
+
+    @SuppressWarnings("unchecked")
+    private HttpResponse errorResponse(HttpRequest request, ErrorMessage errorMessage) {
+        Query query = new Query();
+        Result result = new Result(query, errorMessage);
+        Renderer renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format")));
+
+        result.getTemplating().setRenderer(renderer); // Pre-Vespa 6 Result.getEncoding() expects this TODO: Remove
+
+        return new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer);
+    }
+
+    private HttpResponse invalidParameterResponse(HttpRequest request, RuntimeException e) {
+        return errorResponse(request, ErrorMessage.createInvalidQueryParameter(Exceptions.toMessageString(e)));
+    }
+
+    private HttpResponse illegalQueryResponse(HttpRequest request, RuntimeException e) {
+        return errorResponse(request, ErrorMessage.createIllegalQuery(Exceptions.toMessageString(e)));
+    }
+
+    private HttpResponse internalServerErrorResponse(HttpRequest request, RuntimeException e) {
+        return errorResponse(request, ErrorMessage.createInternalServerError(Exceptions.toMessageString(e)));
+    }
+
+    private HttpSearchResponse handleBody(HttpRequest request) {
+        // Find query profile
+        String queryProfileName = request.getProperty("queryProfile");
+        CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName);
+        boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request);
+
+        // Create query
+        Query query = new Query(request, queryProfile);
+
+        boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers());
+        if (benchmarkCoverage) {
+            query.getPresentation().setReportCoverage(true);
+        }
+
+        // Find and execute search chain if we have a valid query
+        String invalidReason = query.validate();
+        Chain<Searcher> searchChain = null;
+        String searchChainName = null;
+        if (invalidReason == null) {
+            Tuple2<String, Chain<Searcher>> nameAndChain = resolveChain(query.properties().getString(Query.SEARCH_CHAIN));
+            searchChainName = nameAndChain.first;
+            searchChain = nameAndChain.second;
+        }
+
+        // Create the result
+        Result result;
+        if (invalidReason != null) {
+            result = new Result(query, ErrorMessage.createIllegalQuery(invalidReason));
+        } else if (queryProfile == null && queryProfileName != null) {
+            result = new Result(
+                    query,
+                    ErrorMessage.createIllegalQuery("Could not resolve query profile '" + queryProfileName + "'"));
+        } else if (searchChain == null) {
+            result = new Result(
+                    query,
+                    ErrorMessage.createInvalidQueryParameter("No search chain named '" + searchChainName + "' was found"));
+        } else {
+            String pathAndQuery = UriTools.rawRequest(request.getUri());
+            result = search(pathAndQuery, query, searchChain, searchChainRegistry);
+        }
+
+        Renderer renderer;
+        if (result.getTemplating().usesDefaultTemplate()) {
+            renderer = toRendererCopy(query.getPresentation().getRenderer());
+            result.getTemplating().setRenderer(renderer); // pre-Vespa 6 Result.getEncoding() expects this to be set. TODO: Remove
+        }
+        else { // somebody explicitly assigned a old style template
+            renderer = perRenderingCopy(result.getTemplating().getRenderer());
+        }
+
+        // Transform result to response
+        HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result),
+                                                             result, query, renderer);
+        if (benchmarkOutput) {
+            VespaHeaders.benchmarkOutput(response.headers(), benchmarkCoverage, response.getTiming(),
+                                         response.getHitCounts(), getErrors(result), response.getCoverage());
+        }
+
+        return response;
+    }
+
+    private static int getErrors(Result result) {
+        return result.hits().getErrorHit() == null ? 0 : 1;
+    }
+
+    @NonNull
+    private Renderer<Result> toRendererCopy(ComponentSpecification format) {
+        Renderer<Result> renderer = rendererRegistry.getRenderer(format);
+        renderer = perRenderingCopy(renderer);
+        return renderer;
+    }
+
+    private Tuple2<String, Chain<Searcher>> resolveChain(String explicitChainName) {
+        String chainName = explicitChainName;
+        if (chainName == null) {
+            chainName = defaultSearchChainName;
+        }
+
+        Chain<Searcher> searchChain = searchChainRegistry.getChain(chainName);
+        if (searchChain == null && explicitChainName == null) { // explicit
+                                                                // search chain
+                                                                // not found
+                                                                // should cause
+                                                                // error
+            chainName = fallbackSearchChain;
+            searchChain = searchChainRegistry.getChain(chainName);
+        }
+        return new Tuple2<>(chainName, searchChain);
+    }
+
+    /** Used from container SDK, for internal use only */
+    public Result searchAndFill(Query query, Chain<? extends Searcher> searchChain, SearchChainRegistry registry) {
+        Result errorResult = validateQuery(query);
+        if (errorResult != null) return errorResult;
+
+        Renderer<Result> renderer = rendererRegistry.getRenderer(query.getPresentation().getRenderer());
+
+        // docsumClass null means "unset", so we set it (it might be null
+        // here too in which case it will still be "unset" after we
+        // set it :-)
+        if (query.getPresentation().getSummary() == null && renderer instanceof com.yahoo.search.rendering.Renderer)
+            query.getPresentation().setSummary(((com.yahoo.search.rendering.Renderer) renderer).getDefaultSummaryClass());
+
+        Execution execution = new Execution(searchChain,
+                                            new Execution.Context(registry, indexFacts, specialTokens, rendererRegistry, linguistics));
+        query.getModel().setExecution(execution);
+        query.getModel().traceLanguage();
+        execution.trace().setForceTimestamps(query.properties().getBoolean(FORCE_TIMESTAMPS, false));
+        if (query.properties().getBoolean(DETAILED_TIMING_LOGGING, false)) {
+            // check and set (instead of set directly) to avoid overwriting stuff from prepareForBreakdownAnalysis()
+            execution.context().setDetailedDiagnostics(true);
+        }
+        Result result = execution.search(query);
+
+        if (result.getTemplating() == null)
+            result.getTemplating().setRenderer(renderer);
+
+        ensureQuerySet(result, query);
+        execution.fill(result, result.getQuery().getPresentation().getSummary());
+
+        traceExecutionTimes(query, result);
+        traceVespaSVersion(query);
+        traceRequestAttributes(query);
+        return result;
+    }
+
+    private void traceRequestAttributes(Query query) {
+        int miminumTraceLevel = 7;
+        if (query.getTraceLevel() >= 7) {
+            query.trace("Request attributes: " + query.getHttpRequest().getJDiscRequest().context(), miminumTraceLevel);
+        }
+    }
+
+    /**
+     * For internal use only
+     */
+    public Renderer<Result> getRendererCopy(ComponentSpecification spec) { // TODO: Deprecate this
+        Renderer<Result> renderer = rendererRegistry.getRenderer(spec);
+        return perRenderingCopy(renderer);
+    }
+
+    @NonNull
+    private Renderer<Result> perRenderingCopy(Renderer<Result> renderer) {
+        Renderer<Result> copy = renderer.clone();
+        copy.init();
+        return copy;
+    }
+
+    private void ensureQuerySet(Result result, Query fallbackQuery) {
+        Query query = result.getQuery();
+        if (query == null) {
+            result.setQuery(fallbackQuery);
+        }
+    }
+
+    private Result search(String request, Query query, Chain<Searcher> searchChain, SearchChainRegistry registry) {
+        if (query.getTraceLevel() >= 2) {
+            query.trace("Invoking " + searchChain, false, 2);
+        }
+
+        if (searchConnections != null) {
+            connectionStatistics();
+        } else {
+            log.log(LogLevel.WARNING,
+                    "searchConnections is a null reference, probably a known race condition during startup.",
+                    new IllegalStateException("searchConnections reference is null."));
+        }
+        try {
+            return searchAndFill(query, searchChain, registry);
+        } catch (ParseException e) {
+            ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: "
+                                                                 + Exceptions.toMessageString(e));
+            log.log(LogLevel.DEBUG, () -> error.getDetailedMessage());
+            return new Result(query, error);
+        } catch (IllegalArgumentException e) {
+            ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: "
+                                                               + Exceptions.toMessageString(e));
+            log.log(LogLevel.DEBUG, () -> error.getDetailedMessage());
+            return new Result(query, error);
+        } catch (LinkageError e) {
+            // Should have been an Exception in an OSGi world - typical bundle dependency issue problem
+            ErrorMessage error = ErrorMessage.createErrorInPluginSearcher(
+                            "Error executing " + searchChain + "]: " + Exceptions.toMessageString(e), e);
+            log(request, query, e);
+            return new Result(query, error);
+        } catch (StackOverflowError e) { // Also recoverable
+            ErrorMessage error = ErrorMessage.createErrorInPluginSearcher(
+                            "Error executing " + searchChain + "]: " + Exceptions.toMessageString(e), e);
+            log(request, query, e);
+            return new Result(query, error);
+        } catch (Exception e) {
+            Result result = new Result(query);
+            log(request, query, e);
+            result.hits().setError(
+                    ErrorMessage.createUnspecifiedError("Failed searching: " + Exceptions.toMessageString(e), e));
+            return result;
+        }
+    }
+
+    private void connectionStatistics() {
+        int connections = requestsInFlight.intValue();
+        searchConnections.put(connections);
+        if (maxThreads > 3) {
+            // cast to long to avoid overflows if maxThreads is at no
+            // log value (maxint)
+            final long maxThreadsAsLong = maxThreads;
+            final long connectionsAsLong = connections;
+            // only log when exactly crossing the limit to avoid
+            // spamming the log
+            if (connectionsAsLong < maxThreadsAsLong * 9L / 10L) {
+                // NOP
+            } else if (connectionsAsLong == maxThreadsAsLong * 9L / 10L) {
+                log.log(Level.WARNING, threadConsumptionMessage(connections, maxThreads, "90"));
+            } else if (connectionsAsLong == maxThreadsAsLong * 95L / 100L) {
+                log.log(Level.WARNING, threadConsumptionMessage(connections, maxThreads, "95"));
+            } else if (connectionsAsLong == maxThreadsAsLong) {
+                log.log(Level.WARNING, threadConsumptionMessage(connections, maxThreads, "100"));
+            }
+        }
+    }
+
+    private String threadConsumptionMessage(int connections, int maxThreads, String percentage) {
+        return percentage + "% of possible search connections (" + connections +
+               " of maximum " + maxThreads + ") currently active.";
+    }
+
+    private void log(String request, Query query, Throwable e) {
+        // Attempted workaround for missing stack traces
+        if (e.getStackTrace().length == 0) {
+            log.log(LogLevel.ERROR,
+                    "Failed executing " + query.toDetailString() + " [" + request
+                            + "], received exception with no context", e);
+        } else {
+            log.log(LogLevel.ERROR,
+                    "Failed executing " + query.toDetailString() + " [" + request + "]", e);
+        }
+    }
+
+    private Result validateQuery(Query query) {
+        if (query.getHttpRequest().getProperty(DefaultProperties.MAX_HITS.toString()) != null)
+            throw new RuntimeException(DefaultProperties.MAX_HITS + " must be specified in a query profile.");
+
+        if (query.getHttpRequest().getProperty(DefaultProperties.MAX_OFFSET.toString()) != null)
+            throw new RuntimeException(DefaultProperties.MAX_OFFSET + " must be specified in a query profile.");
+
+        int maxHits = query.properties().getInteger(DefaultProperties.MAX_HITS);
+        int maxOffset = query.properties().getInteger(DefaultProperties.MAX_OFFSET);
+
+        if (query.getHits() > maxHits) {
+            return new Result(query, ErrorMessage.createIllegalQuery(query.getHits() +
+                              " hits requested, configured limit: " + maxHits + "."));
+
+        } else if (query.getOffset() > maxOffset) {
+            return new Result(query,
+                    ErrorMessage.createIllegalQuery("Offset of " + query.getOffset() +
+                                                    " requested, configured limit: " + maxOffset + "."));
+        }
+        return null;
+    }
+
+    private void traceExecutionTimes(Query query, Result result) {
+        if (query.getTraceLevel() < 3) return;
+
+        ElapsedTime elapsedTime = result.getElapsedTime();
+        long now = System.currentTimeMillis();
+        if (elapsedTime.firstFill() != 0) {
+            query.trace("Query time " + query + ": "
+                     + (elapsedTime.firstFill() - elapsedTime.first()) + " ms", false, 3);
+
+            query.trace("Summary fetch time " + query + ": "
+                        + (now - elapsedTime.firstFill()) + " ms", false, 3);
+        } else {
+            query.trace("Total search time " + query + ": "
+                        + (now - elapsedTime.first()) + " ms", false, 3);
+        }
+    }
+
+    private void traceVespaSVersion(Query query) {
+        query.trace("Vespa version: " + VespaSVersionRetriever.getVersion(), false, 4);
+    }
+
+    public SearchChainRegistry getSearchChainRegistry() {
+        return searchChainRegistry;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java
new file mode 100644
index 00000000000..b0460ee6597
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/handler/SearchResponse.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.handler;
+
+import com.yahoo.container.handler.Timing;
+import com.yahoo.container.logging.HitCounts;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.ErrorHit;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+
+/**
+ * Some leftover static methods.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class SearchResponse {
+
+    // Remove (the empty) summary feature field if not requested.
+    static void removeEmptySummaryFeatureFields(Result result) {
+        // TODO: Move to some searcher in Vespa backend search chains
+        if (!result.hits().getQuery().getRanking().getListFeatures())
+            for (Iterator<Hit> i = result.hits().unorderedIterator(); i.hasNext();)
+                i.next().removeField(Hit.RANKFEATURES_FIELD);
+    }
+
+    static void trimHits(Result result) {
+        if (result.getConcreteHitCount() > result.hits().getQuery().getHits()) {
+            result.hits().trim(0, result.hits().getQuery().getHits());
+        }
+    }
+
+    static Iterator<? extends ErrorMessage> getErrorIterator(ErrorHit h) {
+        if (h == null) {
+            return new ArrayList<ErrorMessage>(0).iterator();
+        } else {
+            return h.errorIterator();
+        }
+    }
+
+    static boolean isSuccess(Result r) {
+        if (r.hits().getErrorHit()==null) return true;
+        for (Hit hit : r.hits())
+            if ( ! hit.isMeta()) return true; // contains data : success
+        return false;
+    }
+
+    @SuppressWarnings("deprecation")
+    public static Timing createTiming(Query query, Result result) {
+        return new Timing(result.getElapsedTime().firstFill(),
+                          0,
+                          result.getElapsedTime().first(), query.getTimeout());
+    }
+
+    public static HitCounts createHitCounts(Query query, Result result) {
+        return new HitCounts(result.getHitCount(),
+                          result.getConcreteHitCount(),
+                          result.getTotalHitCount(),
+                          query.getHits(),
+                          query.getOffset());
+    }
+
+}
+
+
diff --git a/container-search/src/main/java/com/yahoo/search/handler/package-info.java b/container-search/src/main/java/com/yahoo/search/handler/package-info.java
new file mode 100644
index 00000000000..fa35495e3f8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/handler/package-info.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * The search handler, which handles search request to the Container by translating the Request into a Query, invoking the
+ * chosen Search Chain to get a Result, which it translates to a Response which is returned to the Container.
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.handler;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/Intent.java b/container-search/src/main/java/com/yahoo/search/intent/model/Intent.java
new file mode 100644
index 00000000000..f9d97e057d1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/Intent.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+/**
+ * A representation of an intent behind a query. Intents have no structure but are just id's of a
+ * set which is predefined in the application.
+ * <p>
+ * Intents are Value Objects.
+ * <p>
+ * Intent ids should be human readable, start with lower case and use camel casing
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Intent {
+
+    private String id;
+
+    public static final Intent Default=new Intent("default");
+
+    /** Creates an intent from a string id */
+    public Intent(String id) {
+        this.id=id;
+    }
+
+    /** Returns the id of this intent, never null */
+    public String getId() { return id; }
+
+    public @Override int hashCode() { return id.hashCode(); }
+
+    public @Override boolean equals(Object other) {
+        if (other==this) return true;
+        if ( ! (other instanceof Intent)) return false;
+        return this.id.equals(((Intent)other).id);
+    }
+
+    /** Returns the id of this intent */
+    public @Override String toString() { return id; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java b/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java
new file mode 100644
index 00000000000..915c8fbd1d1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java
@@ -0,0 +1,90 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+import com.yahoo.search.Query;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.text.interpretation.Interpretation;
+
+import java.util.*;
+
+/**
+ * This is the root node of an intent model.
+ * The intent model represents the intent analysis of a query.
+ * This is a probabilistic model - the query may have multiple interpretations with different probability.
+ * Each interpretation may have multiple
+ * possible intents, making this a tree.
+ *
+ * @author bratseth
+ */
+public class IntentModel extends ParentNode<InterpretationNode> {
+
+    /** The name of the property carrying the intent model string: intentModel */
+    public static final CompoundName intentModelStringName=new CompoundName("intentModel");
+    /** The name of the property carrying the intent model object: IntentModel */
+    public static final CompoundName intentModelObjectName=new CompoundName("IntentModel");
+
+    private static final InterpretationNodeComparator inodeComp = new InterpretationNodeComparator();
+
+    /** Creates an empty intent model */
+    public IntentModel() {
+    }
+
+    /** Creates an intent model from some interpretations */
+    public IntentModel(List<Interpretation> interpretations) {
+        for (Interpretation interpretation : interpretations)
+            children().add(new InterpretationNode(interpretation));
+        sortChildren();
+    }
+
+    /** Creates an intent model from some interpretations */
+    public IntentModel(Interpretation... interpretations) {
+        for (Interpretation interpretation : interpretations)
+            children().add(new InterpretationNode(interpretation));
+        sortChildren();
+    }
+
+    /** Sort interpretations by descending score order */
+    public void sortChildren() {
+        Collections.sort(children(), inodeComp);
+    }
+
+    /**
+     * Returns a flattened list of sources with a normalized appropriateness of each, sorted by
+     * decreasing appropriateness.
+     * This is obtained by summing the source appropriateness vectors of each intent node weighted
+     * by the owning intent and interpretation probabilities.
+     * Sources with a resulting probability of 0 is omitted in the returned list.
+     */
+    public List<SourceNode> getSources() {
+        Map<Source,SourceNode> sources=new HashMap<>();
+        addSources(1.0,sources);
+        List<SourceNode> sourceList=new ArrayList<>(sources.values());
+        Collections.sort(sourceList);
+        return sourceList;
+    }
+
+    /** Returns the names of the sources returned from {@link #getSources} for convenience */
+    public List<String> getSourceNames() {
+        List<String> sourceNames=new ArrayList<>();
+        for (SourceNode sourceNode : getSources())
+            sourceNames.add(sourceNode.getSource().getId());
+        return sourceNames;
+    }
+
+    /** Returns the intent model stored at property key "intentModel" in this query, or null if none */
+    public static IntentModel getFrom(Query query) {
+        return (IntentModel)query.properties().get(intentModelObjectName);
+    }
+
+    /** Stores this intent model at property key "intentModel" in this query */
+    public void setTo(Query query) {
+        query.properties().set(intentModelObjectName,this);
+    }
+
+    static class InterpretationNodeComparator implements Comparator<InterpretationNode> {
+        public int compare(InterpretationNode o1, InterpretationNode o2) {
+            double diff = o2.getScore()-o1.getScore();
+            return (diff>0) ? 1 : ( (diff<0)? -1:0 );
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/IntentNode.java b/container-search/src/main/java/com/yahoo/search/intent/model/IntentNode.java
new file mode 100644
index 00000000000..c77c937b760
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/IntentNode.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+/**
+ * An intent in an intent model tree. The intent node score is the <i>probability</i> of this intent
+ * given the parent interpretation.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class IntentNode extends ParentNode<SourceNode> {
+
+    private Intent intent;
+
+    public IntentNode(Intent intent,double probabilityScore) {
+        super(probabilityScore);
+        this.intent=intent;
+    }
+
+    /** Returns the intent of this node, this is never null */
+    public Intent getIntent() { return intent; }
+
+    public void setIntent(Intent intent) { this.intent=intent; }
+
+    /** Returns intent:probability */
+    public @Override String toString() {
+        return intent + ":" + getScore();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/InterpretationNode.java b/container-search/src/main/java/com/yahoo/search/intent/model/InterpretationNode.java
new file mode 100644
index 00000000000..51e5d00c563
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/InterpretationNode.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+import com.yahoo.text.interpretation.Interpretation;
+
+/**
+ * An interpretation which may have multiple intents. The score of this node is the probability of
+ * the wrapped interpretation.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class InterpretationNode extends ParentNode<IntentNode> {
+
+    private Interpretation interpretation;
+
+    public InterpretationNode(Interpretation interpretation) {
+        super(0); // Super score is not used
+        this.interpretation=interpretation;
+        children().add(new IntentNode(Intent.Default,1.0));
+    }
+
+    /** Returns this interpretation. This is never null. */
+    public Interpretation getInterpretation() { return interpretation; }
+
+    /** Sets this interpretation */
+    public void setInterpretation(Interpretation interpretation) {
+        this.interpretation=interpretation;
+    }
+
+    /** Returns the probability of the interpretation of this */
+    public @Override double getScore() {
+        return interpretation.getProbability();
+    }
+
+    /** Sets the probability of the interpretation of this */
+    public void setScore(double score) {
+        interpretation.setProbability(score);
+    }
+
+    /** Returns interpretations toString() */
+    public @Override String toString() {
+        return interpretation.toString();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/Node.java b/container-search/src/main/java/com/yahoo/search/intent/model/Node.java
new file mode 100644
index 00000000000..ecd3ec712bb
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/Node.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+import java.util.Map;
+
+/**
+ * A node in the <a href="TODO">intent model tree</a>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public abstract class Node implements Comparable<Node> {
+
+    /**
+     * The score, unless getScore/setScore is overridden which is the case with interpretations,
+     * so DO NOT ACCESS SCORE DIRECTLY, ALWAYS USE GET/SET
+     */
+    private double score;
+
+    public Node(double score) {
+        this.score=score;
+    }
+
+    /** Returns the normalized (0-1) score of this node */
+    public double getScore() { return score; }
+
+    /** Sets the normalized (0-1) score of this node */
+    public void setScore(double score) { this.score=score; }
+
+    /** Increases this score by an increment and returns the new score */
+    public double increaseScore(double increment) {
+        setScore(getScore()+increment);
+        return getScore();
+    }
+
+    public int compareTo(Node other) {
+        if (this.getScore()<other.getScore()) return 1;
+        if (this.getScore()>other.getScore()) return -1;
+        return 0;
+    }
+
+    /**
+     * Adds the sources at (and beneath) this node to the given
+     * sparsely represented source vector, weighted by the score of this node
+     * times the given weight from the parent path
+     */
+    abstract void addSources(double weight,Map<Source,SourceNode> sources);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/ParentNode.java b/container-search/src/main/java/com/yahoo/search/intent/model/ParentNode.java
new file mode 100644
index 00000000000..357060be93c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/ParentNode.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A node which is not a leaf in the intent tree
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public abstract class ParentNode<T extends Node> extends Node {
+
+    private List<T> children=new ArrayList<>();
+
+    public ParentNode() {
+        super(1.0);
+    }
+
+    public ParentNode(double score) {
+        super(score);
+    }
+
+    /**
+     * This returns the children of this node in the intent tree.
+     * This is never null. Children can be added and removed from this list to modify this node.
+     */
+    public List<T> children() { return children; }
+
+    @Override void addSources(double weight,Map<Source,SourceNode> sources) {
+        for (T child : children)
+            child.addSources(weight*getScore(),sources);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/Source.java b/container-search/src/main/java/com/yahoo/search/intent/model/Source.java
new file mode 100644
index 00000000000..937b6ca02e4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/Source.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+/**
+ * A representation of a source. Sources have no structure but are just id of a
+ * set which is defined in the application.
+ * <p>
+ * Sources are Value Objects.
+ * <p>
+ * Source ids should be human readable, start with lower case and use camel casing
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Source {
+
+    private String id;
+
+    /** Creates an intent from a string id */
+    public Source(String id) {
+        this.id=id;
+    }
+
+    /** Returns the id of this source, never null */
+    public String getId() { return id; }
+
+    public @Override int hashCode() { return id.hashCode(); }
+
+    public @Override boolean equals(Object other) {
+        if (other==this) return true;
+        if ( ! (other instanceof Source)) return false;
+        return this.id.equals(((Source)other).id);
+    }
+
+    /** Returns the id of this source */
+    public @Override String toString() { return id; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/SourceNode.java b/container-search/src/main/java/com/yahoo/search/intent/model/SourceNode.java
new file mode 100644
index 00000000000..5f63ddbe8d1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/SourceNode.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.intent.model;
+
+import java.util.Map;
+
+/**
+ * A source node in an intent model tree. Represents a source with an appropriateness score
+ * (i.e the score of a source node is called <i>appropriateness</i>).
+ * Sources are ordered by decreasing appropriateness.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SourceNode extends Node {
+
+    private Source source;
+
+    public SourceNode(Source source,double score) {
+        super(score);
+        this.source=source;
+    }
+
+    /** Sets the source of this node */
+    public void setSource(Source source) { this.source=source; }
+
+    /** Returns the source of this node */
+    public Source getSource() { return source; }
+
+    @Override void addSources(double weight,Map<Source,SourceNode> sources) {
+        SourceNode existing=sources.get(source);
+        if (existing!=null)
+            existing.increaseScore(weight*getScore());
+        else
+            sources.put(source,new SourceNode(source,weight*getScore()));
+    }
+
+    /** Returns source:appropriateness */
+    public @Override String toString() {
+        return source + ":" + getScore();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/package-info.java b/container-search/src/main/java/com/yahoo/search/intent/model/package-info.java
new file mode 100644
index 00000000000..1e3e38208c5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/intent/model/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.intent.model;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/match/DocumentDb.java b/container-search/src/main/java/com/yahoo/search/match/DocumentDb.java
new file mode 100644
index 00000000000..f4be6861364
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/match/DocumentDb.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.match;
+
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentOperation;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * A searchable database of documents
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DocumentDb extends Searcher {
+
+    /**
+     * Put a document or apply an update to this document db
+     */
+    public void put(DocumentOperation op) {
+
+    }
+
+    /** Remove a document from this document db */
+    public void remove(Document document) {
+
+    }
+
+    /** Search this document db */
+    @Override
+    public Result search(Query query, Execution execution) {
+        Result r = execution.search(query);
+        return r;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/package-info.java b/container-search/src/main/java/com/yahoo/search/package-info.java
new file mode 100644
index 00000000000..96255d9108b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/package-info.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * <b>The top level classes of the search container.</b> A Query represents the incoming request, which produces a Result
+ * by chained execution of a set of Searchers.
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplate.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplate.java
new file mode 100644
index 00000000000..8c421feae47
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplate.java
@@ -0,0 +1,82 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.FreezableComponent;
+import com.yahoo.search.pagetemplates.model.PageElement;
+import com.yahoo.search.pagetemplates.model.PageTemplateVisitor;
+import com.yahoo.search.pagetemplates.model.Section;
+import com.yahoo.search.pagetemplates.model.Source;
+
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * A page template represents a particular way to organize a return page. It is a recursive structure of
+ * page template elements.
+ *
+ * @author bratseth
+ */
+public final class PageTemplate extends FreezableComponent implements PageElement {
+
+    /** The root section of this page */
+    private Section section=new Section();
+
+    /** The sources mentioned (recursively) in this page template, or null if this is not frozen */
+    private Set<Source> sources=null;
+
+    public PageTemplate(ComponentId id) {
+        super(id);
+    }
+
+    public void setSection(Section section) {
+        ensureNotFrozen();
+        this.section=section;
+    }
+
+    /** Returns the root section of this. This is never null. */
+    public Section getSection() { return section; }
+
+    /**
+     * Returns an unmodifiable set of all the sources this template <i>may</i> include (depending on choice resolution).
+     * If the template allows (somewhere) the "any" source (*), Source.any will be in the set returned.
+     * This operation is fast on frozen page templates (i.e at execution time).
+     */
+    public Set<Source> getSources() {
+        if (isFrozen()) return sources;
+        SourceVisitor sourceVisitor=new SourceVisitor();
+        getSection().accept(sourceVisitor);
+        return Collections.unmodifiableSet(sourceVisitor.getSources());
+    }
+
+    public @Override void freeze() {
+        if (isFrozen()) return;
+        resolvePlaceholders();
+        section.freeze();
+        sources=getSources();
+        super.freeze();
+    }
+
+    /** Validates and creates the necessary internal references between placeholders and their resolving choices */
+    private void resolvePlaceholders() {
+        try {
+            PlaceholderMappingVisitor placeholderMappingVisitor=new PlaceholderMappingVisitor();
+            accept(placeholderMappingVisitor);
+            accept(new PlaceholderReferenceCreatingVisitor(placeholderMappingVisitor.getMap()));
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException(this + " is invalid",e);
+        }
+    }
+
+    /** Accepts a visitor to this structure */
+    public @Override void accept(PageTemplateVisitor visitor) {
+        visitor.visit(this);
+        section.accept(visitor);
+    }
+
+    public @Override String toString() {
+        return "page template '" + getId() + "'";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateRegistry.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateRegistry.java
new file mode 100644
index 00000000000..ffeec4b5dd1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateRegistry.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates;
+
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.pagetemplates.engine.Resolver;
+
+/**
+ * @author bratseth
+ */
+public class PageTemplateRegistry extends ComponentRegistry<PageTemplate> {
+
+    public void register(PageTemplate pageTemplate) {
+        super.register(pageTemplate.getId(), pageTemplate);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java
new file mode 100644
index 00000000000..eb928097e2d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java
@@ -0,0 +1,234 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates;
+
+import com.google.inject.Inject;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.intent.model.IntentModel;
+import com.yahoo.search.pagetemplates.config.PageTemplateConfigurer;
+import com.yahoo.search.pagetemplates.engine.Organizer;
+import com.yahoo.search.pagetemplates.engine.Resolution;
+import com.yahoo.search.pagetemplates.engine.Resolver;
+import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver;
+import com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver;
+import com.yahoo.search.pagetemplates.engine.resolvers.ResolverRegistry;
+import com.yahoo.search.pagetemplates.model.Choice;
+import com.yahoo.search.pagetemplates.model.PageElement;
+import com.yahoo.search.pagetemplates.model.Source;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.*;
+
+/**
+ * Enables page optimization templates.
+ * This searcher should be placed before federation points in the search chain.
+ * <p>
+ * <b>Input query properties:</b>
+ * <ul>
+ * <li><code>page.idList</code> - a List&lt;String&gt; of id strings of the page templates this should choose between</li>
+ * <li><code>page.id</code> - a space-separated string of ids of the page templates this should choose between.
+ *     This property is ignored if <code>page.idList</code> is set</li>
+ * <li><code>page.resolver</code> the id of the resolver to use to resolve choices. This is either the component id
+ *     of a deployed resolver component, or one of the strings
+ *     <code>native.deterministic</code> (which always pics the last choice) or <code>native.random</code></li>
+ * </ul>
+ *
+ * <b>Output query properties:</b>
+ * <ul>
+ * <li><code>page.ListOfPageTemplate</code>A List&lt;PageTemplate&gt;
+ * containing a list of the page templates used for this query
+ * </ul>
+ *
+ * <p>
+ * The set of page templates chosen for the query specifies a list of sources to be queries (the page template sources).
+ * In addition, the query may contain
+ * <ul>
+ *   <li>a set of sources set explicitly in the Request, a query property or a searcher (the query model sources)
+ *   <li>a set of sources specified in the {@link com.yahoo.search.intent.model.IntentModel} (the intent model sources)
+ * </ul>
+ * This searcher combines these sources into a single set in query.model by the following rules:
+ * <ul>
+ *   <li>If the query model sources is set (not empty), it is not changed
+ *   <li>If the page template sources contains the ANY source AND there is an intent model
+ *       the query model sources is set to the union of the page template sources and the intent model sources
+ *   <li>If the page template sources contains the ANY source AND there is no intent model,
+ *       the query model sources is left empty (causing all sources to be queried)
+ *   <li>Otherwise, the query model sources is set to the page template sources
+ * </ul>
+ *
+ * @author bratseth
+ */
+@Provides("PageTemplates")
+public class PageTemplateSearcher extends Searcher {
+
+    /** The name of the query property containing the resolved candidate page template list */
+    public static final CompoundName pagePageTemplateListName=new CompoundName("page.PageTemplateList");
+    /** The name of the query property containing a list of candidate pages to consider */
+    public static final CompoundName pageIdListName=new CompoundName("page.idList");
+    /** The name of the query property containing the page id to use */
+    public static final CompoundName pageIdName=new CompoundName("page.id");
+    /** The name of the query property containing the resolver id to use */
+    public static final CompoundName pageResolverName=new CompoundName("page.resolver");
+
+    private final ResolverRegistry resolverRegistry;
+
+    private final Organizer organizer = new Organizer();
+
+    private final PageTemplateRegistry templateRegistry;
+
+    /** Creates this from a configuration. This will be called by the container. */
+    @Inject
+    public PageTemplateSearcher(PageTemplatesConfig pageTemplatesConfig, ComponentRegistry<Resolver> resolverRegistry) {
+        this(PageTemplateConfigurer.toRegistry(pageTemplatesConfig), resolverRegistry.allComponents());
+    }
+
+    /**
+     * Creates this from an existing page template registry, using only built-in resolvers
+     *
+     * @param templateRegistry the page template registry. This will be frozen by this call.
+     * @param resolvers the resolvers to use, in addition to the default resolvers
+     */
+    public PageTemplateSearcher(PageTemplateRegistry templateRegistry, Resolver... resolvers) {
+        this(templateRegistry, Arrays.asList(resolvers));
+    }
+
+    private PageTemplateSearcher(PageTemplateRegistry templateRegistry, List<Resolver> resolvers) {
+        this.templateRegistry = templateRegistry;
+        templateRegistry.freeze();
+        this.resolverRegistry = new ResolverRegistry(resolvers);
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        // Pre execution: Choose template and sources
+        List<PageElement> pages=selectPageTemplates(query);
+        if (pages.isEmpty()) return execution.search(query); // Bypass if no page template chosen
+        addSources(pages,query);
+
+        // Set the page template list for inspection by other searchers
+        query.properties().set(pagePageTemplateListName, pages);
+
+        // Execute
+        Result result=execution.search(query);
+
+        // Post execution: Resolve choices and organize the result as dictated by the resolved template
+        Choice pageTemplateChoice=Choice.createSingletons(pages);
+        Resolution resolution=selectResolver(query).resolve(pageTemplateChoice,query,result);
+        organizer.organize(pageTemplateChoice,resolution,result);
+        return result;
+    }
+
+    /**
+     * Returns the list of page templates specified in the query, or the default if none, or the
+     * empty list if no default, never null.
+     */
+    private List<PageElement> selectPageTemplates(Query query) {
+        // Determine the list of page template ids
+        @SuppressWarnings("unchecked")
+        List<String> pageIds = (List<String>) query.properties().get(pageIdListName);
+        if (pageIds==null) {
+            String pageIdString=query.properties().getString(pageIdName,"").trim();
+            if (pageIdString.length()>0)
+                pageIds=Arrays.asList(pageIdString.split(" "));
+        }
+
+        // If none set, just return the default or null if none
+        if (pageIds==null) {
+            PageElement defaultPage=templateRegistry.getComponent("default");
+            return (defaultPage==null ? Collections.<PageElement>emptyList() : Collections.singletonList(defaultPage));
+        }
+
+        // Resolve the id list to page templates
+        List<PageElement> pages=new ArrayList<>(pageIds.size());
+        for (String pageId : pageIds) {
+            PageTemplate page=templateRegistry.getComponent(pageId);
+            if (page==null)
+                query.errors().add(ErrorMessage.createInvalidQueryParameter("Could not resolve requested page template '" +
+                                                                            pageId + "'"));
+            else
+                pages.add(page);
+        }
+
+        return pages;
+    }
+
+    private Resolver selectResolver(Query query) {
+        String resolverId=query.properties().getString(pageResolverName);
+        if (resolverId==null) return resolverRegistry.defaultResolver();
+        Resolver resolver=resolverRegistry.getComponent(resolverId);
+        if (resolver==null) throw new IllegalArgumentException("No page template resolver '" + resolverId + "'");
+        return resolver;
+    }
+
+    /** Sets query.getModel().getSources() to the right value and add source parameters specified in templates */
+    private void addSources(List<PageElement> pages,Query query) {
+        // Determine all wanted sources
+        Set<Source> pageSources=new HashSet<>();
+        for (PageElement page : pages)
+            pageSources.addAll(((PageTemplate)page).getSources());
+
+        addErrorIfSameSourceMultipleTimes(pages,pageSources,query);
+
+        if (query.getModel().getSources().size() > 0) {
+            // Add properties if the source list is set explicitly, but do not modify otherwise
+            addParametersForIncludedSources(pageSources,query);
+            return;
+        }
+
+        if (pageSources.contains(Source.any)) {
+            IntentModel intentModel=IntentModel.getFrom(query);
+            if (intentModel!=null) {
+                query.getModel().getSources().addAll(intentModel.getSourceNames());
+                addPageTemplateSources(pageSources,query);
+            }
+            // otherwise leave empty to search all
+        }
+        else { // Let the page templates decide
+            addPageTemplateSources(pageSources,query);
+        }
+    }
+
+    private void addPageTemplateSources(Set<Source> pageSources,Query query) {
+        for (Source pageSource : pageSources) {
+            if (pageSource==Source.any) continue;
+            query.getModel().getSources().add(pageSource.getName());
+            addParameters(pageSource,query);
+        }
+    }
+
+    private void addParametersForIncludedSources(Set<Source> sources,Query query) {
+        for (Source source : sources) {
+            if (source.parameters().size()>0 && query.getModel().getSources().contains(source.getName()))
+                addParameters(source,query);
+        }
+    }
+
+    /** Adds parameters specified in the source to the correct namespace in the query */
+    private void addParameters(Source source,Query query) {
+        for (Map.Entry<String,String> parameter : source.parameters().entrySet())
+            query.properties().set("source." + source.getName() + "." + parameter.getKey(),parameter.getValue());
+    }
+
+    /**
+     * Currently executing multiple queries to the same source with different parameter sets,
+     * is not supported. (Same parameter sets in multiple templates is supported,
+     * and will be just one entry in this set).
+     */
+    private void addErrorIfSameSourceMultipleTimes(List<PageElement> pages,Set<Source> sources,Query query) {
+        Set<String> sourceNames=new HashSet<>();
+        for (Source source : sources) {
+            if (sourceNames.contains(source.getName()))
+                query.errors().add(ErrorMessage.createInvalidQueryParameter(
+                        "Querying the same source multiple times with different parameter sets as part of one query " +
+                        "is not supported. " + pages + " requests this for source '" + source + "'"));
+            sourceNames.add(source.getName());
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java
new file mode 100644
index 00000000000..2d61d17ade8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderMappingVisitor.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates;
+
+import com.yahoo.search.pagetemplates.model.*;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Creates a map from placeholder id to the choice providing its value
+ * for all placeholder values visited.
+ * <p>
+ * This visitor will throw an IllegalArgumentException if the same placeholder id
+ * is referenced by two choices.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class PlaceholderMappingVisitor extends PageTemplateVisitor {
+
+    private Map<String, MapChoice> placeholderIdToChoice=new LinkedHashMap<>();
+
+    public @Override void visit(MapChoice mapChoice) {
+        List<String> placeholderIds=mapChoice.placeholderIds();
+        for (String placeholderId : placeholderIds) {
+            MapChoice existingChoice=placeholderIdToChoice.put(placeholderId,mapChoice);
+            if (existingChoice!=null)
+                throw new IllegalArgumentException("placeholder id '" + placeholderId + "' is referenced by both " +
+                        mapChoice + " and " + existingChoice + ": Only one reference is allowed");
+        }
+    }
+
+    public Map<String, MapChoice> getMap() { return placeholderIdToChoice; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java
new file mode 100644
index 00000000000..2e22ad7291e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PlaceholderReferenceCreatingVisitor.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates;
+
+import com.yahoo.search.pagetemplates.model.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Creates references from all placeholders to the choices which resolves them.
+ * If a placeholder is encountered which is not resolved by any choice, an IllegalArgumentException is thrown.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class PlaceholderReferenceCreatingVisitor extends PageTemplateVisitor {
+
+    private Map<String, MapChoice> placeholderIdToChoice=new HashMap<>();
+
+    public PlaceholderReferenceCreatingVisitor(Map<String, MapChoice> placeholderIdToChoice) {
+        this.placeholderIdToChoice=placeholderIdToChoice;
+    }
+
+    public @Override void visit(Placeholder placeholder) {
+        MapChoice choice=placeholderIdToChoice.get(placeholder.getId());
+        if (choice==null)
+            throw new IllegalArgumentException(placeholder + " is not referenced by any choice");
+        placeholder.setValueContainer(choice);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/SourceVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/SourceVisitor.java
new file mode 100644
index 00000000000..bf2685da56f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/SourceVisitor.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates;
+
+import com.yahoo.search.pagetemplates.model.PageTemplateVisitor;
+import com.yahoo.search.pagetemplates.model.Source;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Visits a page template object structure and records the sources mentioned.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class SourceVisitor extends PageTemplateVisitor {
+
+    private Set<Source> sources=new HashSet<>();
+
+    @Override
+    public void visit(Source source) {
+        sources.add(source);
+    }
+
+    /** Returns the live list of sources collected by this during visiting */
+    public Set<Source> getSources() { return sources; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateConfigurer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateConfigurer.java
new file mode 100644
index 00000000000..5d106a6df8e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateConfigurer.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.config;
+
+import com.yahoo.config.subscription.ConfigSubscriber;
+import com.yahoo.search.pagetemplates.PageTemplatesConfig;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.pagetemplates.PageTemplateRegistry;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Provides a static method to convert a page template config into a PageTemplateRegistry.
+ * In addition, instances of this can be created to subscribe to config and keep an up to date registry reference.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class PageTemplateConfigurer {
+
+    /**
+     * Creates a new page template registry from the content of a config and returns it.
+     * The returned registry will <b>not</b> be frozen. This should be done, by calling freeze(), before it is used.
+     */
+    public static PageTemplateRegistry toRegistry(PageTemplatesConfig config) {
+        List<NamedReader> pageReaders=new ArrayList<>();
+        int pageNumber=0;
+        for (String pageString : config.page())
+            pageReaders.add(new NamedReader("page[" + pageNumber++ + "]",new StringReader(pageString)));
+        return new PageTemplateXMLReader().read(pageReaders,false);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java
new file mode 100644
index 00000000000..46823f30cf2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/PageTemplateXMLReader.java
@@ -0,0 +1,355 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.config;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.pagetemplates.PageTemplate;
+import com.yahoo.search.pagetemplates.PageTemplateRegistry;
+import com.yahoo.search.pagetemplates.model.*;
+import com.yahoo.search.query.Sorting;
+import com.yahoo.text.XML;
+import org.w3c.dom.Element;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * Reads all page template XML files from a given directory (or list of readers).
+ * Instances of this are for single-thread usage only.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class PageTemplateXMLReader {
+
+    private static Logger logger=Logger.getLogger(PageTemplateXMLReader.class.getName());
+
+    /** The registry being constructed */
+    private PageTemplateRegistry registry;
+
+    /** XML elements by page id - available after phase 1. Needed for includes. */
+    private Map<ComponentId, Element> pageElementsByPageId=new LinkedHashMap<>();
+
+    /**
+     * Reads all page template xml files in a given directory.
+     *
+     * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML
+     */
+    public PageTemplateRegistry read(String directory) {
+        List<NamedReader> pageReaders=new ArrayList<>();
+        try {
+            File dir=new File(directory);
+            if ( !dir.isDirectory() ) throw new IllegalArgumentException("Could not read page templates: '" +
+                    directory + "' is not a valid directory.");
+
+            for (File file : sortFiles(dir)) {
+                if ( ! file.getName().endsWith(".xml")) continue;
+                pageReaders.add(new NamedReader(file.getName(),new FileReader(file)));
+            }
+
+            return read(pageReaders,true);
+        }
+        catch (IOException e) {
+            throw new IllegalArgumentException("Could not read page templates from '" + directory + "'",e);
+        }
+        finally {
+            for (NamedReader reader : pageReaders) {
+                try { reader.close(); } catch (IOException e) { }
+            }
+        }
+    }
+
+    /**
+     * Reads a single page template file.
+     *
+     * @throws RuntimeException if <code>fileName</code> is not a readable file, or if there is some error in the XML
+     */
+    public PageTemplate readFile(String fileName) {
+        NamedReader pageReader=null;
+        try {
+            File file=new File(fileName);
+            pageReader=new NamedReader(fileName,new FileReader(file));
+            String firstName=file.getName().substring(0,file.getName().length()-4);
+            return read(Collections.singletonList(pageReader),true).getComponent(firstName);
+        }
+        catch (IOException e) {
+            throw new IllegalArgumentException("Could not read the page template '" + fileName + "'",e);
+        }
+        finally {
+            if (pageReader!=null)
+                try { pageReader.close(); } catch (IOException e) { }
+        }
+    }
+
+    private List<File> sortFiles(File dir) {
+        ArrayList<File> files = new ArrayList<>();
+        files.addAll(Arrays.asList(dir.listFiles()));
+        Collections.sort(files);
+        return files;
+    }
+
+    /**
+     * Reads all page template xml files in a given list of readers. This is called from the Vespa configuration model.
+     *
+     * @param validateReaderNames should be set to true if the readers were created by files, not otherwise
+     * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML
+     */
+    public PageTemplateRegistry read(List<NamedReader> pageReaders,boolean validateReaderNames) {
+        // Initialize state
+        registry=new PageTemplateRegistry();
+
+        // Phase 1
+        pageElementsByPageId=createPages(pageReaders,validateReaderNames);
+        // Phase 2
+        readPages();
+        return registry;
+    }
+
+    private Map<ComponentId,Element> createPages(List<NamedReader> pageReaders,boolean validateReaderNames) {
+        Map<ComponentId,Element> pageElementsByPageId=new LinkedHashMap<>();
+        for (NamedReader reader : pageReaders) {
+            Element pageElement= XML.getDocument(reader).getDocumentElement();
+            if ( ! pageElement.getNodeName().equals("page")) {
+                logger.info("Ignoring '" + reader.getName() +
+                         "': Expected XML root element 'page' but was '" + pageElement.getNodeName() + "'");
+                continue;
+            }
+            String idString=pageElement.getAttribute("id");
+
+            if (idString==null || idString.isEmpty())
+                throw new IllegalArgumentException("Page template '" + reader.getName() + "' has no 'id' attribute in the root element");
+            ComponentId id=new ComponentId(idString);
+            if (validateReaderNames)
+                validateFileName(reader.getName(),id,"page template");
+            registry.register(new PageTemplate(id));
+            pageElementsByPageId.put(id,pageElement);
+        }
+        return pageElementsByPageId;
+    }
+
+    /** Throws an exception if the name is not corresponding to the id */
+    private void validateFileName(final String actualName,ComponentId id,String artifactName) {
+        String expectedCanonicalFileName=id.toFileName();
+        String fileName=new File(actualName).getName();
+        fileName=stripXmlEnding(fileName);
+        String canonicalFileName=ComponentId.fromFileName(fileName).toFileName();
+        if ( ! canonicalFileName.equals(expectedCanonicalFileName))
+            throw new IllegalArgumentException("The file name of " + artifactName + " '" + id +
+                                               "' must be '" + expectedCanonicalFileName + ".xml' but was '" + actualName + "'");
+    }
+
+    private String stripXmlEnding(String fileName) {
+        if (!fileName.endsWith(".xml"))
+            throw new IllegalArgumentException("'" + fileName + "' should have a .xml ending");
+        else
+            return fileName.substring(0,fileName.length()-4);
+    }
+
+    private void readPages() {
+        for (Map.Entry<ComponentId,Element> pageElement : pageElementsByPageId.entrySet()) {
+            try {
+                PageTemplate page=registry.getComponent(pageElement.getValue().getAttribute("id"));
+                readPageContent(pageElement.getValue(),page);
+            }
+            catch (IllegalArgumentException e) {
+                throw new IllegalArgumentException("Could not read page template '" + pageElement.getKey() + "'",e);
+            }
+        }
+    }
+
+    private void readPageContent(Element pageElement,PageTemplate page) {
+        if (page.isFrozen()) return; // Already read
+        Section rootSection=new Section(page.getId().toString());
+        readSection(pageElement,rootSection);
+        page.setSection(rootSection);
+        page.freeze();
+    }
+
+    /** Fills a section with attributes and sub-elements from a "section" or "page" element */
+    private Section readSection(Element sectionElement,Section section) {
+        section.setLayout(Layout.fromString(sectionElement.getAttribute("layout")));
+        section.setRegion(sectionElement.getAttribute("region"));
+        section.setOrder(Sorting.fromString(sectionElement.getAttribute("order")));
+        section.setMax(readOptionalNumber(sectionElement,"max"));
+        section.setMin(readOptionalNumber(sectionElement,"min"));
+        section.elements().addAll(readSourceAttribute(sectionElement));
+        section.elements().addAll(readPageElements(sectionElement));
+        return section;
+    }
+
+    /** Returns all page elements found under the given node */
+    private List<PageElement> readPageElements(Element parent) {
+        List<PageElement> pageElements=new ArrayList<>();
+        for (Element child : XML.getChildren(parent)) {
+            if (child.getNodeName().equals("include"))
+                pageElements.addAll(readInclude(child));
+            else
+                addIfNonNull(readPageElement(child),pageElements);
+        }
+        return pageElements;
+    }
+
+    private void addIfNonNull(PageElement pageElement,List<PageElement> pageElements) {
+        if (pageElement!=null)
+            pageElements.add(pageElement);
+    }
+
+    /** Reads the direct descendant elements of an include */
+    private List<PageElement> readInclude(Element element) {
+        PageTemplate included=registry.getComponent(element.getAttribute("idref"));
+        if (included==null)
+            throw new IllegalArgumentException("Could not find page template '" + element.getAttribute("idref"));
+        readPageContent(pageElementsByPageId.get(included.getId()),included);
+        return included.getSection().elements(Section.class);
+    }
+
+    /** Returns the page element corresponding to the given node, never null */
+    private PageElement readPageElement(Element child) {
+        if (child.getNodeName().equals("choice"))
+            return readChoice(child);
+        else if (child.getNodeName().equals("source"))
+            return readSource(child);
+        else if (child.getNodeName().equals("placeholder"))
+            return readPlaceholder(child);
+        else if (child.getNodeName().equals("section"))
+            return readSection(child,new Section(child.getAttribute("id")));
+        else if (child.getNodeName().equals("renderer"))
+            return readRenderer(child);
+        else if (child.getNodeName().equals("parameter"))
+            return null; // read elsewhere
+        throw new IllegalArgumentException("Unknown node type '" + child.getNodeName() + "'");
+    }
+
+    private List<Source> readSourceAttribute(Element sectionElement) {
+        List<Source> sources=new ArrayList<>();
+        String sourceAttributeString=sectionElement.getAttribute("source");
+        if (sourceAttributeString!=null) {
+            for (String sourceName : sourceAttributeString.split(" ")) {
+                if (sourceName.isEmpty()) continue;
+                if ("*".equals(sourceName))
+                    sources.add(Source.any);
+                else
+                    sources.add(new Source(sourceName));
+            }
+        }
+        return sources;
+    }
+
+    private Source readSource(Element sourceElement) {
+        Source source=new Source(sourceElement.getAttribute("name"));
+        source.setUrl(nullIfEmpty(sourceElement.getAttribute("url")));
+        source.renderers().addAll(readPageElements(sourceElement));
+        /*
+        source.renderers().addAll(readRenderers(XML.children(sourceElement,"renderer")));
+        readChoices(sourceElement,source);
+        */
+        source.parameters().putAll(readParameters(sourceElement));
+        return source;
+    }
+
+    private String nullIfEmpty(String s) {
+        if (s==null) return s;
+        s=s.trim();
+        if (s.isEmpty()) return null;
+        return s;
+    }
+
+    private Placeholder readPlaceholder(Element placeholderElement) {
+        return new Placeholder(placeholderElement.getAttribute("id"));
+    }
+
+    private Renderer readRenderer(Element rendererElement) {
+        Renderer renderer =new Renderer(rendererElement.getAttribute("name"));
+        renderer.setRendererFor(nullIfEmpty(rendererElement.getAttribute("for")));
+        renderer.parameters().putAll(readParameters(rendererElement));
+        return renderer;
+    }
+
+    private int readOptionalNumber(Element element,String attributeName) {
+        String attributeValue=element.getAttribute(attributeName);
+        try {
+            if (attributeValue.isEmpty()) return -1;
+            return Integer.parseInt(attributeValue);
+        }
+        catch (NumberFormatException e) { // Suppress original exception as it conveys no useful information
+            throw new IllegalArgumentException("'" + attributeName + "' in " + element + " must be a number, not '" + attributeValue + "'");
+        }
+    }
+
+    private AbstractChoice readChoice(Element choiceElement) {
+        String method=nullIfEmpty(choiceElement.getAttribute("method"));
+        if (XML.getChildren(choiceElement,"map").size()>0)
+            return readMapChoice(choiceElement,method);
+        else
+            return readNonMapChoice(choiceElement,method);
+    }
+
+    private MapChoice readMapChoice(Element choiceElement,String method) {
+        Element mapElement=XML.getChildren(choiceElement,"map").get(0);
+        MapChoice map=new MapChoice();
+        map.setMethod(method);
+
+        map.placeholderIds().addAll(readSpaceSeparatedAttribute("to",mapElement));
+        for (Element value : XML.getChildren(mapElement)) {
+            if ("item".equals(value.getNodeName()))
+                map.values().add(readPageElements(value));
+            else
+                map.values().add(Collections.singletonList(readPageElement(value)));
+        }
+        return map;
+    }
+
+    private Choice readNonMapChoice(Element choiceElement,String method) {
+        Choice choice=new Choice();
+        choice.setMethod(method);
+
+        for (Element alternative : XML.getChildren(choiceElement)) {
+            if (alternative.getNodeName().equals("alternative")) // Explicit alternative container
+                choice.alternatives().add(readPageElements(alternative));
+            else if (alternative.getNodeName().equals("include")) // Implicit include
+                choice.alternatives().add(readInclude(alternative));
+            else // Other implicit
+                choice.alternatives().add(Collections.singletonList(readPageElement(alternative)));
+        }
+        return choice;
+    }
+
+    /*
+    private void readChoices(Element sourceElement,Source source) {
+        for (Element choiceElement : XML.children(sourceElement,"choice")) {
+            for (Element alternative : XML.children(choiceElement)) {
+                if ("alternative".equals(alternative.getNodeName())) // Explicit alternative container
+                    source.renderer().alternatives().addAll(readRenderers(XML.children(alternative)));
+                else // Implicit alternative - yes implicit and explicit may be combined
+                    source.renderer().alternatives().addAll(readRenderers(Collections.singletonList(alternative)));
+            }
+        }
+    }
+    */
+
+    private Map<String,String> readParameters(Element containingElement) {
+        List<Element> parameterElements=XML.getChildren(containingElement,"parameter");
+        if (parameterElements.size()==0) return Collections.emptyMap(); // Shortcut
+
+        Map<String,String> parameters=new LinkedHashMap<>();
+        for (Element parameter : parameterElements) {
+            String key=parameter.getAttribute("name");
+            String value=XML.getValue(parameter);
+            parameters.put(key,value);
+        }
+        return parameters;
+    }
+
+    private List<String> readSpaceSeparatedAttribute(String attributeName, Element containingElement) {
+        List<String> values=new ArrayList<>();
+        String attributeString=nullIfEmpty(containingElement.getAttribute(attributeName));
+        if (attributeString!=null) {
+            for (String value : attributeString.split(" "))
+                values.add(value);
+        }
+        return values;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/config/package-info.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/package-info.java
new file mode 100644
index 00000000000..40cdfb691ab
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/config/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.pagetemplates.config;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java
new file mode 100644
index 00000000000..00e154d460b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Organizer.java
@@ -0,0 +1,177 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine;
+
+import com.yahoo.search.Result;
+import com.yahoo.search.pagetemplates.PageTemplate;
+import com.yahoo.search.pagetemplates.model.*;
+import com.yahoo.search.pagetemplates.result.SectionHitGroup;
+import com.yahoo.search.query.Sorting;
+import com.yahoo.search.result.*;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Reorganizes and prunes a result as prescribed by a resolved template.
+ * This class is multithread safe.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Organizer {
+
+    /**
+     * Organizes the given result
+     *
+     * @param templateChoice a choice between singleton lists of PageTemplates
+     * @param resolution the resolution of (at least) the template choice and all choices contained in that template
+     * @param result the result to organize
+     */
+    public void organize(Choice templateChoice, Resolution resolution, Result result) {
+        PageTemplate template=(PageTemplate)templateChoice.get(resolution.getResolution(templateChoice)).get(0);
+        SectionHitGroup sectionGroup =toGroup(template.getSection(),resolution,result);
+        ErrorHit errors=result.hits().getErrorHit();
+
+        // transfer state from existing hit
+        sectionGroup.setQuery(result.hits().getQuery());
+        if (errors!=null && errors instanceof DefaultErrorHit)
+            sectionGroup.add((DefaultErrorHit)errors);
+        for (Iterator<Map.Entry<String, Object>> it = result.hits().fieldIterator(); it.hasNext(); ) {
+            Map.Entry<String, Object> field = it.next();
+            sectionGroup.setField(field.getKey(), field.getValue());
+        }
+
+        result.setHits(sectionGroup);
+    }
+
+    /** Creates the hit group corresponding to a section, drawing data from the given result */
+    private SectionHitGroup toGroup(Section section,Resolution resolution,Result result) {
+        SectionHitGroup sectionGroup=new SectionHitGroup("section:" + section.getId());
+        setField("id",section.getId(),sectionGroup);
+        sectionGroup.setLeaf(section.elements(Section.class).size()==0);
+        setField("layout",section.getLayout().getName(),sectionGroup);
+        setField("region",section.getRegion(),sectionGroup);
+
+        List<String> sourceList=new ArrayList<>();
+        renderElements(resolution, result, sectionGroup, sourceList, section.elements());
+
+        // Trim to max
+        if (section.getMax()>=0)
+            sectionGroup.trim(0,section.getMax());
+        if (sectionGroup.size()>1)
+            assignOrderer(section,resolution,sourceList,sectionGroup);
+
+        return sectionGroup;
+    }
+
+    private void renderElements(Resolution resolution, Result result, SectionHitGroup sectionGroup, List<String> sourceList, List<PageElement> elements) {
+        for (PageElement element : elements) {
+            if (element instanceof Section) {
+                sectionGroup.add(toGroup((Section)element,resolution,result));
+            }
+            else if (element instanceof Source) {
+                addSource(resolution,(Source)element,sectionGroup,result,sourceList);
+            }
+            else if (element instanceof Renderer) {
+                sectionGroup.renderers().add((Renderer)element);
+            }
+            else if (element instanceof Choice) {
+                Choice choice=(Choice)element;
+                if (choice.isEmpty()) continue; // Ignore
+                int chosen=resolution.getResolution(choice);
+                renderElements(resolution, result, sectionGroup, sourceList, choice.alternatives().get(chosen));
+            }
+            else if (element instanceof Placeholder) {
+                Placeholder placeholder =(Placeholder)element;
+                List<PageElement> mappedElements=
+                        resolution.getResolution(placeholder.getValueContainer()).get(placeholder.getId());
+                renderElements(resolution,result,sectionGroup,sourceList,mappedElements);
+            }
+        }
+    }
+
+    private void setField(String fieldName,Object value,Hit to) {
+        if (value==null) return;
+        to.setField(fieldName,value);
+    }
+
+    private void addSource(Resolution resolution,Source source,SectionHitGroup sectionGroup,Result result,List<String> sourceList) {
+        renderElements(resolution,result,sectionGroup, sourceList, source.renderers());
+        /*
+        for (PageElement element : source.renderers()) {
+            if (element instanceof Renderer)
+            if (renderer.isEmpty()) continue;
+            sectionGroup.renderers().add(renderer.get(resolution.getResolution(renderer)));
+        }
+        */
+
+        if (source.getUrl()==null)
+            addHitsFromSource(source,sectionGroup,result,sourceList);
+        else
+            sectionGroup.sources().add(source); // source to be rendered by the frontend
+    }
+
+    private void addHitsFromSource(Source source,SectionHitGroup sectionGroup,Result result,List<String> sourceList) {
+        if (source==Source.any) { // Add any source not added yet
+            for (Hit hit : result.hits()) {
+                if ( ! (hit instanceof HitGroup)) continue;
+                String groupId=hit.getId().stringValue();
+                if ( ! groupId.startsWith("source:")) continue;
+                String sourceName=groupId.substring(7);
+                if (sourceList.contains(sourceName)) continue;
+                sectionGroup.addAll(((HitGroup)hit).asList());
+                sourceList.add(sourceName); // Add *'ed sources explicitly
+            }
+        }
+        else {
+            HitGroup sourceGroup=(HitGroup)result.hits().get("source:" + source.getName());
+            if (sourceGroup!=null)
+                sectionGroup.addAll(sourceGroup.asList());
+            sourceList.add(source.getName()); // Add even if not found - may be added later
+        }
+    }
+
+    private void assignOrderer(Section section,Resolution resolution,List<String> sourceList,HitGroup group) {
+        if (section.getOrder()==null) { // then sort by relevance, source
+            group.setOrderer(new HitSortOrderer(new RelevanceComparator(new SourceOrderComparator(sourceList))));
+            return;
+        }
+
+        // replace a source field comparison by one which knows the source list order
+        // and add default sorting at the end if necessary
+        Sorting sorting=section.getOrder();
+        int rankIndex=-1;
+        int sourceIndex=-1;
+        for (int i=0; i<sorting.fieldOrders().size(); i++) {
+            Sorting.FieldOrder order=sorting.fieldOrders().get(i);
+            if ("[relevance]".equals(order.getFieldName()) || "[rank]".equals(order.getFieldName()))
+                rankIndex=i;
+            else if (order.getFieldName().equals("[source]"))
+                sourceIndex=i;
+        }
+
+        ChainableComparator comparator;
+        Sorting beforeSource=null;
+        Sorting afterSource=null;
+        if (sourceIndex>=0) { // replace alphabetical sorting on source by sourceList order sorting
+            if (sourceIndex>0) // sort fields before the source
+                beforeSource=new Sorting(new ArrayList<>(sorting.fieldOrders().subList(0,sourceIndex)));
+            if (sorting.fieldOrders().size()>sourceIndex+1) // sort fields after the source
+                afterSource=new Sorting(new ArrayList<>(sorting.fieldOrders().subList(sourceIndex+1,sorting.fieldOrders().size()+1)));
+
+            comparator=new SourceOrderComparator(sourceList, FieldComparator.create(afterSource));
+            if (beforeSource!=null)
+                comparator=new FieldComparator(beforeSource,comparator);
+
+        }
+        else if (rankIndex>=0) { // add sort by source at the end
+            comparator=new FieldComparator(sorting,new SourceOrderComparator(sourceList));
+        }
+        else { // add sort by rank,source at the end
+            comparator=new FieldComparator(sorting,new RelevanceComparator(new SourceOrderComparator(sourceList)));
+        }
+        group.setOrderer(new HitSortOrderer(comparator));
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/RelevanceComparator.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/RelevanceComparator.java
new file mode 100644
index 00000000000..7489768b5a3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/RelevanceComparator.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine;
+
+import com.yahoo.search.result.ChainableComparator;
+import com.yahoo.search.result.Hit;
+
+import java.util.Comparator;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class RelevanceComparator extends ChainableComparator {
+
+    /**
+     * Creates a relevance comparator, with an optional secondary comparator.
+     * If the secondary is null, the intrinsic hit order is used as secondary.
+     */
+    public RelevanceComparator(Comparator<Hit> secondaryComparator) {
+        super(secondaryComparator);
+    }
+
+    public @Override int compare(Hit h1,Hit h2) {
+        int relevanceComparison=h2.getRelevance().compareTo(h1.getRelevance());
+        if (relevanceComparison!=0) return relevanceComparison;
+
+        return super.compare(h1,h2);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java
new file mode 100644
index 00000000000..d67faf805ad
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolution.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine;
+
+import com.yahoo.search.pagetemplates.model.Choice;
+import com.yahoo.search.pagetemplates.model.MapChoice;
+import com.yahoo.search.pagetemplates.model.PageElement;
+
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A resolution of choices within a template.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Resolution {
+
+    /** A record of choices made as choice → alternative index (id) */
+    private Map<Choice,Integer> choiceResolutions=new IdentityHashMap<>();
+
+    /** A of map choices made as choice → mapping */
+    private Map<MapChoice,Map<String,List<PageElement>>> mapChoiceResolutions=
+            new IdentityHashMap<>();
+
+    public void addChoiceResolution(Choice choice,int alternativeIndex) {
+        choiceResolutions.put(choice,alternativeIndex);
+    }
+
+    public void addMapChoiceResolution(MapChoice choice, Map<String,List<PageElement>> mapping) {
+        mapChoiceResolutions.put(choice,mapping);
+    }
+
+    /**
+     * Returns the resolution of a choice.
+     *
+     * @return the (0-base) index of the choice made. If the given choice has exactly one alternative,
+     *         0 is always returned (whether or not the choice has been attempted resolved).
+     * @throws IllegalArgumentException if the choice is empty, or if it has multiple alternatives but have not
+     *         been resolved in this
+     */
+    public int getResolution(Choice choice) {
+        if (choice.alternatives().size()==1) return 0;
+        if (choice.isEmpty()) throw new IllegalArgumentException("Cannot return a resolution of empty " + choice);
+        Integer resolution=choiceResolutions.get(choice);
+        if (resolution==null) throw new IllegalArgumentException(this + " has no resolution of " + choice);
+        return resolution;
+    }
+
+    /**
+     * Returns the resolution of a map choice.
+     *
+     * @return the chosen mapping - entries from placeholder id to the values to use at the location of that placeholder
+     * @throws IllegalArgumentException if this choice has not been resolved in this
+     */
+    public Map<String,List<PageElement>> getResolution(MapChoice choice) {
+        Map<String,List<PageElement>> resolution=mapChoiceResolutions.get(choice);
+        if (resolution==null) throw new IllegalArgumentException(this + " has no resolution of " + choice);
+        return resolution;
+    }
+
+    public @Override String toString() {
+        return "a resolution of " + choiceResolutions.size() + " choices";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolver.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolver.java
new file mode 100644
index 00000000000..4972b0e4689
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/Resolver.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.pagetemplates.PageTemplate;
+import com.yahoo.search.pagetemplates.model.Choice;
+import com.yahoo.search.pagetemplates.model.MapChoice;
+import com.yahoo.search.pagetemplates.model.PageTemplateVisitor;
+
+/**
+ * Superclass of page template choice resolvers.
+ * <p>
+ * Subclasses overrides one of the two resolve methods to either resolve each choices individually
+ * or look at all choices at once.
+ * <p>
+ * All subclasses of this must be multithread safe. I.e multiple calls may be made
+ * to resolve at the same time from different threads.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public abstract class Resolver extends AbstractComponent {
+
+    public Resolver(String id) {
+        super(new ComponentId(id));
+    }
+
+    public Resolver(ComponentId id) {
+        super(id);
+    }
+
+    protected Resolver() {}
+
+    /**
+     * Override this to resolve choices. Before retuning this method <i>must</i> resolve the given choice
+     * between a set of page templates <i>and</i> all choices found recursively within the <i>chosen</i>
+     * page template. It is permissible but not required to add solutions also to choices present within those
+     * templates which are not chosen.
+     * <p>
+     * This default implementation creates a Resolution and calls
+     * <code>resolve(choice/mapChoice,query,result,resolution)</code> first on the given page template choice, then
+     * on each choice found in that temnplate. This provides a simple API to resolvers which make each choice
+     * independently.
+     *
+     * @param pageTemplate the choice of page templates to resolve - a choice containing singleton lists of PageTemplate elements
+     * @param query the query, from which information useful for correct resolution can be found
+     * @param result the result, from which further information useful for correct resolution can be found
+     * @return the resolution of the choices contained in the given page template
+     */
+    public Resolution resolve(Choice pageTemplate, Query query, Result result) {
+        Resolution resolution=new Resolution();
+        resolve(pageTemplate,query,result,resolution);
+        PageTemplate chosenPageTemplate=(PageTemplate)pageTemplate.get(resolution.getResolution(pageTemplate)).get(0);
+        ChoiceResolverVisitor choiceResolverVisitor=new ChoiceResolverVisitor(query,result,resolution);
+        chosenPageTemplate.accept(choiceResolverVisitor);
+        return choiceResolverVisitor.getResolution();
+    }
+
+    /**
+     * Override this to resolve <i>each</i> choice independently.
+     * This default implementation does nothing.
+     *
+     * @param choice the choice to resolve
+     * @param query the query for which this should be resolved, typically used to extract features
+     * @param result the result for which this should be resolved, typically used to extract features
+     * @param resolution the set of resolutions made so far, to which this should be added:
+     *        <code>resolution.addChoiceResolution(choice,chosenAlternativeIndex)</code>
+     */
+    public void resolve(Choice choice,Query query,Result result,Resolution resolution) {
+    }
+
+    /**
+     * Override this to resolve <i>each</i> map choice independently.
+     * This default implementation does nothing.
+     *
+     * @param choice the choice to resolve
+     * @param query the query for which this should be resolved, typically used to extract features
+     * @param result the result for which this should be resolved, typically used to extract features
+     * @param resolution the set of resolutions made so far, to which this should be added:
+     *        <code>resolution.addMapChoiceResolution(choice,chosenMapping)</code>
+     */
+    public void resolve(MapChoice choice,Query query,Result result,Resolution resolution) {
+    }
+
+    private class ChoiceResolverVisitor extends PageTemplateVisitor {
+
+        private Resolution resolution;
+
+        private Query query;
+
+        private Result result;
+
+        public ChoiceResolverVisitor(Query query,Result result,Resolution resolution) {
+            this.query=query;
+            this.result=result;
+            this.resolution=resolution;
+        }
+
+        public @Override void visit(Choice choice) {
+            if (choice.alternatives().size()<2) return; // No choice...
+            resolve(choice,query,result,resolution);
+        }
+
+        public @Override void visit(MapChoice choice) {
+            resolve(choice,query,result,resolution);
+        }
+
+        public Resolution getResolution() { return resolution; }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/SourceOrderComparator.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/SourceOrderComparator.java
new file mode 100644
index 00000000000..b4cd01f0c36
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/SourceOrderComparator.java
@@ -0,0 +1,57 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine;
+
+import com.yahoo.search.result.ChainableComparator;
+import com.yahoo.search.result.Hit;
+
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+class SourceOrderComparator extends ChainableComparator {
+
+    private final List<String> sourceOrder;
+
+    /**
+     * Creates a source order comparator, with no secondary
+     *
+     * @param sourceOrder the sort order of list names. This list gets owned by this and must not be modified
+     */
+    public SourceOrderComparator(List<String> sourceOrder) {
+        this(sourceOrder,null);
+    }
+
+    /**
+     * Creates a source order comparator, with an optional secondary comparator.
+     *
+     * @param sourceOrder the sort order of list names. This list gets owned by this and must not be modified
+     * @param secondaryComparator the comparator to use as secondary, or null to use the intrinsic hit order
+     */
+    public SourceOrderComparator(List<String> sourceOrder,Comparator<Hit> secondaryComparator) {
+        super(secondaryComparator);
+        this.sourceOrder=sourceOrder;
+    }
+
+    public @Override int compare(Hit h1,Hit h2) {
+        int primaryOrder=sourceOrderCompare(h1,h2);
+        if (primaryOrder!=0) return primaryOrder;
+
+        return super.compare(h1,h2);
+    }
+
+    private int sourceOrderCompare(Hit h1,Hit h2) {
+        String h1Source=h1.getSource();
+        String h2Source=h2.getSource();
+
+        if (h1Source==null && h2Source==null) return 0;
+        if (h1Source==null) return 1; // No source -> last
+        if (h2Source==null) return -1; // No source -> last
+
+        if (h1Source.equals(h2Source)) return 0;
+
+        return sourceOrder.indexOf(h1Source)-sourceOrder.indexOf(h2Source);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/package-info.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/package-info.java
new file mode 100644
index 00000000000..6628156cb33
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.pagetemplates.engine;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/DeterministicResolver.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/DeterministicResolver.java
new file mode 100644
index 00000000000..32ed54a6775
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/DeterministicResolver.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine.resolvers;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.pagetemplates.engine.Resolution;
+import com.yahoo.search.pagetemplates.engine.Resolver;
+import com.yahoo.search.pagetemplates.model.Choice;
+import com.yahoo.search.pagetemplates.model.MapChoice;
+import com.yahoo.search.pagetemplates.model.PageElement;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A resolver which
+ * <ul>
+ *   <li>Always chooses the <i>last</i> alternative of any Choice
+ *   <li>Always maps values to placeholders in the order they are listed in the map definition of any MapChoice
+ * </ul>
+ * This is useful for testing.
+ * <p>
+ * The id of this if <code>native.deterministic</code>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DeterministicResolver extends Resolver {
+    public static final String nativeId = "native.deterministic";
+
+    public DeterministicResolver() {}
+
+    protected DeterministicResolver(String id) {
+        super(id);
+    }
+
+    /** Chooses the last alternative of any choice */
+    @Override
+    public void resolve(Choice choice, Query query, Result result, Resolution resolution) {
+        resolution.addChoiceResolution(choice,choice.alternatives().size()-1);
+    }
+
+    /** Chooses a mapping which is always by the literal order given in the source template */
+    @Override
+    public void resolve(MapChoice choice,Query query,Result result,Resolution resolution) {
+        Map<String, List<PageElement>> mapping=new HashMap<>();
+        // Map 1-1 by order
+        List<String> placeholderIds=choice.placeholderIds();
+        List<List<PageElement>> valueList=choice.values();
+        int i=0;
+        for (String placeholderId : placeholderIds)
+            mapping.put(placeholderId,valueList.get(i++));
+        resolution.addMapChoiceResolution(choice,mapping);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/RandomResolver.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/RandomResolver.java
new file mode 100644
index 00000000000..5f06c66795d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/RandomResolver.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine.resolvers;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.pagetemplates.engine.Resolution;
+import com.yahoo.search.pagetemplates.engine.Resolver;
+import com.yahoo.search.pagetemplates.model.Choice;
+import com.yahoo.search.pagetemplates.model.MapChoice;
+import com.yahoo.search.pagetemplates.model.PageElement;
+
+import java.util.*;
+
+/**
+ * A resolver which makes all choices by random.
+ * The id of this is <code>native.random</code>.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class RandomResolver extends Resolver {
+
+    public static final String nativeId = "native.random";
+
+    private Random random = new Random(System.currentTimeMillis()); // Use of this is multithread safe
+
+    public RandomResolver() {}
+
+    protected RandomResolver(String id) {
+        super(id);
+    }
+
+    /** Chooses the last alternative of any choice */
+    @Override
+    public void resolve(Choice choice, Query query, Result result, Resolution resolution) {
+        resolution.addChoiceResolution(choice,random.nextInt(choice.alternatives().size()));
+    }
+
+    /** Chooses a mapping which is always by the literal order given in the source template */
+    @Override
+    public void resolve(MapChoice choice,Query query,Result result,Resolution resolution) {
+        Map<String, List<PageElement>> mapping=new HashMap<>();
+        // Draw a random element from the value list on each iteration and assign it to a placeholder
+        List<String> placeholderIds=choice.placeholderIds();
+        List<List<PageElement>> valueList=new ArrayList<>(choice.values());
+        for (String placeholderId : placeholderIds)
+            mapping.put(placeholderId,valueList.remove(random.nextInt(valueList.size())));
+        resolution.addMapChoiceResolution(choice,mapping);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/ResolverRegistry.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/ResolverRegistry.java
new file mode 100644
index 00000000000..0bbbec655bd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/ResolverRegistry.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.engine.resolvers;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.pagetemplates.engine.Resolver;
+
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * A registry of available resolver components
+ *
+ * @author bratseth
+ */
+public class ResolverRegistry extends ComponentRegistry<Resolver> {
+
+    private final Resolver defaultResolver;
+
+    public ResolverRegistry(List<Resolver> resolvers) {
+        addBuiltInResolvers();
+        for (Resolver component : resolvers)
+            registerResolver(component);
+        defaultResolver = decideDefaultResolver();
+        freeze();
+    }
+
+    private void addBuiltInResolvers() {
+        registerResolver(createNativeDeterministicResolver());
+        registerResolver(createNativeRandomResolver());
+    }
+
+    private Resolver decideDefaultResolver() {
+        Resolver defaultResolver = getComponent("default");
+        if (defaultResolver != null) return defaultResolver;
+        return getComponent("native.random");
+    }
+
+    private Resolver createNativeRandomResolver() {
+        RandomResolver resolver = new RandomResolver();
+        resolver.initId(ComponentId.fromString(RandomResolver.nativeId));
+        return resolver;
+    }
+
+    private DeterministicResolver createNativeDeterministicResolver() {
+        DeterministicResolver resolver = new DeterministicResolver();
+        resolver.initId(ComponentId.fromString(DeterministicResolver.nativeId));
+        return resolver;
+    }
+
+    private void registerResolver(Resolver resolver) {
+        super.register(resolver.getId(), resolver);
+    }
+
+    public Resolver defaultResolver() { return defaultResolver; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/package-info.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/package-info.java
new file mode 100644
index 00000000000..c1e3f218480
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/engine/resolvers/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.pagetemplates.engine.resolvers;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/AbstractChoice.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/AbstractChoice.java
new file mode 100644
index 00000000000..069598b2e02
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/AbstractChoice.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import com.yahoo.component.provider.FreezableClass;
+
+/**
+ * Abstract superclass of various kinds of choices.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public abstract class AbstractChoice extends FreezableClass implements PageElement {
+
+    private String method;
+
+    /**
+     * Returns the choice method to use - a string interpreted by the resolver in use,
+     * or null to use any available method
+     */
+    public String getMethod() { return method; }
+
+    public void setMethod(String method) {
+        ensureNotFrozen();
+        this.method=method;
+    }
+
+    // TODO: is this really choices between classes in general, or e.g. subclasses of Section?
+    /** Returns true if this choice is (partially or completely) a choice between the given type */
+    @SuppressWarnings("rawtypes")
+    public abstract boolean isChoiceBetween(Class pageTemplateModelClass);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Choice.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Choice.java
new file mode 100644
index 00000000000..a1932012236
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Choice.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import java.util.*;
+
+/**
+ * A choice between some alternative lists of page elements.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public final class Choice extends AbstractChoice {
+
+    private List<List<PageElement>> alternatives=new ArrayList<>(3);
+
+    /** Creates an empty choice */
+    public Choice() { }
+
+    /** Creates a choice having a single alternative having a single page element */
+    public static Choice createSingleton(PageElement singletonAlternative) {
+        Choice choice=new Choice();
+        choice.alternatives().add(createSingletonList(singletonAlternative));
+        return choice;
+    }
+
+    /** Creates a choice in which each alternative consists of a single element */
+    public static Choice createSingletons(List<PageElement> alternatives) {
+        Choice choice=new Choice();
+        for (PageElement alternative : alternatives)
+            choice.alternatives().add(createSingletonList(alternative));
+        return choice;
+    }
+
+    private static List<PageElement> createSingletonList(PageElement member) {
+        List<PageElement> list=new ArrayList<>();
+        list.add(member);
+        return list;
+    }
+
+    /**
+     * Creates a choice between some alternatives. This method takes a copy of the given lists.
+     */
+    public Choice(List<List<PageElement>> alternatives) {
+        for (List<PageElement> alternative : alternatives)
+            this.alternatives.add(new ArrayList<>(alternative));
+    }
+
+    /**
+     * Returns the alternatives of this as a live reference to the alternatives of this.
+     * The list and elements may be modified unless this is frozen. This is never null.
+     */
+    public List<List<PageElement>> alternatives() { return alternatives; }
+
+    /** Convenience shorthand of <code>return alternatives().get(index)</code> */
+    public List<PageElement> get(int index) {
+        return alternatives.get(index);
+    }
+
+    /** Convenience shorthand for <code>if (alternative!=null) alternatives().add(alternative)</code> */
+    public void add(List<PageElement> alternative) {
+        if (alternative!=null)
+            alternatives.add(new ArrayList<>(alternative));
+    }
+
+    /** Returns true only if there are no alternatives in this */
+    public boolean isEmpty() { return alternatives.size()==0; }
+
+    /** Answers true if this is either a choice between the given class, or between Lists of the given class */
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Override
+    public boolean isChoiceBetween(Class pageTemplateModelElementClass) {
+        List firstNonEmpty=null;
+        for (List<PageElement> value : alternatives) {
+            if (pageTemplateModelElementClass.isAssignableFrom(value.getClass())) return true;
+            if (value instanceof List) {
+                List listValue=(List)value;
+                if (listValue.size()>0)
+                    firstNonEmpty=listValue;
+            }
+        }
+        if (firstNonEmpty==null) return false;
+        return (pageTemplateModelElementClass.isAssignableFrom(firstNonEmpty.get(0).getClass()));
+    }
+
+    @Override
+    public void freeze() {
+        if (isFrozen()) return;
+        super.freeze();
+        for (ListIterator<List<PageElement>> i=alternatives.listIterator(); i.hasNext(); ) {
+            List<PageElement> alternative=i.next();
+            for (PageElement alternativeElement : alternative)
+                alternativeElement.freeze();
+            i.set(Collections.unmodifiableList(alternative));
+        }
+        alternatives= Collections.unmodifiableList(alternatives);
+    }
+
+    /** Accepts a visitor to this structure */
+    @Override
+    public void accept(PageTemplateVisitor visitor) {
+        visitor.visit(this);
+        for (List<PageElement> alternative : alternatives) {
+            for (PageElement alternativeElement : alternative)
+                alternativeElement.accept(visitor);
+        }
+    }
+
+    @Override
+    public String toString() {
+        if (alternatives.isEmpty()) return "(empty choice)";
+        if (alternatives.size()==1) return alternatives.get(0).toString();
+        return "a choice between " + alternatives;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Layout.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Layout.java
new file mode 100644
index 00000000000..f8e00b78787
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Layout.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+/**
+ * The layout of a section
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+// This is not made an enum, to allow the value set to be extendible.
+// It is not explicitly made immutable
+// to enable adding of internal state later (esp. parameters).
+// If this becomes mutable, the creation scheme must be changed
+// such that each fromString returns a unique instance, and
+// the name must become a (immutable) type.
+public class Layout {
+
+    /** The built in "column" layout */
+    public static final Layout column=new Layout("column");
+    /** The built in "row" layout */
+    public static final Layout row=new Layout("row");
+
+    private String name;
+
+    public Layout(String name) {
+        this.name=name;
+    }
+
+    public String getName() { return name; }
+
+    public @Override int hashCode() { return name.hashCode(); }
+
+    public @Override boolean equals(Object o) {
+        if (o==this) return true;
+        if (! (o instanceof Layout)) return false;
+        Layout other=(Layout)o;
+        return this.name.equals(other.name);
+    }
+
+    /** Returns a layout having this string as name, or null if the given string is null or empty */
+    public static Layout fromString(String layout) {
+        //if (layout==null) return null;
+        //if (layout)
+        if (layout.equals("column")) return column;
+        if (layout.equals("row")) return row;
+        return new Layout(layout);
+    }
+
+    public @Override String toString() { return "layout '" + name + "'"; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/MapChoice.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/MapChoice.java
new file mode 100644
index 00000000000..33c3bba9a77
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/MapChoice.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A choice between different possible mapping functions of a set of values to a set of placeholder ids.
+ * A <i>resolution</i> of this choice consists of choosing a unique value for each placeholder id
+ * (hence a map choice is valid iff there are at least as many values as placeholder ids).
+ * <p>
+ * Each unique set of mappings (pairs) from values to placeholder ids is a separate possible
+ * alternative of this choice. The alternatives are not listed explicitly but are generated as needed.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class MapChoice extends AbstractChoice {
+
+    private List<String> placeholderIds=new ArrayList<>();
+
+    private List<List<PageElement>> values=new ArrayList<>();
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Override
+    public boolean isChoiceBetween(Class pageTemplateModelElementClass) {
+        List<PageElement> firstNonEmpty=null;
+        for (List<PageElement> value : values)
+            if (value.size()>0)
+                firstNonEmpty=value;
+        if (firstNonEmpty==null) return false;
+        return (pageTemplateModelElementClass.isAssignableFrom(firstNonEmpty.get(0).getClass()));
+    }
+
+    /**
+     * Returns the placeholder ids (the "to" of the mapping) of this as a live reference which can be modified unless
+     * this is frozen.
+     */
+    public List<String> placeholderIds() { return placeholderIds; }
+
+    /**
+     * Returns the values (the "from" of the mapping) of this as a live reference which can be modified unless
+     * this is frozen. Note that each single choice of values within this is also a list of values. This is
+     * the inner list.
+     */
+    public List<List<PageElement>> values() { return values; }
+
+    @Override
+    public void freeze() {
+        if (isFrozen()) return;
+        super.freeze();
+        placeholderIds=Collections.unmodifiableList(placeholderIds);
+        values=Collections.unmodifiableList(values);
+    }
+
+    /** Accepts a visitor to this structure */
+    public @Override void accept(PageTemplateVisitor visitor) {
+        visitor.visit(this);
+        for (List<PageElement> valueEntry : values)
+            for (PageElement value : valueEntry)
+                value.accept(visitor);
+    }
+
+    @Override
+    public String toString() {
+        return "mapping to placeholders " + placeholderIds;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/PageElement.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/PageElement.java
new file mode 100644
index 00000000000..fba58f069ec
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/PageElement.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import com.yahoo.component.provider.Freezable;
+
+/**
+ * Implemented by all page template model classes
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public interface PageElement extends Freezable {
+
+    /** Accepts a visitor to this structure */
+    public void accept(PageTemplateVisitor visitor);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/PageTemplateVisitor.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/PageTemplateVisitor.java
new file mode 100644
index 00000000000..d7ebd3d1169
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/PageTemplateVisitor.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import com.yahoo.search.pagetemplates.PageTemplate;
+
+/**
+ * Superclass of visitors over the page template object structure
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class PageTemplateVisitor {
+
+    /** Called each time a page template is encountered. This default implementation does nothing */
+    public void visit(PageTemplate pageTemplate) {
+    }
+
+    /** Called each time a source or source placeholder is encountered. This default implementation does nothing */
+    public void visit(Source source) {
+    }
+
+    /** Called each time a section or section placeholder is encountered. This default implementation does nothing */
+    public void visit(Section section) {
+    }
+
+    /** Called each time a renderer is encountered. This default implementation does nothing */
+    public void visit(Renderer renderer) {
+    }
+
+    /** Called each time a choice is encountered. This default implementation does nothing */
+    public void visit(Choice choice) {
+    }
+
+    /** Called each time a map choice is encountered. This default implementation does nothing */
+    public void visit(MapChoice choice) {
+    }
+
+    /** Called each time a placeholder is encountered. This default implementation does nothing */
+    public void visit(Placeholder placeholder) {
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Placeholder.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Placeholder.java
new file mode 100644
index 00000000000..cf7a85fc779
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Placeholder.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+/**
+ * A source placeholder is replaced with a list of source instances at evaluation time.
+ * Source placeholders may not have any content themselves - attempting to call any setter on this
+ * results in a IllegalStateException.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Placeholder implements PageElement {
+
+    private String id;
+
+    private MapChoice valueContainer=null;
+
+    /** Creates a source placeholder with an id. */
+    public Placeholder(String id) {
+        this.id=id;
+    }
+
+    public String getId() { return id; }
+
+    /** Returns the element which contains the value(s) of this placeholder. Never null. */
+    public MapChoice getValueContainer() { return valueContainer; }
+
+    public void setValueContainer(MapChoice valueContainer) { this.valueContainer=valueContainer; }
+
+    public @Override void freeze() {}
+
+    /** Accepts a visitor to this structure */
+    public @Override void accept(PageTemplateVisitor visitor) {
+        visitor.visit(this);
+    }
+
+    public @Override String toString() {
+        return "source placeholder '" + id + "'";
+    }
+
+    /**
+     * This method always returns false, is a Placeholder always is mutable.
+     * (freeze() is a NOOP.)
+     */
+    @Override
+    public boolean isFrozen() {
+        return false;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Renderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Renderer.java
new file mode 100644
index 00000000000..4564ceeef3c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Renderer.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import com.yahoo.component.provider.FreezableClass;
+import com.yahoo.protect.Validator;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * A description of a way to present data items from a source.
+ * All data items has a default renderer. This can be overridden or parametrized by
+ * an explicit renderer.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public final class Renderer extends FreezableClass implements PageElement {
+
+    private String name;
+
+    private String rendererFor;
+
+    private Map<String,String> parameters =new LinkedHashMap<>();
+
+    public Renderer(String name) {
+        setName(name);
+    }
+
+    /**
+     * Returns the name of this renderer (never null).
+     * The name should be recognized by the system receiving results for rendering
+     */
+    public String getName() { return name; }
+
+    public final void setName(String name) {
+        ensureNotFrozen();
+        Validator.ensureNotNull("renderer name",name);
+        this.name=name;
+    }
+
+    /**
+     * Returns the name of the kind of data this is a renderer for.
+     * This is used to allow frontends to dispatch the right data items (hits) to
+     * the right renderer in the case where the data consists of a heterogeneous list.
+     * <p>
+     * This is null if this is a renderer for a whole section, or if this is a renderer
+     * for all kinds of data from a particular source <i>and</i> this is not frozen.
+     * <p>
+     * Otherwise, it is either the name of the source this is the renderer for,
+     * <i>or</i> the renderer for all data items having this name as a <i>type</i>.
+     * <p>
+     * This, a (frontend) dispatcher of data to renderers should for each data item:
+     * <ul>
+     *     <li>use the renderer having the same name as any <code>type</code> name set of the data item
+     *     <li>if no such renderer, use the renderer having <code>rendererFor</code> equal to the data items <code>source</code>
+     *     <li>if no such renderer, use a default renderer
+     * </ul>
+     */
+    public String getRendererFor() { return rendererFor; }
+
+    public void setRendererFor(String rendererFor) {
+        ensureNotFrozen();
+        this.rendererFor=rendererFor;
+    }
+
+    /**
+     * Returns the parameters of this renderer as a live reference (never null).
+     * The parameters will be passed to the renderer with each result
+     */
+    public Map<String,String> parameters() { return parameters; }
+
+    public @Override void freeze() {
+        if (isFrozen()) return;
+        super.freeze();
+        parameters = Collections.unmodifiableMap(parameters);
+    }
+
+    /** Accepts a visitor to this structure */
+    public @Override void accept(PageTemplateVisitor visitor) {
+        visitor.visit(this);
+    }
+     public @Override String toString() {
+        return "renderer '" + name + "'";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Section.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Section.java
new file mode 100644
index 00000000000..0a980419853
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Section.java
@@ -0,0 +1,177 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import com.yahoo.component.provider.FreezableClass;
+import com.yahoo.search.query.Sorting;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * An element of a page template corresponding to a physical area of the layout of the final physical page.
+ * Pages are freezable - once frozen calling a setter will cause an IllegalStateException, and returned
+ * live collection references are unmodifiable
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Section extends FreezableClass implements PageElement {
+
+    private final String id;
+
+    private Layout layout=Layout.column;
+
+    private String region;
+
+    /** The elements of this - sources, subsections etc. and/or choices of the same */
+    private List<PageElement> elements=new ArrayList<>();
+
+    /** Filtered versions of elements pre-calculated at freeze time */
+    private List<PageElement> sections, sources, renderers;
+
+    private int max=-1;
+
+    private int min=-1;
+
+    private Sorting order=null;
+
+    private static AtomicInteger nextId=new AtomicInteger();
+
+    public Section() {
+        this(null);
+    }
+
+    /** Creates a section with an id (or null if no id) */
+    public Section(String id) {
+        if (id==null || id.isEmpty())
+            this.id=String.valueOf("section_" + nextId.incrementAndGet());
+        else
+            this.id=id;
+    }
+
+    /** Returns a unique id of this section within the page. Used for referencing and identification. Never null. */
+    public String getId() { return id; }
+
+    /**
+     * Returns the layout identifier describing the kind of layout which should be used by the rendering engine to
+     * lay out the content of this section. This is never null. Default: "column".
+     */
+    public Layout getLayout() { return layout; }
+
+    /** Sets the layout. If the layout is set to null it will become Layout.column */
+    public void setLayout(Layout layout) {
+        ensureNotFrozen();
+        if (layout==null) layout=Layout.column;
+        this.layout=layout;
+    }
+
+    /**
+     * Returns the identifier telling the layout of the containing section where this section should be placed.
+     * Permissible values, and whether this is mandatory is determined by the particular layout identifier of the parent.
+     * May be null if a placement is not required by the containing layout, or if this is the top-level section.
+     * This is null by default.
+     */
+    public String getRegion() { return region; }
+
+    public void setRegion(String region) {
+        ensureNotFrozen();
+        this.region=region;
+    }
+
+    /**
+     * Returns the elements of this - sources, subsections and presentations and/or choices of these,
+     * as a live reference which can be modified to change the content of this (unless this is frozen).
+     * <p>
+     * All elements are kept in a single list to allow multiple elements of each type to be nested within separate
+     * choices, and to maintain the internal order of elements of various types, which is sometimes significant.
+     * To extract a certain kind of elements (say, sources), the element list must be traversed to collect
+     * all source elements as well as all choices of sources.
+     * <p>
+     * This list is never null but may be empty.
+     */
+    public List<PageElement> elements() { return elements; }
+
+    /**
+     * Convenience method which returns the elements <b>and choices</b> of the given type in elements as a
+     * read-only list. Not that as this returns both concrete elements and choices betwen them,
+     * the list element cannot be case to the given class - this must be used in conjunction
+     * with a resolve which contains the resolution to the choices.
+     *
+     * @param pageTemplateModelElementClass type to returns elements and choices of, a subtype of PageElement
+     */
+    public List<PageElement> elements(@SuppressWarnings("rawtypes") Class pageTemplateModelElementClass) {
+        if (isFrozen()) { // Use precalculated lists
+            if (pageTemplateModelElementClass==Section.class)
+                return sections;
+            else if (pageTemplateModelElementClass==Source.class)
+                return sources;
+            else if (pageTemplateModelElementClass==Renderer.class)
+                return renderers;
+        }
+        return createElementList(pageTemplateModelElementClass);
+    }
+
+    @SuppressWarnings("unchecked")
+    private List<PageElement> createElementList(@SuppressWarnings("rawtypes") Class pageTemplateModelElementClass) {
+        List<PageElement> filteredElements=new ArrayList<>();
+        for (PageElement element : elements) {
+            if (pageTemplateModelElementClass.isAssignableFrom(element.getClass()))
+                filteredElements.add(element);
+            else if (element instanceof AbstractChoice)
+                if (((AbstractChoice)element).isChoiceBetween(pageTemplateModelElementClass))
+                    filteredElements.add(element);
+        }
+        return Collections.unmodifiableList(filteredElements);
+    }
+
+    /** Returns the choice of ways to sort immediate children in this, or empty meaning sort by default order (relevance) */
+    public Sorting getOrder() { return order; }
+
+    public void setOrder(Sorting order) {
+        ensureNotFrozen();
+        this.order=order;
+    }
+
+    /** Returns max number of (immediate) elements/sections permissible within this, -1 means unrestricted. Default: -1. */
+    public int getMax() { return max; }
+
+    public void setMax(int max) {
+        ensureNotFrozen();
+        this.max=max;
+    }
+
+    /** Returns min number of (immediate) elements/sections desired within this, -1 means unrestricted. Default: -1. */
+    public int getMin() { return min; }
+
+    public void setMin(int min) {
+        ensureNotFrozen();
+        this.min=min;
+    }
+
+    public @Override void freeze() {
+        if (isFrozen()) return;
+
+        for (PageElement element : elements)
+            element.freeze();
+        elements=Collections.unmodifiableList(elements);
+        sections=createElementList(Section.class);
+        sources=createElementList(Source.class);
+        renderers=createElementList(Renderer.class);
+
+        super.freeze();
+    }
+
+    /** Accepts a visitor to this structure */
+    public @Override void accept(PageTemplateVisitor visitor) {
+        visitor.visit(this);
+        for (PageElement element : elements)
+            element.accept(visitor);
+    }
+
+    public @Override String toString() {
+        if (id==null || id.isEmpty()) return "a section";
+        return "section '" + id + "'";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Source.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Source.java
new file mode 100644
index 00000000000..91c403eae84
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/Source.java
@@ -0,0 +1,137 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.model;
+
+import com.yahoo.component.provider.FreezableClass;
+import com.yahoo.protect.Validator;
+
+import java.util.*;
+
+/**
+ * A source mentioned in a page template.
+ * <p>
+ * Two sources are equal if they have the same name and parameters.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Source extends FreezableClass implements PageElement {
+
+    /** The "any" source - used to mark that any source is acceptable here */
+    public static final Source any=new Source("*",true);
+
+    /** The obligatory name of a source */
+    private String name;
+
+    private List<PageElement> renderers =new ArrayList<>();
+
+    private Map<String,String> parameters =new LinkedHashMap<>();
+
+    private String url;
+
+    /** The precalculated hashCode of this object, or 0 if this is not frozen */
+    private int hashCode=0;
+
+    public Source(String name) {
+        this(name,false);
+    }
+
+    /** Creates a source and optionally immediately freezes it */
+    private Source(String name,boolean freeze) {
+        setName(name);
+        if (freeze)
+            freeze();
+    }
+
+    /** Returns the name of this source (never null) */
+    public String getName() { return name; }
+
+    public final void setName(String name) {
+        ensureNotFrozen();
+        Validator.ensureNotNull("Source name",name);
+        this.name=name;
+    }
+
+    /** Returns the url of this source or null if none */
+    public String getUrl() { return url; }
+
+    /**
+     * Sets the url of this source. If a source has an url (i.e this returns non-null), the content of
+     * the url is <i>not</i> fetched - fetching is left to the frontend by exposing this url in the result.
+     */
+    public void setUrl(String url) {
+        ensureNotFrozen();
+        this.url=url;
+    }
+
+    /**
+     * Returns the renderers or choices of renderers to apply on individual items of this source
+     * <p>
+     * If this contains multiple renderers/choices, they are to be used on different types of hits returned by this source.
+     */
+    public List<PageElement> renderers() { return renderers; }
+
+    /**
+     * Returns the parameters of this source as a live reference (never null).
+     * The parameters will be passed to the provider getting source data.
+     */
+    public Map<String,String> parameters() { return parameters; }
+
+    public @Override void freeze() {
+        if (isFrozen()) return;
+        for (PageElement element : renderers) {
+            if (element instanceof Renderer) {
+                assignRendererForIfNotSet((Renderer)element);
+            }
+            else if (element instanceof Choice) {
+                for (List<PageElement> renderersAlternative : ((Choice)element).alternatives()) {
+                    for (PageElement rendererElement : renderersAlternative) {
+                        Renderer renderer=(Renderer)rendererElement;
+                        if (renderer.getRendererFor()==null)
+                            renderer.setRendererFor(name);
+                    }
+                }
+            }
+            element.freeze();
+        }
+        parameters = Collections.unmodifiableMap(parameters);
+        hashCode=hashCode();
+        super.freeze();
+    }
+
+    private void assignRendererForIfNotSet(Renderer renderer) {
+        if (renderer.getRendererFor()==null)
+            renderer.setRendererFor(name);
+    }
+
+    /** Accepts a visitor to this structure */
+    public @Override void accept(PageTemplateVisitor visitor) {
+        visitor.visit(this);
+        for (PageElement renderer : renderers)
+            renderer.accept(visitor);
+    }
+
+    public @Override int hashCode() {
+        if (isFrozen()) return hashCode;
+        int hashCode=name.hashCode();
+        int i=0;
+        for (Map.Entry<String,String> parameter : parameters.entrySet())
+            hashCode+=i*17*parameter.getKey().hashCode()+i*31*parameter.getValue().hashCode();
+        return hashCode;
+    }
+
+    public @Override boolean equals(Object other) {
+        if (other==this) return true;
+        if (! (other instanceof Source)) return false;
+        Source otherSource=(Source)other;
+        if (! this.name.equals(otherSource.name)) return false;
+        if (this.parameters.size() != otherSource.parameters.size()) return false;
+        for (Map.Entry<String,String> thisParameter : this.parameters.entrySet())
+            if ( ! thisParameter.getValue().equals(otherSource.parameters.get(thisParameter.getKey())))
+                return false;
+        return true;
+    }
+
+    public @Override String toString() {
+        return "source '" + name + "'";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/model/package-info.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/package-info.java
new file mode 100644
index 00000000000..22a004d7555
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/model/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.pagetemplates.model;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/package-info.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/package-info.java
new file mode 100644
index 00000000000..0368351a6dc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.pagetemplates;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/SectionHitGroup.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/SectionHitGroup.java
new file mode 100644
index 00000000000..00f6c6350fc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/SectionHitGroup.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.pagetemplates.result;
+
+import com.yahoo.search.pagetemplates.model.Renderer;
+import com.yahoo.search.pagetemplates.model.Source;
+import com.yahoo.search.result.HitGroup;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A hit group corresponding to a section - contains some additional information
+ * in proper getters and setters which is used during rendering.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SectionHitGroup extends HitGroup {
+
+    private static final long serialVersionUID = -9048845836777953538L;
+    private List<Source> sources=new ArrayList<>(0);
+    private List<Renderer> renderers=new ArrayList<>(0);
+    private final String displayId;
+
+    private boolean leaf=false;
+
+    public SectionHitGroup(String id) {
+        super(id);
+        if (id.startsWith("section:section_"))
+            displayId=null; // Don't display section ids when not named explicitly
+        else
+            displayId=id;
+        types().add("section");
+    }
+
+    @Override
+    public String getDisplayId() { return displayId; }
+
+    /**
+     * Returns the live, modifiable list of sources which are not fetched by the framework but should
+     * instead be included in the result
+     */
+    public List<Source> sources() { return sources; }
+
+    /** Returns the live, modifiable list of renderers in this section */
+    public List<Renderer> renderers() { return renderers; }
+
+    /** Returns whether this is a leaf section containing no subsections */
+    public boolean isLeaf() { return leaf; }
+
+    public void setLeaf(boolean leaf) { this.leaf=leaf; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/package-info.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/package-info.java
new file mode 100644
index 00000000000..7d006aad551
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.pagetemplates.result;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java
new file mode 100644
index 00000000000..588580dda4d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/Model.java
@@ -0,0 +1,521 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.LocaleFactory;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.parser.Parsable;
+import com.yahoo.search.query.parser.Parser;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.*;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * The parameters defining the recall of a query.
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Model implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+    private static final CompoundName argumentTypeName;
+
+    public static final String MODEL = "model";
+    public static final String PROGRAM = "program";
+    public static final String QUERY_STRING = "queryString";
+    public static final String TYPE = "type";
+    public static final String FILTER = "filter";
+    public static final String DEFAULT_INDEX = "defaultIndex";
+    public static final String LANGUAGE = "language";
+    public static final String ENCODING = "encoding";
+    public static final String SOURCES = "sources";
+    public static final String SEARCH_PATH = "searchPath";
+    public static final String RESTRICT = "restrict";
+
+    static {
+        argumentType =new QueryProfileType(MODEL);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        //argumentType.addField(new FieldDescription(PROGRAM, "string", "yql")); // TODO: Custom type
+        argumentType.addField(new FieldDescription(QUERY_STRING, "string", "query"));
+        argumentType.addField(new FieldDescription(TYPE, "string", "type"));
+        argumentType.addField(new FieldDescription(FILTER, "string","filter"));
+        argumentType.addField(new FieldDescription(DEFAULT_INDEX, "string", "default-index def-idx defidx"));
+        argumentType.addField(new FieldDescription(LANGUAGE, "string", "language lang"));
+        argumentType.addField(new FieldDescription(ENCODING, "string", "encoding"));
+        argumentType.addField(new FieldDescription(SOURCES, "string", "sources search"));
+        argumentType.addField(new FieldDescription(SEARCH_PATH, "string", "searchpath"));
+        argumentType.addField(new FieldDescription(RESTRICT, "string", "restrict"));
+        argumentType.freeze();
+        argumentTypeName=new CompoundName(argumentType.getId().getName());
+    }
+
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    /** The name of the query property used for generating hit count estimate queries. */
+    public static final CompoundName ESTIMATE = new CompoundName("hitcountestimate");
+
+    private String encoding = null;
+    private String queryString = "";
+    private String filter = null;
+    private Language language = null;
+    private Locale locale = null;
+    private QueryTree queryTree = null; // The actual query. This is lazily created from the program
+    private String defaultIndex = null;
+    private Query.Type type = Query.Type.ALL;
+    private Query parent;
+    private Set<String> sources=new LinkedHashSet<>();
+    private Set<String> restrict=new LinkedHashSet<>();
+    private String searchPath;
+    private String documentDbName = null;
+    private Execution execution=new Execution(new Execution.Context(null, null, null, null, null));
+
+    public Model(Query query) {
+        setParent(query);
+    }
+
+    /**
+     * Creates trace a message of language detection results into this Model
+     * instance's parent query. Do note this will give bogus results if the
+     * Execution instance is not set correctly. This is done automatically
+     * inside {@link Execution#search(Query)}. If tracing the same place as
+     * creating the query instance, {@link #setExecution(Execution)} has to be
+     * invoked first with the same Execution instance the query is intended to
+     * be run by.
+     */
+    public void traceLanguage() {
+        if (getParent().getTraceLevel()<2) return;
+        if (language != null) {
+            getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2);
+        }
+        else {
+            Language l = getParsingLanguage();
+            // Don't include the query, it will trigger query parsing
+            getParent().trace("Detected language: " + l, false, 2);
+            getParent().trace("Language " + l + " determined by " +
+                              (Language.fromEncoding(encoding) != Language.UNKNOWN ? "query encoding" :
+                               "the characters in the terms") + ".", false, 2);
+        }
+    }
+
+    /**
+     * Gets the language to use for parsing. If this is explicitly set, that language is returned, otherwise
+     * it is guessed from the query string. If this does not yield an actual language, English is
+     * returned as the default.
+     *
+     * @return the language determined, never null
+     */
+    public Language getParsingLanguage() {
+        Language language = getLanguage();
+        if (language != null) {
+            return language;
+        }
+        language = Language.fromEncoding(encoding);
+        if (language != Language.UNKNOWN) {
+            return language;
+        }
+        Linguistics linguistics = execution.context().getLinguistics();
+        if (linguistics != null) {
+            language = linguistics.getDetector().detect(queryString, null).getLanguage();
+        }
+        if (language != Language.UNKNOWN) {
+            return language;
+        }
+        return Language.ENGLISH;
+    }
+
+    /** Returns the explicitly set parsing language of this query model, or null if none */
+    public Language getLanguage() { return language; }
+
+    /** Explicitly sets the language to be used during parsing */
+    public void setLanguage(Language language) { this.language = language; }
+
+    /**
+     * <p>Explicitly sets the language to be used during parsing. The argument is first normalized by replacing
+     * underscores with hyphens (to support locale strings being used as RFC 5646 language tags), and then forwarded to
+     * {@link #setLocale(String)} so that the Locale information of the tag is preserved.</p>
+     *
+     * @param language The language string to parse.
+     * @see #getLanguage()
+     * @see #setLocale(String)
+     */
+    public void setLanguage(String language) {
+        setLocale(language.replace("_", "-"));
+    }
+
+    /**
+     * <p>Returns the explicitly set parsing locale of this query model, or null if none.</p>
+     *
+     * @return The locale of this.
+     * @see #setLocale(Locale)
+     */
+    public Locale getLocale() {
+        return locale;
+    }
+
+    /**
+     * <p>Explicitly sets the locale to be used during parsing. This method also calls {@link #setLanguage(Language)}
+     * with the corresponding {@link Language} instance.</p>
+     *
+     * @param locale The locale to set.
+     * @see #getLocale()
+     * @see #setLanguage(Language)
+     */
+    public void setLocale(Locale locale) {
+        this.locale = locale;
+        setLanguage(Language.fromLocale(locale));
+    }
+
+    /**
+     * <p>Explicitly sets the locale to be used during parsing. This creates a Locale instance from the given language
+     * tag, and passes that to {@link #setLocale(Locale)}.</p>
+     *
+     * @param languageTag The language tag to parse.
+     * @see #setLocale(Locale)
+     */
+    public void setLocale(String languageTag) {
+        setLocale(LocaleFactory.fromLanguageTag(languageTag));
+    }
+
+    /** Returns the encoding used in the query as a lowercase string */
+    public String getEncoding() { return encoding; }
+
+    /** Sets the encoding which was used in the received query string */
+    public void setEncoding(String encoding) {
+        this.encoding = toLowerCase(encoding);
+    }
+
+    /** Set the path for which backend nodes to forward the search too. */
+    public void setSearchPath(String searchPath) { this.searchPath = searchPath; }
+
+    public String getSearchPath() { return searchPath; }
+
+    /**
+     * Set the query from a string. This will not be parsed into a query tree until that tree is attempted accessed.
+     * Note that setting this will clear the current query tree. Usually, this should <i>not</i> be modified -
+     * changes to the query should be implemented as modifications on the query tree structure.
+     * <p>
+     * Passing null causes this to be set to an empty string.
+     */
+    public void setQueryString(String queryString) {
+        if (queryString==null) queryString="";
+        this.queryString = queryString;
+        queryTree=null; // Cause parsing of the new query string next time the tree is accessed
+    }
+
+    /**
+     * Returns the query string which caused the original query tree of this model to come about.
+     * Note that changes to the query tree are <b>not</b> reflected in this query string.
+     *
+     * @return the original (or reassigned) query string - never null
+     */
+    public String getQueryString() { return queryString; }
+
+    /**
+     * Returns the query as an object structure.
+     * This causes parsing of the query string if it has changed since this was last called
+     * (i.e query parsing is lazy)
+     */
+    public QueryTree getQueryTree() {
+        if (queryTree == null) {
+            Parser parser = ParserFactory.newInstance(type, ParserEnvironment.fromExecutionContext(execution.context()));
+            queryTree = parser.parse(Parsable.fromQueryModel(this));
+            if (parent.getTraceLevel() >= 2) {
+                parent.trace("Query parsed to: " + parent.yqlRepresentation(), 2);
+            }
+        }
+        return queryTree;
+    }
+
+    /**
+     * Returns the filter string set for this query.
+     * The filter is included in the query tree at the time the query tree is parsed
+     */
+    public String getFilter() { return filter; }
+
+    /**
+     * Sets the filter string set for this query.
+     * The filter is included in the query tree at the time the query tree is parsed.
+     * Setting this does <i>not</i> cause the query to be reparsed.
+     */
+    public void setFilter(String filter) {  this.filter = filter; }
+
+    /**
+     * Returns the default index for this query.
+     * The default index is taken into account at the time the query tree is parsed.
+     */
+    public String getDefaultIndex() { return defaultIndex; }
+
+    /**
+     * Sets the default index for this query.
+     * The default index is taken into account at the time the query tree is parsed.
+     * Setting this does <i>not</i> cause the query to be reparsed.
+     */
+    public void setDefaultIndex(String defaultIndex) { this.defaultIndex = defaultIndex; }
+
+    /**
+     * Sets the query type of for this query.
+     * The type is taken into account at the time the query tree is parsed.
+     */
+    public Query.Type getType() { return type; }
+
+    /**
+     * Sets the query type of for this query.
+     * The type is taken into account at the time the query tree is parsed.
+     * Setting this does <i>not</i> cause the query to be reparsed.
+     */
+    public void setType(Query.Type type) { this.type = type; }
+
+    /**
+     * Sets the query type of for this query.
+     * The type is taken into account at the time the query tree is parsed.
+     * Setting this does <i>not</i> cause the query to be reparsed.
+     */
+    public void setType(String typeString) { this.type = Query.Type.getType(typeString); }
+
+    public boolean equals(Object o) {
+        if ( ! (o instanceof Model)) return false;
+
+        Model other = (Model) o;
+        if ( ! (
+                QueryHelper.equals(other.encoding, this.encoding) &&
+                QueryHelper.equals(other.language, this.language) &&
+                QueryHelper.equals(other.searchPath, this.searchPath) &&
+                QueryHelper.equals(other.sources, this.sources) &&
+                QueryHelper.equals(other.restrict, this.restrict) &&
+                QueryHelper.equals(other.defaultIndex, this.defaultIndex) &&
+                QueryHelper.equals(other.type, this.type) ))
+            return false;
+
+        if (other.queryTree == null && this.queryTree == null) // don't cause query parsing
+            return QueryHelper.equals(other.queryString, this.queryString) &&
+                   QueryHelper.equals(other.filter, this.filter);
+        else // make sure we compare a parsed variant of both
+            return QueryHelper.equals(other.getQueryTree(), this.getQueryTree());
+    }
+
+    @Override
+    public int hashCode() {
+        return getClass().hashCode() +
+               QueryHelper.combineHash(encoding,filter,language,getQueryTree(),sources,restrict,defaultIndex,type,searchPath);
+    }
+
+
+    public Object clone() {
+        try {
+            Model clone = (Model) super.clone();
+            if (queryTree != null)
+                clone.queryTree = this.queryTree.clone();
+            if (sources !=null)
+                clone.sources = new LinkedHashSet<>(this.sources);
+            if (restrict !=null)
+                clone.restrict = new LinkedHashSet<>(this.restrict);
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Someone inserted a noncloneable superclass",e);
+        }
+    }
+
+    public Model cloneFor(Query q)  {
+        Model model = (Model) this.clone();
+        model.setParent(q);
+        return model;
+    }
+
+    /** returns the query owning this, never null */
+    public Query getParent() { return parent; }
+
+    /** Assigns the query owning this */
+    public void setParent(Query parent) {
+        if (parent==null) throw new NullPointerException("A query models owner cannot be null");
+        this.parent = parent;
+    }
+
+    /** Sets the set of sources this query will search from a comma-separated string of source names */
+    public void setSources(String sourceString) {
+        setFromString(sourceString,sources);
+    }
+
+    /**
+     * Returns the set of sources this query will search.
+     * This set can be modified to change the set of sources. If all sources are to be searched, this returns
+     * an empty set
+     *
+     * @return the set of sources to search, never null
+     */
+    public Set<String> getSources() { return sources; }
+
+    /**
+     * Sets the set of types (document type or search definition names) this query will search from a
+     * comma-separated string of type names. This is useful to narrow a search to just a subset of the types available
+     * from a sources
+     */
+    public void setRestrict(String restrictString) {
+        setFromString(restrictString,restrict);
+    }
+
+    /**
+     * Returns the set of types this query will search.
+     * This set can be modified to change the set of types. If all types are to be searched, this returns
+     * an empty set.
+     *
+     * @return the set of types to search, never null
+     */
+    public Set<String> getRestrict() { return restrict; }
+
+    /** Sets the execution working on this. For internal use. */
+    public void setExecution(Execution execution) {
+        if (execution==this.execution) return;
+
+        // If not already coupled, bind the trace of the new execution into the existing execution trace
+        if (execution.trace().traceNode().isRoot()
+            && execution.trace().traceNode() != this.execution.trace().traceNode().root()) {
+            this.execution.trace().traceNode().add(execution.trace().traceNode());
+        }
+
+        this.execution = execution;
+    }
+
+    /** Sets the document database this will search - a document type */
+    public void setDocumentDb(String documentDbName) {
+        this.documentDbName = documentDbName;
+    }
+
+    /** Returns the name of the document db this should search, or null if not set. */
+    public String getDocumentDb() { return documentDbName; }
+
+    /** Returns the Execution working on this, or a null execution if none. For internal use. */
+    public Execution getExecution() { return execution; }
+
+    private void setFromString(String string,Set<String> set) {
+        set.clear();
+        for (String item : string.split(","))
+            set.add(item.trim());
+    }
+
+    public static Model getFrom(Query q) {
+        return (Model)q.properties().get(argumentTypeName);
+    }
+
+    public @Override String toString() {
+        return "query representation [queryTree: " + queryTree + ", filter: " + filter + "]";
+    }
+
+    /** Prepares this for binary serialization. For internal use. */
+    public void prepare(Ranking ranking) {
+        prepareRankFeaturesFromModel(ranking);
+    }
+
+    private void prepareRankFeaturesFromModel(Ranking ranking) {
+        Item root = getQueryTree().getRoot();
+        if (root != null) {
+            List<Item> tagged = setUniqueIDs(root);
+            addLabels(tagged, ranking);
+            addConnectivityRankProperties(tagged, ranking);
+            addSignificances(tagged, ranking);
+        }
+    }
+
+    private List<Item> setUniqueIDs(Item root) {
+        List<Item> items = new ArrayList<>();
+        collectTaggableItems(root, items);
+        int id = 1;
+        for (Item i : items) {
+            TaggableItem t = (TaggableItem) i;
+            t.setUniqueID(id++);
+        }
+        return items;
+    }
+
+    private void addLabels(List<Item> candidates, Ranking ranking) {
+        for (Item candidate : candidates) {
+            String label = candidate.getLabel();
+            if (label != null) {
+                String name = "vespa.label." + label + ".id";
+                TaggableItem t = (TaggableItem) candidate;
+                ranking.getProperties().put(name, String.valueOf(t.getUniqueID()));
+            }
+        }
+    }
+
+    private void addConnectivityRankProperties(List<Item> connectedItems, Ranking ranking) {
+        for (Item link : connectedItems) {
+            TaggableItem t = (TaggableItem) link;
+            Item connectedTo = t.getConnectedItem();
+            if (connectedTo != null && strictContains(connectedTo, connectedItems)) {
+                TaggableItem t2 = (TaggableItem) connectedTo;
+                String name = "vespa.term." + t.getUniqueID() + ".connexity";
+                ranking.getProperties().put(name, String.valueOf(t2.getUniqueID()));
+                ranking.getProperties().put(name, String.valueOf(t.getConnectivity()));
+            }
+        }
+    }
+
+    private void addSignificances(List<Item> candidates, Ranking ranking) {
+        for (Item  candidate : candidates) {
+            TaggableItem t = (TaggableItem) candidate;
+            if ( ! t.hasExplicitSignificance()) continue;
+            String name = "vespa.term." + t.getUniqueID() + ".significance";
+            ranking.getProperties().put(name, String.valueOf(t.getSignificance()));
+        }
+    }
+
+    private void collectTaggableItems(Item root, List<Item> terms) {
+        if (root == null) return;
+
+        if (root instanceof TaggableItem) {
+            // This is tested before descending, as phrases are viewed
+            // as leaf nodes in the ranking code in the backend
+            terms.add(root);
+        } else if (root instanceof CompositeItem) {
+            CompositeItem c = (CompositeItem) root;
+            for (Iterator<Item> i = c.getItemIterator(); i.hasNext();) {
+                collectTaggableItems(i.next(), terms);
+            }
+        } else {} // nop
+    }
+
+    private boolean strictContains(Object needle, Collection<?> haystack) {
+        for (Object pin : haystack)
+            if (pin == needle) return true;
+        return false;
+    }
+
+
+    /**
+     * Set the YTrace header value to use when transmitting this model to a
+     * search backend (of some kind).
+     *
+     * @param next string representation of header value
+     * @deprecated Not use, ytrace is done
+     */
+    @Deprecated
+    public void setYTraceHeaderToNext(String next) { }
+
+    /**
+     * Get the YTrace header value to use when transmitting this model to a
+     * search backend (of some kind). Returns null if no ytrace data is not
+     * turned on.
+     * @deprecated Not use, ytrace is done
+     */
+    @Deprecated
+    public String getYTraceHeaderToNext() {
+        return null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java b/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java
new file mode 100644
index 00000000000..a27e1bfde55
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ParameterParser.java
@@ -0,0 +1,88 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import static com.yahoo.container.util.Util.quote;
+
+/**
+ * Wrapper class to avoid code duplication of common parsing requirements.
+ *
+ * @author <a href="steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ParameterParser {
+
+    /**
+     * Tries to return the given object as a Long. If it is a Number, treat it
+     * as a number of seconds, i.e. get a Long representation and multiply by
+     * 1000. If it has a String representation, try to parse this as a floating
+     * point number, followed by by an optional unit (seconds and an SI prefix,
+     * a couple of valid examples are "s" and "ms". Only a very small subset of
+     * SI prefixes are supported). If no unit is given, seconds are assumed.
+     *
+     * @param value
+     *                some representation of a number of seconds
+     * @param defaultValue
+     *                returned if value is null
+     * @return value as a number of milliseconds
+     * @throws NumberFormatException
+     *                 if value is not a Number instance and its String
+     *                 representation cannot be parsed as a number followed
+     *                 optionally by time unit
+     */
+    public static Long asMilliSeconds(Object value, Long defaultValue) {
+        if (value == null) {
+            return defaultValue;
+        }
+        if (value instanceof Number) {
+            Number n = (Number) value;
+            return Long.valueOf(n.longValue() * 1000L);
+        }
+        return parseTime(value.toString());
+    }
+
+    private static Long parseTime(String time) throws NumberFormatException {
+
+        time = time.trim();
+        try {
+            int unitOffset = findUnitOffset(time);
+            double measure = Double.valueOf(time.substring(0, unitOffset));
+            double multiplier = parseUnit(time.substring(unitOffset));
+            return Long.valueOf((long) (measure * multiplier));
+        } catch (RuntimeException e) {
+            throw new IllegalArgumentException("Error parsing " + quote(time), e);
+        }
+    }
+
+    private static int findUnitOffset(String time) {
+        int unitOffset = 0;
+        while (unitOffset < time.length()) {
+            char c = time.charAt(unitOffset);
+            if (c == '.' || (c >= '0' && c <= '9')) {
+                unitOffset += 1;
+            } else {
+                break;
+            }
+        }
+        if (unitOffset == 0) {
+            throw new NumberFormatException("Invalid number " + quote(time));
+        }
+        return unitOffset;
+    }
+
+    private static double parseUnit(String unit) {
+        unit = unit.trim();
+        final double multiplier;
+        if ("ks".equals(unit)) {
+            multiplier = 1e6d;
+        } else if ("s".equals(unit)) {
+            multiplier = 1000.0d;
+        } else if ("ms".equals(unit)) {
+            multiplier = 1.0d;
+        } else if ("\u00B5s".equals(unit)) {
+            // microseconds
+            multiplier = 1e-3d;
+        } else {
+            multiplier = 1000.0d;
+        }
+        return multiplier;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java
new file mode 100644
index 00000000000..466ddf88299
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java
@@ -0,0 +1,211 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.google.common.base.Splitter;
+import com.yahoo.collections.LazySet;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.prelude.query.*;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.rendering.RendererRegistry;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+
+/**
+ * Parameters deciding how the result of a query should be presented
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ */
+public class Presentation implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static QueryProfileType argumentType;
+
+    public static final String PRESENTATION = "presentation";
+    public static final String BOLDING = "bolding";
+    public static final String TIMING = "timing";
+    public static final String SUMMARY = "summary";
+    public static final String REPORT_COVERAGE = "reportCoverage";
+    public static final String SUMMARY_FIELDS = "summaryFields";
+
+    /** The (short) name of the parameter holding the name of the return format to use */
+    public static final String FORMAT = "format";
+
+    static {
+        argumentType=new QueryProfileType(PRESENTATION);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(BOLDING, "boolean", "bolding"));
+        argumentType.addField(new FieldDescription(TIMING, "boolean", "timing"));
+        argumentType.addField(new FieldDescription(SUMMARY, "string", "summary"));
+        argumentType.addField(new FieldDescription(REPORT_COVERAGE, "string", "reportcoverage"));
+        argumentType.addField(new FieldDescription(FORMAT, "string", "format template"));
+        argumentType.addField(new FieldDescription(SUMMARY_FIELDS, "string", "summaryFields"));
+        argumentType.freeze();
+    }
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    /** How the result should be highlighted */
+    private Highlight highlight= null;
+
+    /** The terms to highlight in the result (only used by BoldingSearcher, may be removed later). */
+    private List<IndexedItem> boldingData = null;
+
+    /** Whether or not to do highlighting */
+    private boolean bolding = true;
+
+    /** The summary class to be shown */
+    private String summary = null;
+
+    /** Whether coverage information (how much of the indices was searched should be included in the result */
+    private boolean reportCoverage=false;
+
+    /** The name of the renderer to use for rendering the hits. */
+    private ComponentSpecification format = RendererRegistry.defaultRendererId.toSpecification();
+
+    /** Whether optional timing data should be rendered */
+    private boolean timing = false;
+
+    /** Set of explicitly requested summary fields, instead of summary classes */
+    @NonNull
+    private Set<String> summaryFields = LazySet.newHashSet();
+
+    private static final Splitter COMMA_SPLITTER = Splitter.on(',').omitEmptyStrings().trimResults();
+
+    public Presentation(Query parent) { }
+
+    /** Returns how terms in this result should be highlighted, or null if not set */
+    public Highlight getHighlight() { return highlight; }
+
+    /** Sets how terms in this result should be highlighted. Set to null to turn highlighting off */
+    public void setHighlight(Highlight highlight) { this.highlight = highlight; }
+
+    /** Returns the name of the summary class to be used to present hits from this query, or null if not set */
+    public String getSummary() { return summary; }
+
+    /** Sets the name of the summary class to be used to present hits from this query */
+    public void setSummary(String summary) { this.summary = summary; }
+
+    /** Returns whether matching query terms should be bolded in the result. Default is true. */
+    public boolean getBolding() { return bolding; }
+
+    /** Sets whether matching query terms should be bolded in the result */
+    public void setBolding(boolean bolding) { this.bolding = bolding; }
+
+    /** Returns whether coverage information should be returned in the result, if available. Default is false */
+    public boolean getReportCoverage() { return reportCoverage; }
+
+    /** Sets whether coverage information should be returned in the result, if available */
+    public void setReportCoverage(boolean reportCoverage) { this.reportCoverage=reportCoverage; }
+
+    /** Get the name of the format desired for result rendering. */
+    @NonNull
+    public ComponentSpecification getRenderer() { return format; }
+
+    /** Set the desired format for result rendering. If null, use the default renderer. */
+    public void setRenderer(@Nullable ComponentSpecification format) {
+        this.format = (format != null) ? format : RendererRegistry.defaultRendererId.toSpecification();
+    }
+
+    /**
+     * Get the name of the format desired for result rendering.
+     */
+    @NonNull
+    public String getFormat() { return format.getName(); }
+
+    /**
+     * Set the desired format for result rendering. If null, use the default renderer.
+     */
+    public void setFormat(@Nullable String format) {
+        setRenderer(ComponentSpecification.fromString(format));
+    }
+
+    @Override
+    public Object clone()  {
+        try {
+            Presentation clone = (Presentation)super.clone();
+            if (boldingData != null)
+                clone.boldingData = new ArrayList<>(boldingData);
+
+            if (highlight != null)
+                clone.highlight = highlight.clone();
+
+            if (summaryFields != null) {
+                clone.summaryFields = LazySet.newHashSet();
+                clone.summaryFields.addAll(this.summaryFields);
+            }
+
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Someone inserted a noncloneable superclass",e);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || !(o instanceof Presentation)) return false;
+        Presentation p = (Presentation) o;
+        return QueryHelper.equals(bolding,p.bolding) && QueryHelper.equals(summary,p.summary);
+    }
+
+    @Override
+    public int hashCode() {
+        return QueryHelper.combineHash(bolding, summary);
+    }
+
+    /**
+     * @return whether to add optional timing data to the rendered result
+     */
+    public boolean getTiming() {
+        return timing;
+    }
+
+    public void setTiming(boolean timing) {
+        this.timing = timing;
+    }
+
+    /**
+     * Return the set of explicitly requested fields. Returns an empty set if no
+     * fields are specified outside of summary classes. The returned set is
+     * mutable and fields may be added or removed before passing on the query.
+     *
+     * @return the set of names of requested fields, never null
+     */
+    @NonNull
+    public Set<String> getSummaryFields() {
+        return summaryFields;
+    }
+
+    /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */
+    public void prepare() {
+        if (highlight != null)
+            highlight.prepare();
+    }
+
+    /**
+     * Parse the given string as a comma delimited set of field names and
+     * overwrite the set of summary fields. Whitespace will be trimmed. If you
+     * want to add or remove fields programmatically, use
+     * {@link #getSummaryFields()} and modify the returned set.
+     *
+     * @param asString
+     *            the summary fields requested, e.g. "price,author,title"
+     */
+    public void setSummaryFields(String asString) {
+        summaryFields.clear();
+        for (String field : COMMA_SPLITTER.split(asString)) {
+            summaryFields.add(field);
+        }
+
+    }
+
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/query/Properties.java b/container-search/src/main/java/com/yahoo/search/query/Properties.java
new file mode 100644
index 00000000000..df3d120c337
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/Properties.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.yahoo.search.Query;
+
+/**
+ * Object properties keyed by name which can be looked up using default values and
+ * with conversion to various primitive wrapper types.
+ * <p>
+ * Multiple property implementations can be chained to provide unified access to properties
+ * backed by multiple sources as a Chain of Responsibility.
+ * <p>
+ * For better performance, prefer CompoundName argument constants over Strings.
+ * <p>
+ * Properties can be cloned. Cloning a properties instance returns a new instance
+ * which chains new instances of all chained instances. The content within each instance
+ * is cloned to the extent determined appropriate by that implementation.
+ * <p>
+ * This base class simply passes all access on to the next in chain.
+ *
+ * @author bratseth
+ */
+public abstract class Properties extends com.yahoo.processing.request.Properties {
+
+    @Override
+    public Properties chained() { return (Properties)super.chained(); }
+
+    @Override
+    public Properties clone() {
+        return (Properties)super.clone();
+    }
+
+    /** The query owning this property object.
+     *  Only guaranteed to work if this instance is accessible as query.properties()
+     */
+    public Query getParentQuery() {
+        if (chained() == null) {
+            throw new RuntimeException("getParentQuery should only be called on a properties instance accessible as query.properties()");
+        } else {
+            return chained().getParentQuery();
+        }
+    }
+
+    /**
+     * Invoked during deep cloning of the parent query.
+     */
+    public void setParentQuery(Query query) {
+        if (chained() != null)
+            chained().setParentQuery(query);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryHelper.java b/container-search/src/main/java/com/yahoo/search/query/QueryHelper.java
new file mode 100644
index 00000000000..d4b6f257c11
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/QueryHelper.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+/**
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ */
+class QueryHelper {
+
+    /** Compares two objects which may be null */
+    public static boolean equals(Object a,Object b) {
+        if (a == null) return b == null;
+        return a.equals(b);
+    }
+
+    /**
+     * Helper method that finds the hashcode for a group of objects.
+     * Inspired by java.util.List
+     */
+    public static int combineHash(Object... objs) {
+       int hash = 1;
+       for (Object o:objs) {
+           hash = 31*hash + (o == null ? 0 : o.hashCode());
+       }
+       return hash;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java
new file mode 100644
index 00000000000..3a501853388
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.yahoo.prelude.query.*;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * The root node of a query tree. This is always present above the actual semantic root to ease query manipulation,
+ * especially replacing the actual semantic root, but does not have any search semantics on its own.
+ *
+ * <p>To ease recursive manipulation of the query tree, this is a composite having one child, which is the actual root.
+ * <ul>
+ * <li>Setting the root item (at position 0, either directly or though the iterator of this, works as expected.
+ * Setting at any other position is disallowed.
+ * <li>Removing the root is allowed and causes this to be a null query.
+ * <li>Adding an item is only allowed if this is currently a null query (having no root)
+ * </ul>
+ *
+ * <p>This is also the home of accessor methods which eases querying into and manipulation of the query tree.</p>
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ */
+public class QueryTree extends CompositeItem {
+
+    public QueryTree(Item root) {
+        setRoot(root);
+    }
+
+    public void setIndexName(String index) {
+        if (getRoot() != null)
+            getRoot().setIndexName(index);
+    }
+
+    public ItemType getItemType() {
+        throw new RuntimeException("Packet type access attempted. " +
+                "A query tree has no packet code. This is probably a misbehaving searcher.");
+    }
+
+    public String getName() { return "ROOT"; }
+
+    public int encode(ByteBuffer buffer) {
+        if (getRoot() == null) return 0;
+        return getRoot().encode(buffer);
+    }
+
+    //Lets not pollute toString() by adding "ROOT"
+    protected void appendHeadingString(StringBuilder sb) {
+    }
+
+    /** Returns the query root. This is null if this is a null query. */
+    public Item getRoot() {
+        if (getItemCount()==0) return null;
+        return getItem(0);
+    }
+
+    public final void setRoot(Item root) {
+        if (root==this) throw new IllegalArgumentException("Cannot make a root point at itself");
+        if (root == null) throw new IllegalArgumentException("Root must not be null, use NullItem instead.");
+        if (root instanceof QueryTree) throw new IllegalArgumentException("Do not use a new QueryTree instance as a root.");
+        if (this.getItemCount()==0) // initializing
+            super.addItem(root);
+        else
+            setItem(0,root); // replacing
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if( !(o instanceof QueryTree)) return false;
+        return super.equals(o);
+    }
+
+    /** Returns a deep copy of this */
+    @Override
+    public QueryTree clone() {
+        QueryTree clone = (QueryTree) super.clone();
+        fixClonedConnectivityReferences(clone);
+        return clone;
+    }
+
+    private void fixClonedConnectivityReferences(QueryTree clone) {
+        // TODO!
+    }
+
+    @Override
+    public void addItem(Item item) {
+        if (getItemCount()==0)
+            super.addItem(item);
+        else
+            throw new RuntimeException("Programming error: Cannot add multiple roots");
+    }
+
+    @Override
+    public void addItem(int index, Item item) {
+        if (getItemCount()==0 && index==0)
+            super.addItem(index,item);
+        else
+            throw new RuntimeException("Programming error: Cannot add multiple roots, have '" + getRoot() + "'");
+    }
+
+    /** Returns true if this represents the null query */
+    public boolean isEmpty() {
+        return getRoot() instanceof NullItem;
+    }
+
+    // -------------- Facade
+
+    /** Modifies this query to become the current query AND the given item */
+    // TODO: Make sure this is complete, unit test and make it public
+    private void and(Item item) {
+        if (isEmpty()) {
+            setRoot(item);
+        }
+        else if (getRoot() instanceof NotItem && item instanceof NotItem) {
+            throw new IllegalArgumentException("Can't AND two NOTs"); // TODO: Complete
+        }
+        else if (getRoot() instanceof NotItem){
+            NotItem notItem = (NotItem)getRoot();
+            notItem.addPositiveItem(item);
+        }
+        else if (item instanceof NotItem){
+            NotItem notItem = (NotItem)item;
+            notItem.addPositiveItem(getRoot());
+            setRoot(notItem);
+        }
+        else {
+            AndItem andItem = new AndItem();
+            andItem.addItem(getRoot());
+            andItem.addItem(item);
+            setRoot(andItem);
+        }
+    }
+
+    /** Returns a flattened list of all positive query terms under the given item */
+    public static List<IndexedItem> getPositiveTerms(Item item) {
+        List<IndexedItem> items = new ArrayList<>();
+        getPositiveTerms(item,items);
+        return items;
+    }
+
+    private static void getPositiveTerms(Item item, List<IndexedItem> terms) {
+        if (item instanceof NotItem) {
+            getPositiveTerms(((NotItem) item).getPositiveItem(), terms);
+        } else if (item instanceof PhraseItem) {
+            PhraseItem pItem = (PhraseItem)item;
+            terms.add(pItem);
+        } else if (item instanceof CompositeItem) {
+            for (Iterator<Item> i = ((CompositeItem) item).getItemIterator(); i.hasNext();) {
+                getPositiveTerms(i.next(), terms);
+            }
+        } else if (item instanceof TermItem) {
+            terms.add((TermItem)item);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java
new file mode 100644
index 00000000000..e543589f74d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java
@@ -0,0 +1,246 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.prelude.Freshness;
+import com.yahoo.prelude.Location;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.ranking.MatchPhase;
+import com.yahoo.search.query.ranking.RankFeatures;
+import com.yahoo.search.query.ranking.RankProperties;
+import com.yahoo.search.result.ErrorMessage;
+
+/**
+ * The ranking (hit ordering) settings of a query
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author bratseth
+ */
+public class Ranking implements Cloneable {
+
+    /** An alias for listing features */
+    public static final com.yahoo.processing.request.CompoundName RANKFEATURES =
+            new com.yahoo.processing.request.CompoundName("rankfeatures");
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+    private static final CompoundName argumentTypeName;
+
+    public static final String RANKING = "ranking";
+    public static final String LOCATION = "location";
+    public static final String PROFILE = "profile";
+    public static final String SORTING = "sorting";
+    public static final String LIST_FEATURES = "listFeatures";
+    public static final String FRESHNESS = "freshness";
+    public static final String QUERYCACHE = "queryCache";
+    public static final String MATCH_PHASE = "matchPhase";
+    public static final String DIVERSITY = "diversity";
+    public static final String FEATURES = "features";
+    public static final String PROPERTIES = "properties";
+
+    static {
+        argumentType =new QueryProfileType(RANKING);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(LOCATION, "string", "location"));
+        argumentType.addField(new FieldDescription(PROFILE, "string", "ranking"));
+        argumentType.addField(new FieldDescription(SORTING, "string", "sorting sortspec"));
+        argumentType.addField(new FieldDescription(LIST_FEATURES, "string", RANKFEATURES.toString()));
+        argumentType.addField(new FieldDescription(FRESHNESS, "string", "datetime"));
+        argumentType.addField(new FieldDescription(QUERYCACHE, "string"));
+        argumentType.addField(new FieldDescription(MATCH_PHASE, "query-profile", "matchPhase"));
+        argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature"));
+        argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty"));
+        argumentType.freeze();
+        argumentTypeName=new CompoundName(argumentType.getId().getName());
+    }
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    private Query parent;
+
+    /** The location of the query is used for distance ranking */
+    private Location location = null;
+
+    /** The name of the rank profile to use */
+    private String profile = null;
+
+    /** How the query should be sorted */
+    private Sorting sorting = null;
+
+    /** Set to true to include the value of "all" rank features in the result */
+    private boolean listFeatures = false;
+
+    private Freshness freshness;
+
+    private boolean queryCache = false;
+
+    private RankProperties rankProperties = new RankProperties();
+
+    private RankFeatures rankFeatures = new RankFeatures();
+
+    private MatchPhase matchPhase = new MatchPhase();
+
+    public Ranking(Query parent) {
+        this.parent = parent;
+    }
+
+    /**
+     * Returns whether a rank profile has been explicitly set.
+     *
+     * This is only used in serializing the packet properly to FS4.
+     */
+    public boolean hasRankProfile() {
+        return profile != null;
+    }
+
+    /** Get the freshness search parameters associated with this query */
+    public Freshness getFreshness() {
+        return freshness;
+    }
+
+    /** Set the freshness search parameters for this query */
+    public void setFreshness(String dateTime) {
+       try {
+            Freshness freshness = new Freshness(dateTime);
+            setFreshness(freshness);
+        } catch (NumberFormatException e) {
+           parent.errors().add(ErrorMessage.createInvalidQueryParameter("Datetime reference could not be converted from '"
+                                                                        + dateTime + "' to long"));
+        }
+    }
+
+    public void setFreshness(Freshness freshness) {
+        this.freshness = freshness;
+    }
+
+    /**
+     * Returns whether feature caching is turned on in the backed.
+     * Feature caching allows us to avoid sending the query during document summary retrieval
+     * and recalculate feature scores, it is typically beneficial to turn it on if
+     * fan-out is low or queries are large.
+     * <p>
+     * Default is false (off).
+     */
+    public void setQueryCache(boolean queryCache) { this.queryCache = queryCache; }
+
+    public boolean getQueryCache() { return queryCache; }
+
+    /** Returns the location of this query, or null if none */
+    public Location getLocation() { return location; }
+
+    public void setLocation(Location location) { this.location = location; }
+
+    /** Sets the location from a string, see {@link Location} for syntax */
+    public void setLocation(String str) { this.location = new Location(str); }
+
+    /** Returns the name of the rank profile to be used. Returns "default" if nothing is set. */
+    public String getProfile() { return profile == null ? "default" : profile; }
+
+    /** Sets the name of the rank profile to use. This cannot be set to null. */
+    public void setProfile(String profile) {
+        if (profile==null) throw new NullPointerException("The ranking profile cannot be set to null");
+        this.profile = profile;
+    }
+
+    /**
+     * Returns the rank features of this, an empty container (never null) if none are set.
+     * The returned object can be modified directly to change the rank properties of this.
+     */
+    public RankFeatures getFeatures() {
+        return rankFeatures;
+    }
+
+    /**
+     * Returns the rank properties of this, an empty container (never null) if none are set.
+     * The returned object can be modified directly to change the rank properties of this.
+     */
+    public RankProperties getProperties() {
+        return rankProperties;
+    }
+
+    /** Set whether rank features should be included with the result of this query */
+    public void setListFeatures(boolean listFeatures) { this.listFeatures = listFeatures; }
+
+    /** Returns whether rank features should be dumped with the result of this query, default false */
+    public boolean getListFeatures() { return listFeatures; }
+
+    /** Returns the match phase rank settings of this. This is never null. */
+    public MatchPhase getMatchPhase() { return matchPhase; }
+
+    @Override
+    public Object clone() {
+        try {
+            Ranking clone = (Ranking) super.clone();
+
+            if (sorting != null) clone.sorting = this.sorting.clone();
+
+            clone.rankProperties = this.rankProperties.clone();
+            clone.rankFeatures = this.rankFeatures.clone();
+            clone.matchPhase = this.matchPhase.clone();
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Someone inserted a noncloneable superclass",e);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if( ! (o instanceof Ranking)) return false;
+
+        Ranking other = (Ranking) o;
+
+        if ( ! QueryHelper.equals(rankProperties, other.rankProperties)) return false;
+        if ( ! QueryHelper.equals(rankFeatures, other.rankFeatures)) return false;
+        if ( ! QueryHelper.equals(freshness, other.freshness)) return false;
+        if ( ! QueryHelper.equals(this.sorting, other.sorting)) return false;
+        if ( ! QueryHelper.equals(this.location, other.location)) return false;
+        if ( ! QueryHelper.equals(this.profile, other.profile)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        hash += 11 * rankFeatures.hashCode();
+        hash += 13 * rankProperties.hashCode();
+        hash += 17 * matchPhase.hashCode();
+        return Ranking.class.hashCode() + QueryHelper.combineHash(sorting,location,profile,hash);
+    }
+
+    /** Returns the sorting spec of this query, or null if none is set */
+    public Sorting getSorting() { return sorting; }
+
+    /** Sets how this query should be sorted. Set to null to turn off explicit sorting. */
+    public void setSorting(Sorting sorting) { this.sorting = sorting; }
+
+    /** Sets sorting from a string. See {@link Sorting} on syntax */
+    public void setSorting(String sortingString) {
+        if (sortingString==null)
+            setSorting((Sorting)null);
+        else
+            setSorting(new Sorting(sortingString));
+    }
+
+    public static Ranking getFrom(Query q) {
+        return (Ranking) q.properties().get(argumentTypeName);
+    }
+
+    public void prepare() {
+        rankFeatures.prepare(rankProperties);
+        matchPhase.prepare(rankProperties);
+        prepareNow(freshness);
+    }
+
+    private void prepareNow(Freshness freshness) {
+        if (freshness == null) return;
+        // TODO: See what freshness is doing with the internal props and simplify
+        if (rankProperties.get("vespa.now") == null || rankProperties.get("vespa.now").isEmpty()) {
+            rankProperties.put("vespa.now", "" + freshness.getRefTime());
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/SessionId.java b/container-search/src/main/java/com/yahoo/search/query/SessionId.java
new file mode 100644
index 00000000000..7f8ca6385e1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/SessionId.java
@@ -0,0 +1,36 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.yahoo.container.Server;
+import com.yahoo.text.Utf8String;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A query id which is unique across this cluster - consisting of
+ * container runtime id + timestamp + serial.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SessionId {
+
+    private static final String serverId = Server.get().getServerDiscriminator();
+    private static final AtomicLong sequenceCounter = new AtomicLong();
+
+    private final Utf8String id;
+
+    private SessionId(String serverId, long timestamp, long sequence) {
+        this.id = new Utf8String(serverId + "." + timestamp + "." + sequence);
+    }
+
+    public Utf8String asUtf8String() { return id; }
+
+    /**
+     * Creates a session id which is unique across the cluster this runtime is a member of each time this is called.
+     * Calling this causes synchronization.
+     */
+    public static SessionId next() {
+        return new SessionId(serverId, System.currentTimeMillis(), sequenceCounter.getAndIncrement());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java
new file mode 100644
index 00000000000..3af9bc34940
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java
@@ -0,0 +1,407 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.ibm.icu.text.Collator;
+import com.ibm.icu.util.ULocale;
+import com.yahoo.text.Utf8;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+
+/**
+ * Specifies how a query is sorted by a list of fields with a sort order
+ *
+ * @author Arne Bergene Fossaa
+ */
+public class Sorting implements Cloneable {
+
+    public static final String STRENGTH_IDENTICAL = "identical";
+    public static final String STRENGTH_QUATERNARY = "quaternary";
+    public static final String STRENGTH_TERTIARY = "tertiary";
+    public static final String STRENGTH_SECONDARY = "secondary";
+    public static final String STRENGTH_PRIMARY = "primary";
+    public static final String UCA = "uca";
+    public static final String RAW = "raw";
+    public static final String LOWERCASE = "lowercase";
+
+    private final List<FieldOrder> fieldOrders = new ArrayList<>(2);
+
+    /** Creates an empty sort spec */
+    public Sorting() { }
+
+    public Sorting(List<FieldOrder> fieldOrders) {
+        this.fieldOrders.addAll(fieldOrders);
+    }
+
+    /** Creates a sort spec from a string */
+    public Sorting(String sortSpec) {
+        setSpec(sortSpec);
+    }
+
+    /**
+     * Creates a new sorting from the given string and returns it, or returns null if the argument does not contain
+     * any sorting criteria (e.g it is null or the empty string)
+     */
+    public static Sorting fromString(String sortSpec) {
+        if (sortSpec==null) return null;
+        if ("".equals(sortSpec)) return null;
+        return new Sorting(sortSpec);
+    }
+
+    private void setSpec(String rawSortSpec) {
+        String[] vectors = rawSortSpec.split(" ");
+
+        for (String sortString:vectors) {
+            // A sortspec element must be at least two characters long,
+            // a sorting order and an attribute vector name
+            if (sortString.length() < 1) {
+                continue;
+            }
+            char orderMarker = sortString.charAt(0);
+            int funcAttrStart = 0;
+            if ((orderMarker == '+') || (orderMarker == '-')) {
+                funcAttrStart = 1;
+            }
+            AttributeSorter sorter = null;
+            int startPar = sortString.indexOf('(',funcAttrStart);
+            int endPar = sortString.lastIndexOf(')');
+            if ((startPar > 0) && (endPar > startPar)) {
+                String funcName = sortString.substring(funcAttrStart, startPar);
+                if (LOWERCASE.equalsIgnoreCase(funcName)) {
+                    sorter = new LowerCaseSorter(sortString.substring(startPar+1, endPar));
+                } else if (RAW.equalsIgnoreCase(funcName)) {
+                    sorter = new RawSorter(sortString.substring(startPar+1, endPar));
+                } else if (UCA.equalsIgnoreCase(funcName)) {
+                    int commaPos = sortString.indexOf(',', startPar+1);
+                    if ((startPar+1 < commaPos) && (commaPos < endPar)) {
+                        int commaopt = sortString.indexOf(',', commaPos + 1);
+                        UcaSorter.Strength strength = UcaSorter.Strength.UNDEFINED;
+                        if (commaopt > 0) {
+                            String s = sortString.substring(commaopt+1, endPar);
+                            if (STRENGTH_PRIMARY.equalsIgnoreCase(s)) {
+                                strength = UcaSorter.Strength.PRIMARY;
+                            } else if (STRENGTH_SECONDARY.equalsIgnoreCase(s)) {
+                                strength = UcaSorter.Strength.SECONDARY;
+                            } else if (STRENGTH_TERTIARY.equalsIgnoreCase(s)) {
+                                strength = UcaSorter.Strength.TERTIARY;
+                            } else if (STRENGTH_QUATERNARY.equalsIgnoreCase(s)) {
+                                strength = UcaSorter.Strength.QUATERNARY;
+                            } else if (STRENGTH_IDENTICAL.equalsIgnoreCase(s)) {
+                                strength = UcaSorter.Strength.IDENTICAL;
+                            } else {
+                                throw new IllegalArgumentException("Unknown collation strength: '" + s + "'");
+                            }
+                            sorter = new UcaSorter(sortString.substring(startPar+1, commaPos), sortString.substring(commaPos+1, commaopt), strength);
+                        } else {
+                            sorter = new UcaSorter(sortString.substring(startPar+1, commaPos), sortString.substring(commaPos+1, endPar), strength);
+                        }
+                    } else {
+                        sorter = new UcaSorter(sortString.substring(startPar+1, endPar));
+                    }
+                } else {
+                    if (funcName.isEmpty()) {
+                        throw new IllegalArgumentException("No sort function specified");
+                    } else {
+                        throw new IllegalArgumentException("Unknown sort function '" + funcName + "'");
+                    }
+                }
+            } else {
+                sorter = new AttributeSorter(sortString.substring(funcAttrStart));
+            }
+            Order order = Order.UNDEFINED;
+            if (funcAttrStart != 0) {
+                // Override in sortspec
+                order = (orderMarker == '+') ? Order.ASCENDING : Order.DESCENDING;
+            }
+            fieldOrders.add(new FieldOrder(sorter, order));
+        }
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+
+        String space = "";
+        for (FieldOrder spec : fieldOrders)   {
+            sb.append(space);
+            if (spec.getSortOrder() == Order.DESCENDING) {
+                sb.append("-");
+            } else {
+                sb.append("+");
+            }
+            sb.append(spec.getFieldName());
+            space = " ";
+        }
+        return sb.toString();
+    }
+
+
+    public enum Order {ASCENDING,DESCENDING,UNDEFINED}
+
+    /**
+     * Returns the field orders of this sort specification as list. This is never null but can be empty.
+     * This list can be modified to change this sort spec.
+     */
+    public List<FieldOrder> fieldOrders() { return fieldOrders; }
+
+    public Sorting clone() {
+        return new Sorting(this.fieldOrders);
+    }
+
+    public static class AttributeSorter implements Cloneable {
+        private static final Pattern legalAttributeName = Pattern.compile("[\\[]*[a-zA-Z_][\\.a-zA-Z0-9_-]*[\\]]*");
+
+        private String fieldName;
+        public AttributeSorter(String fieldName) {
+            if (legalAttributeName.matcher(fieldName).matches()) {
+                this.fieldName = fieldName;
+            } else {
+                throw new IllegalArgumentException("Illegal attribute name '" + fieldName + "' for sorting. Requires '" + legalAttributeName.pattern() + "'");
+            }
+        }
+        public String getName() { return fieldName; }
+        public void setName(String fieldName) { this.fieldName = fieldName; }
+        @Override
+        public String toString() { return fieldName; }
+        @Override
+        public int hashCode() { return fieldName.hashCode(); }
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof AttributeSorter)) {
+                return false;
+            }
+            return ((AttributeSorter) other).fieldName.equals(fieldName);
+        }
+        @Override
+        public AttributeSorter clone() {
+            try {
+                return (AttributeSorter)super.clone();
+            }
+            catch (CloneNotSupportedException e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+        @SuppressWarnings({ "rawtypes", "unchecked" })
+        public int compare(Comparable a, Comparable b) {
+            return a.compareTo(b);
+        }
+
+    }
+    public static class RawSorter extends AttributeSorter
+    {
+        public RawSorter(String fieldName) { super(fieldName); }
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof RawSorter)) {
+                return false;
+            }
+            return super.equals(other);
+        }
+    }
+    public static class LowerCaseSorter extends AttributeSorter
+    {
+        public LowerCaseSorter(String fieldName) { super(fieldName); }
+        @Override
+        public String toString() { return "lowercase(" + getName() + ')'; }
+        @Override
+        public int hashCode() { return 1 + 3*super.hashCode(); }
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof LowerCaseSorter)) {
+                return false;
+            }
+            return super.equals(other);
+        }
+        @SuppressWarnings({ "rawtypes", "unchecked" })
+        public int compare(Comparable a, Comparable b) {
+            if ((a instanceof String) && (b instanceof String)) {
+                return ((String)a).compareToIgnoreCase((String) b);
+            }
+            return a.compareTo(b);
+        }
+    }
+    public static class UcaSorter extends AttributeSorter
+    {
+        public enum Strength { PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL, UNDEFINED };
+        private String locale = null;
+        private Strength strength = Strength.UNDEFINED;
+        private Collator collator;
+        public UcaSorter(String fieldName, String locale, Strength strength) { super(fieldName); setLocale(locale, strength); }
+        public UcaSorter(String fieldName) { super(fieldName); }
+        static private int strength2Collator(Strength strength) {
+             switch (strength) {
+                 case PRIMARY: return Collator.PRIMARY;
+                 case SECONDARY: return Collator.SECONDARY;
+                 case TERTIARY: return Collator.TERTIARY;
+                 case QUATERNARY: return Collator.QUATERNARY;
+                 case IDENTICAL: return Collator.IDENTICAL;
+                 case UNDEFINED: return Collator.PRIMARY;
+             }
+             return Collator.PRIMARY;
+        }
+        public void setLocale(String locale, Strength strength) {
+            this.locale = locale;
+            this.strength = strength;
+            ULocale uloc;
+            try {
+                uloc = new ULocale(locale);
+            } catch (Throwable e) {
+                throw new RuntimeException("ULocale("+locale+") failed with exception " + e.toString());
+            }
+            try {
+                collator = Collator.getInstance(uloc);
+                if (collator == null) {
+                    throw new RuntimeException("No collator available for: " + locale);
+                }
+            } catch (Throwable e) {
+                throw new RuntimeException("Collator.getInstance(ULocale("+locale+")) failed with exception " + e.toString());
+            }
+            collator.setStrength(strength2Collator(strength));
+            // collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION);
+        }
+        public String getLocale() { return locale; }
+        public Strength getStrength() { return strength; }
+        public Collator getCollator() { return collator; }
+        public String getDecomposition() { return (collator.getDecomposition() == Collator.CANONICAL_DECOMPOSITION) ? "CANONICAL_DECOMPOSITION" : "NO_DECOMPOSITION"; }
+        @Override
+        public String toString() { return "uca(" + getName() + ',' + locale + ',' + ((strength != Strength.UNDEFINED) ? strength.toString() : "PRIMARY") + ')'; }
+        @Override
+        public int hashCode() { return 1 + 3*locale.hashCode() + 5*strength.hashCode() + 7*super.hashCode(); }
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof UcaSorter)) {
+                return false;
+            }
+            return super.equals(other) && locale.equals(((UcaSorter)other).locale) && (strength == ((UcaSorter)other).strength);
+        }
+        public UcaSorter clone() {
+            UcaSorter clone = (UcaSorter)super.clone();
+            if (locale != null) {
+                clone.setLocale(locale, strength);
+            }
+            return clone;
+        }
+        @SuppressWarnings({ "rawtypes", "unchecked" })
+        public int compare(Comparable a, Comparable b) {
+            if ((a instanceof String) && (b instanceof String)) {
+                return collator.compare((String)a, (String) b);
+            }
+            return a.compareTo(b);
+        }
+    }
+    /**
+     * An attribute (field) and how it should be sorted
+     */
+    public static class FieldOrder implements Cloneable {
+
+        private AttributeSorter fieldSorter;
+        private Order sortOrder;
+
+        /**
+         * Creates an attribute vector
+         *
+         * @param fieldSorter the sorter of this attribute
+         * @param sortOrder    whether to sort this ascending or descending
+         */
+        public FieldOrder(AttributeSorter fieldSorter, Order sortOrder) {
+            this.fieldSorter = fieldSorter;
+            this.sortOrder = sortOrder;
+        }
+
+        /**
+         * Returns the name of this attribute
+         */
+        public String getFieldName() {
+            return fieldSorter.getName();
+        }
+
+        /**
+         * Returns the sorter of this attribute
+         */
+        public AttributeSorter getSorter() { return fieldSorter; }
+        public void setSorter(AttributeSorter sorter) { fieldSorter = sorter; }
+
+        /**
+         * Returns the sorting order of this attribute
+         */
+        public Order getSortOrder() {
+            return sortOrder;
+        }
+
+        /**
+         * Decide if sortorder is ascending or not.
+         */
+        public void setAscending(boolean asc) {
+            sortOrder = asc ? Order.ASCENDING : Order.DESCENDING;
+        }
+
+        @Override
+        public String toString() {
+            return sortOrder.toString() + ":" + fieldSorter.toString();
+        }
+
+        @Override
+        public int hashCode() {
+            return sortOrder.hashCode() + 17 * fieldSorter.hashCode();
+        }
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof FieldOrder)) {
+                return false;
+            }
+            FieldOrder otherAttr = (FieldOrder) other;
+
+            return otherAttr.sortOrder.equals(sortOrder)
+                    && otherAttr.fieldSorter.equals(fieldSorter);
+        }
+        @Override
+        public FieldOrder clone() {
+            return new FieldOrder(fieldSorter.clone(), sortOrder);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return fieldOrders.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if( ! (o instanceof Sorting)) return false;
+
+        Sorting ss = (Sorting) o;
+        return fieldOrders.equals(ss.fieldOrders);
+    }
+
+    public int encode(ByteBuffer buffer) {
+        int usedBytes = 0;
+        byte[] nameBuffer;
+        buffer.position();
+        byte space = '.';
+        for (FieldOrder fieldOrder : fieldOrders) {
+            if (space == ' ')   {
+                buffer.put(space);
+                usedBytes++;
+            }
+            if (fieldOrder.getSortOrder() == Order.ASCENDING) {
+                buffer.put((byte) '+');
+            } else {
+                buffer.put((byte) '-');
+            }
+            usedBytes++;
+            nameBuffer = Utf8.toBytes(fieldOrder.getSorter().toString());
+            buffer.put(nameBuffer);
+            usedBytes += nameBuffer.length;
+            // If this isn't the last element, append a separating space
+            //if (i + 1 < sortSpec.size()) {
+            space = ' ';
+        }
+        return usedBytes;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java
new file mode 100644
index 00000000000..e59f8589903
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.context;
+
+import com.yahoo.processing.execution.Execution;
+import com.yahoo.search.Query;
+import com.yahoo.search.rendering.DefaultRenderer;
+import com.yahoo.text.XMLWriter;
+import com.yahoo.yolean.trace.TraceNode;
+
+import java.io.Writer;
+import java.util.Iterator;
+
+
+/**
+ * A proxy to the Execution.trace() which exists for legacy reasons.
+ * Calls to this is forwarded to owningQuery.getModel().getExecution().trace().
+ *
+ * @since 4.2
+ * @author  <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class QueryContext implements Cloneable {
+
+    public static final String ID = "context";
+    private Query owner;
+
+    public QueryContext(int ignored,Query owner) {
+        this.owner=owner;
+    }
+
+    //---------------- Public API ---------------------------------------------------------------------------------
+
+    /** Adds a context message to this context */
+    public void trace(String message, int traceLevel) {
+        owner.getModel().getExecution().trace().trace(message,traceLevel);
+    }
+
+    /**
+     * Adds a key-value which will be logged to the access log for this query (by doing toString() on the value
+     * Multiple values may be set to the same key. A value cannot be removed once set.
+     */
+    public void logValue(String key,Object value) {
+        owner.getModel().getExecution().trace().logValue(key, value.toString());
+    }
+
+    /** Returns the values to be written to the access log for this */
+    public Iterator<Execution.Trace.LogValue> logValueIterator() {
+        return owner.getModel().getExecution().trace().logValueIterator();
+    }
+
+    /**
+     * Adds a property key-value to this context.
+     * If the same name is set multiple times, the behavior is thus:
+     * <ul>
+     *   <li>Within a single context (thread/query clone), the last value set is used</li>
+     *   <li>Across multiple traces, the <i>last</i> value from the <i>last</i> deepest nested thread/clone is used.
+     *       In the case of multiple threads writing the value concurrently to their clone, it is of course undefined
+     *       which one will be used.</li>
+     * </ul>
+     *
+     * @param name the name of the property
+     * @param value the value of the property, or null to set this property to null
+     */
+    public void setProperty(String name,Object value) {
+        owner.getModel().getExecution().trace().setProperty(name,value);
+    }
+
+    /**
+     * Returns a property set anywhere in this context.
+     * Note that even though this call is itself "thread robust", the object values returned
+     * may in some scenarios not be written behind a synchronization barrier, so when accessing
+     * objects which are not inherently thread safe, synchronization should be considered.
+     * <p>
+     * Note that this method have a time complexity which is proportional to
+     * the number of cloned/created queries times the average number of properties in each.
+     */
+    public Object getProperty(String name) {
+        return owner.getModel().getExecution().trace().getProperty(name);
+    }
+
+    /** Returns a short string description of this (includes the first few messages only, and no newlines) */
+    @Override
+    public String toString() {
+        return owner.getModel().getExecution().trace().toString();
+    }
+
+    public boolean render(Writer writer) throws java.io.IOException {
+        if (owner.getTraceLevel()!=0) {
+            XMLWriter xmlWriter=XMLWriter.from(writer);
+            xmlWriter.openTag("meta").attribute("type",ID);
+            TraceNode traceRoot=owner.getModel().getExecution().trace().traceNode().root();
+            traceRoot.accept(new DefaultRenderer.RenderingVisitor(xmlWriter,owner.getStartTime()));
+            xmlWriter.closeTag();
+        }
+        return true;
+    }
+
+    public QueryContext cloneFor(Query cloneOwner) {
+        try {
+            QueryContext clone=(QueryContext)super.clone();
+            clone.owner=cloneOwner;
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /** Returns the execution trace this delegates to */
+    public Execution.Trace getTrace() { return owner.getModel().getExecution().trace(); }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/context/package-info.java b/container-search/src/main/java/com/yahoo/search/query/context/package-info.java
new file mode 100644
index 00000000000..c19e5abedd0
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/context/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.context;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/package-info.java b/container-search/src/main/java/com/yahoo/search/query/package-info.java
new file mode 100644
index 00000000000..2384169c52b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * The search query model
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
new file mode 100644
index 00000000000..92601a5464d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
@@ -0,0 +1,112 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.parser;
+
+import com.yahoo.language.Language;
+import com.yahoo.search.query.Model;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * <p>This class encapsulates all the parameters required to call {@link Parser#parse(Parsable)}. Because all set-
+ * methods return a reference to self, you can write very compact calls to the parser:</p>
+ *
+ * <pre>
+ * parser.parse(new Parsable()
+ *                  .setQuery("foo")
+ *                  .setFilter("bar")
+ *                  .setDefaultIndexName("default")
+ *                  .setLanguage(Language.ENGLISH))
+ * </pre>
+ *
+ * <p>In case you are parsing the content of a {@link Model}, you can use the {@link #fromQueryModel(Model)} factory for
+ * convenience.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @since 5.1.4
+ */
+public final class Parsable {
+
+    private final Set<String> sourceList = new HashSet<>();
+    private final Set<String> restrictList = new HashSet<>();
+    private String query;
+    private String filter;
+    private String defaultIndexName;
+    private Language language;
+
+    public String getQuery() {
+        return query;
+    }
+
+    public Parsable setQuery(String query) {
+        this.query = query;
+        return this;
+    }
+
+    public String getFilter() {
+        return filter;
+    }
+
+    public Parsable setFilter(String filter) {
+        this.filter = filter;
+        return this;
+    }
+
+    public String getDefaultIndexName() {
+        return defaultIndexName;
+    }
+
+    public Parsable setDefaultIndexName(String defaultIndexName) {
+        this.defaultIndexName = defaultIndexName;
+        return this;
+    }
+
+    public Language getLanguage() {
+        return language;
+    }
+
+    public Parsable setLanguage(Language language) {
+        this.language = language;
+        return this;
+    }
+
+    public Set<String> getSources() {
+        return sourceList;
+    }
+
+    public Parsable addSource(String sourceName) {
+        sourceList.add(sourceName);
+        return this;
+    }
+
+    public Parsable addSources(Collection<String> sourceNames) {
+        sourceList.addAll(sourceNames);
+        return this;
+    }
+
+    public Set<String> getRestrict() {
+        return restrictList;
+    }
+
+    public Parsable addRestrict(String restrictName) {
+        restrictList.add(restrictName);
+        return this;
+    }
+
+    public Parsable addRestricts(Collection<String> restrictNames) {
+        restrictList.addAll(restrictNames);
+        return this;
+    }
+
+    public static Parsable fromQueryModel(Model model) {
+        return new Parsable()
+                .setQuery(model.getQueryString())
+                .setFilter(model.getFilter())
+                .setLanguage(model.getParsingLanguage())
+                .setDefaultIndexName(model.getDefaultIndex())
+                .addSources(model.getSources())
+                .addRestricts(model.getRestrict());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java
new file mode 100644
index 00000000000..3822b9b67d8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parser.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.parser;
+
+import com.yahoo.search.query.QueryTree;
+
+/**
+ * Defines the interface of a query parser. To construct an instance of this class, use the {@link ParserFactory}.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+public interface Parser {
+
+    /**
+     * Parser the given {@link Parsable}, and returns a corresponding
+     * {@link QueryTree}. If parsing fails without an exception, the contained
+     * root will be an instance of {@link com.yahoo.prelude.query.NullItem}.
+     *
+     * @param query
+     *            the Parsable to parse
+     * @return the parsed QueryTree, never null
+     */
+    QueryTree parse(Parsable query);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
new file mode 100644
index 00000000000..b00afa27bf6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.parser;
+
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * This class encapsulates the environment of a {@link Parser}. In case you are creating a parser from within a
+ * {@link Searcher}, you can use the {@link #fromExecutionContext(Execution.Context)} factory for convenience.
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @since 5.1.4
+ */
+public final class ParserEnvironment {
+
+    private IndexFacts indexFacts = new IndexFacts();
+    private Linguistics linguistics = new SimpleLinguistics();
+    private SpecialTokens specialTokens = new SpecialTokens();
+
+    public IndexFacts getIndexFacts() {
+        return indexFacts;
+    }
+
+    public ParserEnvironment setIndexFacts(IndexFacts indexFacts) {
+        this.indexFacts = indexFacts;
+        return this;
+    }
+
+    public Linguistics getLinguistics() {
+        return linguistics;
+    }
+
+    public ParserEnvironment setLinguistics(Linguistics linguistics) {
+        this.linguistics = linguistics;
+        return this;
+    }
+
+    public SpecialTokens getSpecialTokens() {
+        return specialTokens;
+    }
+
+    public ParserEnvironment setSpecialTokens(SpecialTokens specialTokens) {
+        this.specialTokens = specialTokens;
+        return this;
+    }
+
+    public static ParserEnvironment fromExecutionContext(Execution.Context context) {
+        ParserEnvironment env = new ParserEnvironment();
+        if (context == null) {
+            return env;
+        }
+        if (context.getIndexFacts() != null) {
+            env.setIndexFacts(context.getIndexFacts());
+        }
+        if (context.getLinguistics() != null) {
+            env.setLinguistics(context.getLinguistics());
+        }
+        SpecialTokenRegistry registry = context.getTokenRegistry();
+        if (registry != null) {
+            env.setSpecialTokens(registry.getSpecialTokens("default"));
+        }
+        return env;
+    }
+
+    public static ParserEnvironment fromParserEnvironment(ParserEnvironment environment) {
+        return new ParserEnvironment()
+                .setIndexFacts(environment.indexFacts)
+                .setLinguistics(environment.linguistics)
+                .setSpecialTokens(environment.specialTokens);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
new file mode 100644
index 00000000000..e0a3338fec2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
@@ -0,0 +1,48 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.parser;
+
+import com.yahoo.prelude.query.parser.*;
+import com.yahoo.search.Query;
+import com.yahoo.search.yql.YqlParser;
+
+/**
+ * <p>Implements a factory for {@link Parser}.</p>
+ *
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @since 5.1.4
+ */
+public final class ParserFactory {
+
+    private ParserFactory() {
+        // hide
+    }
+
+    /**
+     * Creates a {@link Parser} appropriate for the given <tt>Query.Type</tt>, providing the Parser with access to
+     * the {@link ParserEnvironment} given.
+     *
+     * @param type        the query type for which to create a Parser
+     * @param environment the environment settings to attach to the Parser
+     * @return the created Parser
+     */
+    public static Parser newInstance(Query.Type type, ParserEnvironment environment) {
+        switch (type) {
+        case ALL:
+            return new AllParser(environment);
+        case ANY:
+            return new AnyParser(environment);
+        case PHRASE:
+            return new PhraseParser(environment);
+        case ADVANCED:
+            return new AdvancedParser(environment);
+        case WEB:
+            return new WebParser(environment);
+        case PROGRAMMATIC:
+            return new ProgrammaticParser();
+        case YQL:
+            return new YqlParser(environment);
+        default:
+            throw new UnsupportedOperationException(type.toString());
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/package-info.java b/container-search/src/main/java/com/yahoo/search/query/parser/package-info.java
new file mode 100644
index 00000000000..ddae3e83ddb
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Provides access to parsing query strings into queries
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.parser;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java
new file mode 100644
index 00000000000..393aba2b002
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileFieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+final class AllReferencesQueryProfileVisitor extends PrefixQueryProfileVisitor {
+
+    /** A map of query profile types */
+    private Set<CompoundName> references = new HashSet<>();
+
+    public AllReferencesQueryProfileVisitor(CompoundName prefix) {
+        super(prefix);
+    }
+
+    @Override
+    public void onValue(String name, Object value, DimensionBinding binding, QueryProfile owner) {}
+
+    @Override
+    public void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) {
+        references.add(currentPrefix);
+    }
+
+    /** Returns the values resulting from this visiting */
+    public Set<CompoundName> getResult() { return references; }
+
+    /** Returns false - we are not done until we have seen all */
+    public boolean isDone() { return false; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java
new file mode 100644
index 00000000000..fb9638a958b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileFieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+final class AllTypesQueryProfileVisitor extends PrefixQueryProfileVisitor {
+
+    /** A map of query profile types */
+    private Map<CompoundName, QueryProfileType> types = new HashMap<>();
+
+    public AllTypesQueryProfileVisitor(CompoundName prefix) {
+        super(prefix);
+    }
+
+    @Override
+    public void onValue(String name, Object value, DimensionBinding binding, QueryProfile owner) {}
+
+
+    @Override
+    public void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) {
+        if (profile.getType() != null)
+            addReachableTypes(currentPrefix, profile.getType());
+    }
+
+    private void addReachableTypes(CompoundName name, QueryProfileType type) {
+        types.put(name, type);
+        for (FieldDescription fieldDescription : type.fields().values()) {
+            if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) continue;
+            QueryProfileFieldType fieldType = (QueryProfileFieldType)fieldDescription.getType();
+            if (fieldType.getQueryProfileType() !=null) {
+                addReachableTypes(name.append(fieldDescription.getName()), fieldType.getQueryProfileType());
+            }
+        }
+    }
+
+    /** Returns the values resulting from this visiting */
+    public Map<CompoundName, QueryProfileType> getResult() { return types; }
+
+    /** Returns false - we are not done until we have seen all */
+    public boolean isDone() { return false; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java
new file mode 100644
index 00000000000..65c3480272e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java
@@ -0,0 +1,45 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+final class AllUnoverridableQueryProfileVisitor extends PrefixQueryProfileVisitor {
+
+    /** A map of query profile types */
+    private Set<CompoundName> unoverridables = new HashSet<>();
+
+    public AllUnoverridableQueryProfileVisitor(CompoundName prefix) {
+        super(prefix);
+    }
+
+    @Override
+    public void onValue(String name, Object value, DimensionBinding binding, QueryProfile owner) {
+        addUnoverridable(name, currentPrefix.append(name), binding, owner);
+    }
+
+    @Override
+    public void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) {
+        addUnoverridable(currentPrefix.last(), currentPrefix, binding, owner);
+    }
+
+    private void addUnoverridable(String localName, CompoundName fullName, DimensionBinding binding, QueryProfile owner) {
+        if (owner == null) return;
+
+        Boolean isOverridable = owner.isLocalOverridable(localName, binding);
+        if (isOverridable != null &&  ! isOverridable)
+            unoverridables.add(fullName);
+    }
+
+    /** Returns the values resulting from this visiting */
+    public Set<CompoundName> getResult() { return unoverridables; }
+
+    /** Returns false - we are not done until we have seen all */
+    public boolean isDone() { return false; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java
new file mode 100644
index 00000000000..bef5b00c51b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor {
+
+    private Map<String,Object> values=new HashMap<>();
+
+    /* Lists all values starting at prefix */
+    public AllValuesQueryProfileVisitor(CompoundName prefix) {
+        super(prefix);
+    }
+
+    public @Override void onValue(String localName, Object value, DimensionBinding binding, QueryProfile owner) {
+        putValue(localName, value, values);
+    }
+
+    public @Override void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner) {
+        putValue("", profile.getValue(), values);
+    }
+
+    private final void putValue(String key, Object value, Map<String, Object> values) {
+        if (value == null) return;
+        CompoundName fullName = currentPrefix.append(key);
+        if (fullName.isEmpty()) return; // Avoid putting a non-leaf (subtree) root in the list
+        if (values.containsKey(fullName.toString())) return; // The first value encountered has priority
+        values.put(fullName.toString(), value);
+    }
+
+    /** Returns the values resulting from this visiting */
+    public Map<String, Object> getResult() { return values; }
+
+    /** Returns false - we are not done until we have seen all */
+    public boolean isDone() { return false; }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java
new file mode 100644
index 00000000000..71b27c6da63
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java
@@ -0,0 +1,139 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.protect.Validator;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>A wrapper of a query profile where overrides to the values in the referenced
+ * profile can be set.</p>
+ *
+ * <p>This is used to allow configured overrides (in a particular referencing profile) of a referenced query profile.
+ *
+ * <p>Properties which are defined as not overridable in the type (if any) of the referenced query profile
+ * cannot be set.</p>
+ *
+ * @author bratseth
+ */
+public class BackedOverridableQueryProfile extends OverridableQueryProfile implements Cloneable {
+
+    /** The backing read only query profile, or null if this is not backed */
+    private QueryProfile backingProfile;
+
+    /**
+     * Creates an overridable profile from the given backing profile. The backing profile will never be
+     * written to.
+     *
+     * @param backingProfile the backing profile, which is assumed read only, never null
+     */
+    public BackedOverridableQueryProfile(QueryProfile backingProfile) {
+        Validator.ensureNotNull("An overridable query profile must be backed by a real query profile",backingProfile);
+        setType(backingProfile.getType());
+        this.backingProfile=backingProfile;
+    }
+
+    @Override
+    public synchronized void freeze() {
+        super.freeze();
+        backingProfile.freeze();
+    }
+
+    @Override
+    protected Object localLookup(String localName, DimensionBinding dimensionBinding) {
+        Object valueInThis=super.localLookup(localName,dimensionBinding);
+        if (valueInThis!=null) return valueInThis;
+        return backingProfile.localLookup(localName,dimensionBinding);
+    }
+
+    protected Boolean isLocalInstanceOverridable(String localName) {
+        Boolean valueInThis=super.isLocalInstanceOverridable(localName);
+        if (valueInThis!=null) return valueInThis;
+        return backingProfile.isLocalInstanceOverridable(localName);
+    }
+
+    @Override
+    protected QueryProfile createSubProfile(String name,DimensionBinding dimensionBinding) {
+        Object backing=backingProfile.lookup(new CompoundName(name),true,dimensionBinding.createFor(backingProfile.getDimensions()));
+        if (backing!=null && backing instanceof QueryProfile)
+            return new BackedOverridableQueryProfile((QueryProfile)backing);
+        else
+            return new OverridableQueryProfile(); // Nothing is set in this branch, so nothing to override, but need override checking
+    }
+
+    /** Returns a clone of this which can be independently overridden, but which refers to the same backing profile */
+    @Override
+    public BackedOverridableQueryProfile clone() {
+        BackedOverridableQueryProfile clone=(BackedOverridableQueryProfile)super.clone();
+        return clone;
+    }
+
+    /** Returns the query profile backing this */
+    public QueryProfile getBacking() { return backingProfile; }
+
+    @Override
+    public void addInherited(QueryProfile inherited) {
+        backingProfile.addInherited(inherited);
+    }
+
+    void addInheritedHere(QueryProfile inherited) {
+        super.addInherited(inherited);
+    }
+
+    @Override
+    protected void visitVariants(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) {
+        super.visitVariants(allowContent, visitor, dimensionBinding);
+        if (visitor.isDone()) return;
+        backingProfile.visitVariants(allowContent, visitor, dimensionBinding);
+    }
+
+    @Override
+    protected void visitInherited(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) {
+        super.visitInherited(allowContent,visitor,dimensionBinding, owner);
+        if (visitor.isDone()) return;
+        backingProfile.visitInherited(allowContent,visitor,dimensionBinding,owner);
+    }
+
+    /** Returns a value from the content of this: The value in this, or the value from the backing if not set in this */
+    protected Object getContent(String localKey) {
+        Object value=super.getContent(localKey);
+        if (value!=null) return value;
+        return backingProfile.getContent(localKey);
+    }
+
+    /**
+     * Returns all the content from this:
+     * All the values in this, and all values in the backing where an overriding value is not set in this
+     */
+    @Override
+    protected Map<String,Object> getContent() {
+        Map<String,Object> thisContent=super.getContent();
+        Map<String,Object> backingContent=backingProfile.getContent();
+        if (thisContent.isEmpty()) return backingContent; // Shortcut
+        if (backingContent.isEmpty()) return thisContent; // Shortcut
+        Map<String,Object> content=new HashMap<>(backingContent);
+        content.putAll(thisContent);
+        return content;
+    }
+
+    @Override
+    public String toString() {
+        return "overridable wrapper of " + backingProfile.toString();
+    }
+
+    @Override
+    public boolean isExplicit() {
+        return backingProfile.isExplicit();
+    }
+
+    @Override
+    public List<String> getDimensions() {
+        List<String> dimensions=super.getDimensions();
+        if (dimensions!=null) return dimensions;
+        return backingProfile.getDimensions();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java b/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java
new file mode 100644
index 00000000000..3c02677b676
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/CopyOnWriteContent.java
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.component.provider.FreezableClass;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A HashMap wrapper which can be cloned without copying the wrapped map.
+ * Copying of the map is deferred until there is a write access to the wrapped map.
+ * This may be frozen, at which point no further modifications are allowed.
+ * Note that <b>until</b> this is cloned, the internal map may be both read and written.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class CopyOnWriteContent extends FreezableClass implements Cloneable {
+
+    // TODO: Now that we used CompiledQueryProfiles at runtime we can remove this
+
+    // Possible states:
+    // WRITABLE:    The map can be freely modified - it is only used by this
+    //              -> !isFrozen() && (map!=null || unmodifiableMap==null)
+    // COPYONWRITE: The map is referred by at least one clone - further modification must cause a copy
+    //              -> !isFrozen() && (map==null && unmodifiableMap!=null)
+    // FROZEN:      No further changes are allowed to the state of this, ever
+    //              -> isFrozen()
+
+    // Possible start states:
+    // WRITABLE:    When created using the public constructor
+    // COPYONWRITE: When created by cloning
+
+    // Possible state transitions:
+    // WRITABLE->COPYONWRITE:          When this is cloned
+    // COPYONWRITE->WRITABLE:          When a clone is written to
+    // (COPYONWRITE,WRITABLE)->FROZEN: When a profile is frozen
+
+    /** The modifiable content of this. Null if this is empty or if this is not in the WRITABLE state */
+    private Map<String,Object> map=null;
+    /**
+     * If map is non-null this is either null (not instantiated yet) or an unmodifiable wrapper of map,
+     * if map is null this is either null (this is empty) or a reference to the map of the content this was cloned from
+     */
+    private Map<String,Object> unmodifiableMap =null;
+
+    /** Create a WRITABLE, empty instance */
+    public CopyOnWriteContent() {
+    }
+
+    /** Create a COPYONWRITE instance with some initial state */
+    private static CopyOnWriteContent createInCopyOnWriteState(Map<String,Object> unmodifiableMap) {
+        CopyOnWriteContent content=new CopyOnWriteContent();
+        content.unmodifiableMap = unmodifiableMap;
+        return content;
+    }
+
+    /** Create a WRITABLE instance with some initial state */
+    private static CopyOnWriteContent createInWritableState(Map<String,Object> map) {
+        CopyOnWriteContent content=new CopyOnWriteContent();
+        content.map = map;
+        return content;
+    }
+
+    @Override
+    public void freeze() {
+        // Freeze this
+        if (unmodifiableMap==null)
+            unmodifiableMap= map!=null ? Collections.unmodifiableMap(map) : Collections.<String, Object>emptyMap();
+        map=null; // just to keep the states simpler
+
+        // Freeze content
+        for (Map.Entry<String,Object> entry : unmodifiableMap.entrySet()) {
+            if (entry.getValue() instanceof QueryProfile)
+                ((QueryProfile)entry.getValue()).freeze();
+        }
+        super.freeze();
+    }
+
+    private boolean isEmpty() {
+        return (map==null || map.isEmpty()) && (unmodifiableMap ==null || unmodifiableMap.isEmpty());
+    }
+
+    private boolean isWritable() {
+        return !isFrozen() && (map!=null || unmodifiableMap==null);
+    }
+
+    @Override
+    public CopyOnWriteContent clone() {
+        if (isEmpty()) return new CopyOnWriteContent(); // No referencing is necessary in this case
+        if (isDeepUnmodifiable(unmodifiableMap())) {
+            // Create an instance pointing to this and put both in the COPYONWRITE state
+            unmodifiableMap(); // Make sure we have an unmodifiable reference to the map below
+            map=null; // Put this into the COPYONWRITE state (unless it is already frozen, in which case this is a noop)
+            return createInCopyOnWriteState(unmodifiableMap());
+        }
+        else {
+            // This contains query profiles, don't try to defer copying
+            return createInWritableState(deepClone(map));
+        }
+    }
+
+    private boolean isDeepUnmodifiable(Map<String,Object> map) {
+        for (Object value : map.values())
+            if (value instanceof QueryProfile && !((QueryProfile)value).isFrozen()) return false;
+        return true; // all other values are primitives
+    }
+
+    /** Deep clones a map - this handles all value types which can be found in a query profile */
+    static Map<String,Object> deepClone(Map<String,Object> map) {
+        if (map==null) return null;
+        Map<String,Object> mapClone=new HashMap<>(map.size());
+        for (Map.Entry<String,Object> entry : map.entrySet())
+            mapClone.put(entry.getKey(),QueryProfile.cloneIfNecessary(entry.getValue()));
+        return mapClone;
+    }
+
+
+    //------- Content access -------------------------------------------------------
+
+    public Map<String,Object> unmodifiableMap() {
+        if (isEmpty()) return Collections.emptyMap();
+        if (map==null) // in COPYONWRITE or FROZEN state
+            return unmodifiableMap;
+        // In WRITABLE state: Create unmodifiable wrapper if necessary and return it
+        if (unmodifiableMap==null)
+            unmodifiableMap=Collections.unmodifiableMap(map);
+        return unmodifiableMap;
+    }
+
+    public Object get(String key) {
+        if (map!=null) return map.get(key);
+        if (unmodifiableMap!=null) return unmodifiableMap.get(key);
+        return null;
+    }
+
+    public void put(String key,Object value) {
+        ensureNotFrozen();
+        copyIfNotWritable();
+        if (map==null)
+            map=new HashMap<>();
+        map.put(key,value);
+    }
+
+    public void remove(String key) {
+        ensureNotFrozen();
+        copyIfNotWritable();
+        if (map!=null)
+            map.remove(key);
+    }
+
+    private void copyIfNotWritable() {
+        if (isWritable()) return;
+        // move from COPYONWRITE to WRITABLE state
+        map=new HashMap<>(unmodifiableMap); // deep clone is not necessary as this map is shallowly modifiable
+        unmodifiableMap=null; // will be created as needed
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java
new file mode 100644
index 00000000000..9adacee74af
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java
@@ -0,0 +1,223 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An immutable, binding of a list of dimensions to dimension values
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DimensionBinding {
+
+    /** The dimensions of this */
+    private List<String> dimensions=null;
+
+    /** The values matching those dimensions */
+    private DimensionValues values;
+
+    /** The binding from those dimensions to values, and possibly other values */
+    private Map<String,String> context;
+
+    public static final DimensionBinding nullBinding =
+        new DimensionBinding(Collections.<String>unmodifiableList(Collections.<String>emptyList()), DimensionValues.empty, null);
+
+    public static final DimensionBinding invalidBinding =
+        new DimensionBinding(Collections.<String>unmodifiableList(Collections.<String>emptyList()), DimensionValues.empty, null);
+
+    /** Whether the value array contains only nulls */
+    private boolean containsAllNulls;
+
+    /** Creates a binding from a variant and a context. Any of the arguments may be null. */
+    public static DimensionBinding createFrom(List<String> dimensions, Map<String,String> context) {
+        if (dimensions==null || dimensions.size()==0) {
+            if (context==null) return nullBinding;
+            if (dimensions==null) return new DimensionBinding(null,DimensionValues.empty,context); // Null, but must preserve context
+        }
+
+        return new DimensionBinding(dimensions,extractDimensionValues(dimensions,context),context);
+    }
+
+    /** Creates a binding from a variant and a context. Any of the arguments may be null. */
+    public static DimensionBinding createFrom(List<String> dimensions, DimensionValues dimensionValues) {
+        if (dimensionValues==null || dimensionValues==DimensionValues.empty) return nullBinding;
+        if (dimensions==null) return new DimensionBinding(null,dimensionValues,null); // Null, but preserve raw material for creating a context later (in createFor)
+
+        return new DimensionBinding(dimensions,dimensionValues,null);
+    }
+
+    /** Returns a binding for a (possibly) new set of variants. Variants may be null, but not bindings */
+    public DimensionBinding createFor(List<String> newDimensions) {
+        if (newDimensions==null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding
+        // if (this.context==null && values.length==0) return nullBinding; // No data from which to create a non-null binding
+        if (this.dimensions==newDimensions) return this; // Avoid creating a new object if the dimensions are the same
+
+        Map<String,String> context=this.context;
+        if (context==null)
+            context=this.values.asContext(this.dimensions !=null ? this.dimensions : newDimensions);
+        return new DimensionBinding(newDimensions,extractDimensionValues(newDimensions,context),context);
+    }
+
+    /**
+     * Creates a dimension binding. The dimensions list given should be unmodifiable.
+     * The array will not be modified. The context is needed in order to convert this binding to another
+     * given another set of variant dimensions.
+     */
+    private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String,String> context) {
+        this.dimensions=dimensions;
+        this.values=values;
+        this.context = context;
+        containsAllNulls=values.isEmpty();
+    }
+
+    /** Returns a read-only list of the dimensions of this. This value is undefined if this isNull() */
+    public List<String> getDimensions() { return dimensions; }
+
+    /** Returns a context created from the dimensions and values of this */
+    public Map<String,String> getContext() {
+        if (context !=null) return context;
+        context =values.asContext(dimensions);
+        return context;
+    }
+
+    /**
+     * Returns the values for the dimensions of this. This value is undefined if this isEmpty()
+     * This array is always of the same length as the
+     * length of the dimension list - missing elements are represented as nulls.
+     * This is never null but may be empty.
+     */
+    public DimensionValues getValues() { return values; }
+
+    /** Returns true only if this binding is null (contains no values for its dimensions (if any) */
+    public boolean isNull() { return dimensions==null || containsAllNulls; }
+
+    /**
+     * Returns an array of the dimension values corresponding to the dimensions of this from the given context,
+     * in the corresponding order. The array is always of the same length as the number of dimensions.
+     * Dimensions which are not set in this context get a null value.
+     */
+    private static DimensionValues extractDimensionValues(List<String> dimensions,Map<String,String> context) {
+        String[] dimensionValues=new String[dimensions.size()];
+        if (context==null || context.size()==0) return DimensionValues.createFrom(dimensionValues);
+        for (int i=0; i<dimensions.size(); i++)
+            dimensionValues[i]=context.get(dimensions.get(i));
+        return DimensionValues.createFrom(dimensionValues);
+    }
+
+    /**
+     * Combines this binding with another if compatible.
+     * Two bindings are incompatible if
+     * <ul>
+     *     <li>They contain a different value for the same key, or</li>
+     *     <li>They contain the same pair of dimensions in a different order</li>
+     * </ul>
+     *
+     * @return the combined binding, or the special invalidBinding if these two bindings are incompatible
+     */
+    public DimensionBinding combineWith(DimensionBinding binding) {
+        List<String> combinedDimensions = combineDimensions(getDimensions(), binding.getDimensions());
+        if (combinedDimensions == null) return invalidBinding;
+
+        // not runtime, so assume we don't need to preserve values outside the dimensions
+        Map<String, String> combinedValues = combineValues(getContext(), binding.getContext());
+        if (combinedValues == null) return invalidBinding;
+
+        return DimensionBinding.createFrom(combinedDimensions, combinedValues);
+    }
+
+    /**
+     * Returns a combined list of dimensions from two separate lists,
+     * or null if they are incompatible.
+     * This is to combine two lists to one such that the partial order in both is preserved
+     * (or return null if impossible).
+     */
+    private List<String> combineDimensions(List<String> d1, List<String> d2) {
+        List<String> combined = new ArrayList<>();
+        int d1Index = 0, d2Index=0;
+        while (d1Index < d1.size() && d2Index < d2.size()) {
+            if (d1.get(d1Index).equals(d2.get(d2Index))) { // agreement on next element
+                combined.add(d1.get(d1Index));
+                d1Index++;
+                d2Index++;
+            }
+            else if ( ! d2.contains(d1.get(d1Index))) { // next in d1 is independent from d2
+                combined.add(d1.get(d1Index++));
+            }
+            else if ( ! d1.contains(d2.get(d2Index))) { // next in d2 is independent from d1
+                combined.add(d2.get(d2Index++));
+            }
+            else {
+                return null; // no independent and no agreement
+            }
+        }
+        if (d1Index < d1.size())
+            combined.addAll(d1.subList(d1Index, d1.size()));
+        else if (d2Index < d2.size())
+            combined.addAll(d2.subList(d2Index, d2.size()));
+
+        return combined;
+    }
+
+    /**
+     * Returns a combined map of dimension values from two separate maps,
+     * or null if they are incompatible.
+     */
+    private Map<String, String> combineValues(Map<String, String> m1, Map<String, String> m2) {
+        Map<String, String> combinedValues = new HashMap<>(m1);
+        for (Map.Entry<String, String> m2Entry : m2.entrySet()) {
+            if (m2Entry.getValue() == null) continue;
+            String m1Value = m1.get(m2Entry.getKey());
+            if (m1Value != null && ! m1Value.equals(m2Entry.getValue()))
+                return null; // conflicting values of a key
+            combinedValues.put(m2Entry.getKey(), m2Entry.getValue());
+        }
+        return combinedValues;
+    }
+
+    private boolean intersects(List<String> l1, List<String> l2) {
+        for (String l1Item : l1)
+            if (l2.contains(l1Item))
+                return true;
+        return false;
+    }
+
+    /**
+     * Returns true if <code>this == invalidBinding</code>
+     */
+    public boolean isInvalid() { return this == invalidBinding; }
+
+    @Override
+    public String toString() {
+        if (isInvalid()) return "Invalid DimensionBinding";
+        if (dimensions==null) return "DimensionBinding []";
+        StringBuilder b=new StringBuilder("DimensionBinding [");
+        for (int i=0; i<dimensions.size(); i++) {
+            b.append(dimensions.get(i)).append("=").append(values.get(i));
+            if (i<dimensions.size()-1)
+                b.append(", ");
+        }
+        b.append("]");
+        return b.toString();
+    }
+
+    /** Two bindings are equal if they contain the same dimensions and the same non-null values */
+    @Override
+    public boolean equals(Object o) {
+        if (o==this) return true;
+        if (! (o instanceof DimensionBinding)) return false;
+        DimensionBinding other = (DimensionBinding)o;
+        if ( ! this.dimensions.equals(other.dimensions)) return false;
+        if ( ! this.values.equals(other.values)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return dimensions.hashCode() + 17 * values.hashCode();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java
new file mode 100644
index 00000000000..10435c4c6b5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An immutable set of dimension values.
+ * Note that this may contain more or fewer values than needed given a set of dimensions.
+ * Any missing values are treated as null.
+ */
+public class DimensionValues implements Comparable<DimensionValues> {
+
+    private final String[] values;
+
+    public static final DimensionValues empty=new DimensionValues(new String[] {});
+
+    public static DimensionValues createFrom(String[] values) {
+        if (values==null || values.length==0 || containsAllNulls(values)) return empty;
+        return new DimensionValues(values);
+    }
+
+    /**
+     * Creates a set of dimension values, where the input array <b>must</b> be of
+     * the right size, and where no copying is done.
+     *
+     * @param values the dimension values. This need not be normalized to the right size.
+     *        The input array is copied by this.
+     */
+    private DimensionValues(String[] values) {
+        if (values==null) throw new NullPointerException("Dimension values cannot be null");
+        this.values=Arrays.copyOf(values,values.length);
+    }
+
+    /** Returns true if this is has the same value every place it has a value as the givenValues. */
+    public boolean matches(DimensionValues givenValues) {
+        for (int i=0; i<this.size() || i<givenValues.size() ; i++)
+            if ( ! matches(this.get(i),givenValues.get(i)))
+                return false;
+        return true;
+    }
+
+    private final boolean matches(String conditionString,String checkString) {
+        if (conditionString==null) return true;
+        return conditionString.equals(checkString);
+    }
+
+    /**
+     * Implements the sort order of this which is based on specificity
+     * where dimensions to the left are more significant:
+     * -1 is returned if this is more specific than other,
+     * 1 is returned if other is more specific than this,
+     * 0 is returned if none is more specific than the other.
+     * <p>
+     * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions
+     * are <i>set</i>, regardless of what they are set <i>to</i>.
+     */
+    @Override
+    public int compareTo(DimensionValues other) {
+        for (int i=0; i<this.size() || i<other.size(); i++) {
+            if (get(i)!=null && other.get(i)==null)
+                return -1;
+            if (get(i)==null && other.get(i)!=null)
+                return 1;
+        }
+        return 0;
+    }
+
+    /** Helper method which uses compareTo to return whether this is most specific */
+    public boolean isMoreSpecificThan(DimensionValues other) {
+        return this.compareTo(other)<0;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this==o) return true;
+        if ( ! (o instanceof DimensionValues)) return false;
+        DimensionValues other=(DimensionValues)o;
+        for (int i=0; i<this.size() || i<other.size(); i++) {
+            if (get(i)==null) {
+                if (other.get(i)!=null) return false;
+            }
+            else {
+                if ( ! get(i).equals(other.get(i))) return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int hashCode = 0;
+        int i = 0;
+        for (String value : values) {
+            i++;
+            if (value != null)
+                hashCode += value.hashCode() * i;
+        }
+        return hashCode;
+    }
+
+    @Override
+    public String toString() { return Arrays.toString(values); }
+
+    public boolean isEmpty() {
+        return this==empty;
+    }
+
+    private static boolean containsAllNulls(String[] values) {
+        for (String value : values)
+            if (value!=null) return false;
+        return true;
+    }
+
+    public Map<String,String> asContext(List<String> dimensions) {
+        Map<String,String> context=new HashMap<>();
+        if (dimensions==null) return context;
+        for (int i=0; i<dimensions.size(); i++) {
+            context.put(dimensions.get(i),get(i));
+        }
+        return context;
+    }
+
+    /** Returns the string at the given index, <b>or null if it has no value at this index.</b> */
+    public String get(int index) {
+        if (index>=values.length) return null;
+        return values[index];
+    }
+
+    /** Returns the number of values in this (some of which may be null) */
+    public int size() { return values.length; }
+
+    /** Returns  copy of the values in this in an array */
+    public String[] getValues() {
+        return Arrays.copyOf(values,values.length);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DumpTool.java b/container-search/src/main/java/com/yahoo/search/query/profile/DumpTool.java
new file mode 100644
index 00000000000..b9d631cdd10
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/DumpTool.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import java.io.File;
+import java.util.Map;
+
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.config.QueryProfileXMLReader;
+
+/**
+ * A standalone tool for dumping query profile properties
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DumpTool {
+
+    /** Creates and returns a dump from some parameters */
+    public String resolveAndDump(String... args) {
+        if (args.length==0 || args[0].startsWith("-")) {
+            StringBuilder result=new StringBuilder();
+            result.append("Dumps all resolved query profile properties for a set of dimension values\n");
+            result.append("USAGE: dump [query-profile] [dir]? [parameters]?\n");
+            result.append("  and [query-profile] is the name of the query profile to dump the values of\n");
+            result.append("  and [dir] is a path to an application package or query profile directory. Default: current dir\n");
+            result.append("  and   [parameters] is the http request encoded dimension keys used during resolving. Default: none\n");
+            result.append("Examples:\n");
+            result.append("  dump default\n");
+            result.append("  - dumps the 'default' profile non-variant values in the current dir\n");
+            result.append("  dump default x=x1&y=y1\n");
+            result.append("  - dumps the 'default' profile resolved with dimensions values x=x1 and y=y1 in the current dir\n");
+            result.append("  dump default myapppackage\n");
+            result.append("  - dumps the 'default' profile non-variant values in myapppackage/search/query-profiles\n");
+            result.append("  dump default dev/myprofiles x=x1&y=y1\n");
+            result.append("  - dumps the 'default' profile resolved with dimensions values x=x1 and y=y1 in dev/myprofiles\n");
+            return result.toString();
+        }
+
+        // Find what the arguments means
+        if (args.length>=3) {
+            return dump(args[0],args[1],args[2]);
+        }
+        else if (args.length==2) {
+            if (args[1].indexOf("=")>=0)
+                return dump(args[0],"",args[1]);
+            else
+                return dump(args[0],args[1],"");
+        }
+        else { // args.length=1
+            return dump(args[0],"","");
+        }
+    }
+
+    private String dump(String profileName,String dir,String parameters) {
+        // Import profiles
+        if (dir.isEmpty())
+            dir=".";
+        File dirInAppPackage=new File(dir,"search/query-profiles");
+        if (dirInAppPackage.exists())
+            dir=dirInAppPackage.getPath();
+        QueryProfileXMLReader reader = new QueryProfileXMLReader();
+        QueryProfileRegistry registry = reader.read(dir);
+        registry.freeze();
+
+        // Dump (through query to get wiring & parameter parsing done easily)
+        Query query = new Query("?" + parameters, registry.compile().findQueryProfile(profileName));
+        Map<String,Object> properties=query.properties().listProperties();
+
+        // Create result
+        StringBuilder b=new StringBuilder();
+        for (Map.Entry<String,Object> property : properties.entrySet()) {
+            b.append(property.getKey());
+            b.append("=");
+            b.append(property.getValue().toString());
+            b.append("\n");
+        }
+        return b.toString();
+    }
+
+    public static void main(String... args) {
+        try {
+            System.out.print(new DumpTool().resolveAndDump(args));
+        }
+        catch (Exception e) {
+            System.err.println(Exceptions.toMessageString(e));
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/FieldDescriptionQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/FieldDescriptionQueryProfileVisitor.java
new file mode 100644
index 00000000000..73c0fcd2cb1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/FieldDescriptionQueryProfileVisitor.java
@@ -0,0 +1,70 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.search.query.profile.types.FieldDescription;
+
+import java.util.List;
+
+/**
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+final class FieldDescriptionQueryProfileVisitor extends QueryProfileVisitor {
+
+    /** The result, or null if none */
+    private FieldDescription result = null;
+
+    private final List<String> name;
+
+    private int nameIndex=-1;
+
+    private boolean enteringContent=false;
+
+    public FieldDescriptionQueryProfileVisitor(List<String> name) {
+        this.name=name;
+    }
+
+    @Override
+    public String getLocalKey() {
+        return name.get(nameIndex);
+    }
+
+    @Override
+    public boolean enter(String name) {
+        if (nameIndex+2<this.name.size()) {
+            nameIndex++;
+            enteringContent=true;
+        }
+        else {
+            enteringContent=false;
+        }
+        return enteringContent;
+    }
+
+    @Override
+    public void leave(String name) {
+        nameIndex--;
+    }
+
+    @Override
+    public void onValue(String name,Object value, DimensionBinding binding, QueryProfile owner) {
+    }
+
+    @Override
+    public void onQueryProfile(QueryProfile profile, DimensionBinding binding, QueryProfile owner) {
+        if (enteringContent) return; // not at leaf query profile
+        if (profile.getType() == null) return;
+        result = profile.getType().getField(name.get(name.size()-1));
+    }
+
+    @Override
+    public boolean isDone() {
+        return result != null;
+    }
+
+    public FieldDescription result() { return result; }
+
+    @Override
+    public String toString() {
+        return "a query profile type visitor (hash " + hashCode() + ") with current value " + result;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java
new file mode 100644
index 00000000000..242c551f876
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.properties.PropertyMap;
+
+/**
+ * A map which stores all types which cannot be stored in a query profile
+ * that is rich model objects.
+ * <p>
+ * This map will deep copy not only the model object map, but also each
+ * clonable member in the map.
+ *
+ * @author bratseth
+ */
+public class ModelObjectMap extends PropertyMap {
+
+    /** Returns true if the class of the value is not acceptable as a query profile value */
+    @Override
+    protected boolean shouldSet(CompoundName name,Object value) {
+        if (value==null) return true;
+        return FieldType.fromClass(value.getClass())==null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java
new file mode 100644
index 00000000000..5d0bffa1ea8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/OverridableQueryProfile.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+/**
+ * A regular query profile which knows it is storing overrides (not configured profiles)
+ * and that implements override legality checking.
+ *
+ * @author bratseth
+ */
+public class OverridableQueryProfile extends QueryProfile {
+
+    private static final String simpleClassName = OverridableQueryProfile.class.getSimpleName();
+
+    /** Creates an unbacked overridable query profile */
+    protected OverridableQueryProfile() {
+        super(ComponentId.createAnonymousComponentId(simpleClassName));
+    }
+
+    @Override
+    protected Object checkAndConvertAssignment(String localName, Object inputValue, QueryProfileRegistry registry) {
+        Object value=super.checkAndConvertAssignment(localName, inputValue, registry);
+        if (value!=null && value.getClass() == QueryProfile.class) { // We are assigning a query profile - make it overridable
+            return new BackedOverridableQueryProfile((QueryProfile)value);
+        }
+        return value;
+    }
+
+    @Override
+    protected QueryProfile createSubProfile(String name,DimensionBinding binding) {
+        return new OverridableQueryProfile(); // Nothing is set in this branch, so nothing to override, but need override checking
+    }
+
+    /** Returns a clone of this which can be independently overridden */
+    @Override
+    public OverridableQueryProfile clone() {
+        if (isFrozen()) return this;
+        OverridableQueryProfile clone=(OverridableQueryProfile)super.clone();
+        clone.initId(ComponentId.createAnonymousComponentId(simpleClassName));
+        return clone;
+    }
+
+    @Override
+    public String toString() {
+        return "an overridable query profile with no backing";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java
new file mode 100644
index 00000000000..2a22d58d8b7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/PrefixQueryProfileVisitor.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+
+/**
+ * A query profile visitor which keeps track of name prefixes and can skip values outside a given prefix
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+abstract class PrefixQueryProfileVisitor extends QueryProfileVisitor {
+
+    /** Only call onValue/onQueryProfile for nodes having this prefix */
+    private final CompoundName prefix;
+
+    /** The current prefix, relative to prefix. */
+    protected CompoundName currentPrefix = CompoundName.empty;
+
+    private int prefixComponentIndex = -1;
+
+    public PrefixQueryProfileVisitor(CompoundName prefix) {
+        if (prefix == null)
+            prefix = CompoundName.empty;
+        this.prefix = prefix;
+    }
+
+    @Override
+    public final void onQueryProfile(QueryProfile profile, DimensionBinding binding, QueryProfile owner) {
+        if (prefixComponentIndex < prefix.size()) return; // Not in the prefix yet
+        onQueryProfileInsidePrefix(profile, binding, owner);
+    }
+
+    protected abstract void onQueryProfileInsidePrefix(QueryProfile profile, DimensionBinding binding, QueryProfile owner);
+
+    @Override
+    public final boolean enter(String name) {
+        prefixComponentIndex++;
+        if (prefixComponentIndex-1 < prefix.size()) return true; // we're in the given prefix, which should not be included in the name
+        currentPrefix = currentPrefix.append(name);
+        return true;
+    }
+
+    @Override
+    public final void leave(String name) {
+        prefixComponentIndex--;
+        if (prefixComponentIndex < prefix.size()) return; // we're in the given prefix, which should not be included in the name
+        if ( ! name.isEmpty() && ! currentPrefix.isEmpty())
+            currentPrefix = currentPrefix.first(currentPrefix.size() - 1);
+    }
+
+    /**
+     * Returns the correct prefix component if we are still going down the prefix path,
+     * or null to get all if we are inside the prefix
+     */
+    @Override
+    public String getLocalKey() {
+        if (prefixComponentIndex < prefix.size())
+            return prefix.get(prefixComponentIndex);
+        else
+            return null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java
new file mode 100644
index 00000000000..55210717305
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java
@@ -0,0 +1,835 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.FreezableSimpleComponent;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.processing.request.Properties;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileFieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A query profile is a data container with an id and a class (type). More precisely, it contains
+ * <ul>
+ *   <li>An id, on the form name:version, where the version is optional, and follows the same rules as for other search container components.
+ *   <li>A class id referring to the class defining this profile (see Query Profile Classes below)
+ *   <li>A (possibly empty) list of ids of inherited query profiles
+ *   <li>A (possibly empty) list of declarative predicates over search request parameters which defines when this query profile is applicable (see Query Profile Selection below)
+ *   <li>The data content, which consists of
+ *       <ul>
+ *         <li>named values
+ *         <li>named references to other profiles
+ *       </ul>
+ * </ul>
+ *
+ * This serves the purpose of an intermediate format between configuration and runtime structures - the runtime
+ * structure used is QueryProfileProperties.
+ *
+ * @author bratseth
+ */
+public class QueryProfile extends FreezableSimpleComponent implements Cloneable {
+
+    /** Defines the permissible content of this, or null if any content is permissible */
+    private QueryProfileType type=null;
+
+    /** The value at this query profile - allows non-fields to have values, e.g a=value1, a.b=value2 */
+    private Object value=null;
+
+    /** The variants of this, or null if none */
+    private QueryProfileVariants variants=null;
+
+    /** The resolved variant dimensions of this, or null if none or not resolved yet (is resolved at freeze) */
+    private List<String> resolvedDimensions=null;
+
+    /** The query profiles inherited by this, or null if none */
+    private List<QueryProfile> inherited=null;
+
+    /** The content of this profile. The values may be primitives, substitutable strings or other query profiles */
+    private CopyOnWriteContent content=new CopyOnWriteContent();
+
+    /**
+     * Field override settings: fieldName→OverrideValue. These overrides the override
+     * setting in the type (if any) of this field). If there are no query profile level settings, this is null.
+     */
+    private Map<String,Boolean> overridable=null;
+
+    /**
+     * Creates a new query profile from an id.
+     * The query profile can be modified freely (but not accessed) until it is {@link #freeze frozen}.
+     * At that point it becomes readable but unmodifiable, which it stays until it goes out of reference.
+     */
+    public QueryProfile(ComponentId id) {
+        super(id);
+        if ( ! id.isAnonymous())
+            validateName(id.getName());
+    }
+
+    /** Convenience shorthand for new QueryProfile(new ComponentId(idString)) */
+    public QueryProfile(String idString) {
+        this(new ComponentId(idString));
+    }
+
+    // ----------------- Public API -------------------------------------------------------------------------------
+
+    // ----------------- Setters and getters
+
+    /** Returns the type of this or null if it has no type */
+    public QueryProfileType getType() { return type; }
+
+    /** Sets the type of this, or set to null to not use any type checking in this profile */
+    public void setType(QueryProfileType type) { this.type=type; }
+
+    /** Returns the virtual variants of this, or null if none */
+    public QueryProfileVariants getVariants() { return variants; }
+
+    /**
+     * Returns the list of profiles inherited by this.
+     * Note that order matters for inherited profiles - variables are resolved depth first in the order found in
+     * the inherited list. This always returns an unmodifiable list - use addInherited to add.
+     */
+    public List<QueryProfile> inherited() {
+        if (isFrozen()) return inherited; // Frozen profiles always have an unmodifiable, non-null list
+        if (inherited==null) return Collections.emptyList();
+        return Collections.unmodifiableList(inherited);
+    }
+
+    /** Adds a profile to the end of the inherited list of this. Throws an exception if this is frozen. */
+    public void addInherited(QueryProfile profile) {
+        addInherited(profile,(DimensionValues)null);
+    }
+
+    public final void addInherited(QueryProfile profile,String[] dimensionValues) {
+        addInherited(profile,DimensionValues.createFrom(dimensionValues));
+    }
+
+    /** Adds a profile to the end of the inherited list of this for the given variant. Throws an exception if this is frozen. */
+    public void addInherited(QueryProfile profile, DimensionValues dimensionValues) {
+        ensureNotFrozen();
+
+        DimensionBinding dimensionBinding=DimensionBinding.createFrom(getDimensions(),dimensionValues);
+        if (dimensionBinding.isNull()) {
+            if (inherited==null)
+                inherited=new ArrayList<>();
+            inherited.add(profile);
+        }
+        else {
+            if (variants==null)
+                variants=new QueryProfileVariants(dimensionBinding.getDimensions(), this);
+            variants.inherit(profile,dimensionBinding.getValues());
+        }
+    }
+
+    /**
+     * Returns the content fields declared in this (i.e not including those inherited) as a read-only map.
+     * @throws IllegalStateException if this is frozen
+     */
+    public Map<String,Object> declaredContent() {
+        ensureNotFrozen();
+        return content.unmodifiableMap();
+    }
+
+    /**
+     * Returns if the given field is declared explicitly as overridable or not in this or any <i>nested</i> profiles
+     * (i.e not including overridable settings <i>inherited</i> and from <i>types</i>).
+     *
+     * @param  name the (possibly dotted) field name to return
+     * @param  context the context in which the name is resolved, or null if none
+     * @return true/false if this is declared overridable/not overridable in this instance, null if it is not
+     *         given any value is <i>this</i> profile instance
+     * @throws IllegalStateException if this is frozen
+     */
+    public Boolean isDeclaredOverridable(String name, Map<String,String> context) {
+        return isDeclaredOverridable(new CompoundName(name),DimensionBinding.createFrom(getDimensions(),context));
+    }
+
+    /** Sets the dimensions over which this may vary. Note: This will erase any currently defined variants */
+    public void setDimensions(String[] dimensions) {
+        ensureNotFrozen();
+        variants=new QueryProfileVariants(dimensions, this);
+    }
+
+    /** Returns the value set at this node, to allow non-leafs to have values. Returns null if none. */
+    public Object getValue() { return value; }
+
+    public void setValue(Object value) {
+        ensureNotFrozen();
+        this.value=value;
+    }
+
+    /** Returns the variant dimensions to be used in this - an unmodifiable list of dimension names */
+    public List<String> getDimensions() {
+        if (isFrozen()) return resolvedDimensions;
+        if (variants!=null) return variants.getDimensions();
+        if (inherited==null) return null;
+        for (QueryProfile inheritedProfile : inherited) {
+            List<String> inheritedDimensions=inheritedProfile.getDimensions();
+            if (inheritedDimensions!=null) return inheritedDimensions;
+        }
+        return null;
+    }
+
+    // ----------------- Query profile facade API
+
+    /**
+     * Sets the overridability of a field in this profile,
+     * this overrides the corresponding setting in the type (if any)
+     */
+    public final void setOverridable(String fieldName, boolean overridable, Map<String,String> context) {
+        setOverridable(new CompoundName(fieldName), overridable,DimensionBinding.createFrom(getDimensions(), context));
+    }
+
+    /**
+     * Return all objects that start with the given prefix path using no context. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public final Map<String, Object> listValues(String prefix) { return listValues(new CompoundName(prefix)); }
+
+    /**
+     * Return all objects that start with the given prefix path using no context. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, null); }
+
+    /**
+     * Return all objects that start with the given prefix path. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public final Map<String, Object> listValues(String prefix, Map<String,String> context) {
+        return listValues(new CompoundName(prefix), context);
+    }
+
+    /**
+     * Return all objects that start with the given prefix path. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public final Map<String, Object> listValues(CompoundName prefix, Map<String,String> context) {
+        return listValues(prefix, context, null);
+    }
+
+    /**
+     * Adds all objects that start with the given path prefix to the given value map. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public Map<String, Object> listValues(CompoundName prefix, Map<String, String> context, Properties substitution) {
+        DimensionBinding dimensionBinding=DimensionBinding.createFrom(getDimensions(),context);
+
+        AllValuesQueryProfileVisitor visitor=new AllValuesQueryProfileVisitor(prefix);
+        accept(visitor,dimensionBinding, null);
+        Map<String,Object>  values=visitor.getResult();
+
+        if (substitution==null) return values;
+        for (Map.Entry<String,Object> entry : values.entrySet()) {
+            if (entry.getValue().getClass()==String.class) continue; // Shortcut
+            if (entry.getValue() instanceof SubstituteString)
+                entry.setValue(((SubstituteString)entry.getValue()).substitute(context,substitution));
+        }
+        return values;
+    }
+
+    /**
+     * Lists types reachable from this, indexed by the prefix having that type.
+     * If this is itself typed, this' type will be included with an empty prefix
+     */
+    Map<CompoundName, QueryProfileType> listTypes(CompoundName prefix, Map<String, String> context) {
+        DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(), context);
+        AllTypesQueryProfileVisitor visitor = new AllTypesQueryProfileVisitor(prefix);
+        accept(visitor, dimensionBinding, null);
+        return visitor.getResult();
+    }
+
+    /**
+     * Lists references reachable from this.
+     */
+    Set<CompoundName> listReferences(CompoundName prefix, Map<String, String> context) {
+        DimensionBinding dimensionBinding=DimensionBinding.createFrom(getDimensions(),context);
+        AllReferencesQueryProfileVisitor visitor=new AllReferencesQueryProfileVisitor(prefix);
+        accept(visitor,dimensionBinding,null);
+        return visitor.getResult();
+    }
+
+    /**
+     * Lists every entry (value or reference) reachable from this which is not overridable
+     */
+    Set<CompoundName> listUnoverridable(CompoundName prefix, Map<String, String> context) {
+        DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(),context);
+        AllUnoverridableQueryProfileVisitor visitor = new AllUnoverridableQueryProfileVisitor(prefix);
+        accept(visitor, dimensionBinding, null);
+        return visitor.getResult();
+    }
+
+    /**
+     * Returns a value from this query profile by resolving the given name:
+     * <ul>
+     *   <li>The name up to the first dot is the value looked up in the value of this profile
+     *   <li>The rest of the name (if any) is used as the name to look up in the referenced query profile
+     * </ul>
+     *
+     * If this name does not resolve <i>completely</i> into a value in this or any inherited profile, null is returned.
+     */
+    public final Object get(String name) { return get(name,(Map<String,String>)null); }
+
+    /** Returns a value from this using the given property context for resolution and using this for substitution */
+    public final Object get(String name, Map<String,String> context) {
+        return get(name,context,null);
+    }
+
+    /** Returns a value from this using the given dimensions for resolution */
+    public final Object get(String name, String[] dimensionValues) {
+        return get(name,dimensionValues,null);
+    }
+
+    public final Object get(String name, String[] dimensionValues, Properties substitution) {
+        return get(name,DimensionValues.createFrom(dimensionValues),substitution);
+    }
+
+    /** Returns a value from this using the given dimensions for resolution */
+    public final Object get(String name, DimensionValues dimensionValues, Properties substitution) {
+        return get(name,DimensionBinding.createFrom(getDimensions(),dimensionValues),substitution);
+    }
+
+    public final Object get(String name, Map<String,String> context, Properties substitution) {
+        return get(name,DimensionBinding.createFrom(getDimensions(),context),substitution);
+    }
+
+    public final Object get(CompoundName name, Map<String,String> context, Properties substitution) {
+        return get(name,DimensionBinding.createFrom(getDimensions(),context),substitution);
+    }
+
+    final Object get(String name, DimensionBinding binding,Properties substitution) {
+        return get(new CompoundName(name),binding,substitution);
+    }
+
+    final Object get(CompoundName name, DimensionBinding binding, Properties substitution) {
+        Object node=get(name,binding);
+        if (node!=null && node.getClass()==String.class) return node; // Shortcut
+        if (node instanceof SubstituteString) return ((SubstituteString)node).substitute(binding.getContext(),substitution);
+        return node;
+    }
+
+    final Object get(CompoundName name,DimensionBinding dimensionBinding) {
+        return lookup(name,false,dimensionBinding);
+    }
+
+    /**
+     * Returns the node at the position prescribed by the given name (without doing substitutions) -
+     * a primitive value, a substitutable string, a query profile, or null if not found.
+     */
+    public final Object lookup(String name, Map<String,String> context) {
+        return lookup(new CompoundName(name),true,DimensionBinding.createFrom(getDimensions(),context));
+    }
+
+    /** Sets a value in this or any nested profile using null as context */
+    public final void set(String name, Object value, QueryProfileRegistry registry) {
+        set(name,value,(Map<String,String>)null, registry);
+    }
+
+    /**
+     * Sets a value in this or any nested profile. Any missing structure needed to set this will be created.
+     * If this value is already set, this will overwrite the previous value.
+     *
+     * @param name the name of the field, possibly a dotted name which will cause setting of a variable in a subprofile
+     * @param value the value to assign to the name, a primitive wrapper, string or a query profile
+     * @param context the context used to resolve where this value should be set, or null if none
+     * @throws IllegalArgumentException if the given name is illegal given the types of this or any nested query profile
+     * @throws IllegalStateException if this query profile is frozen
+     */
+    public final void set(CompoundName name,Object value,Map<String,String> context, QueryProfileRegistry registry) {
+        set(name, value, DimensionBinding.createFrom(getDimensions(), context), registry);
+    }
+
+    public final void set(String name,Object value,Map<String,String> context, QueryProfileRegistry registry) {
+        set(new CompoundName(name), value, DimensionBinding.createFrom(getDimensions(), context), registry);
+    }
+
+    public final void set(String name,Object value,String[] dimensionValues, QueryProfileRegistry registry) {
+        set(name,value,DimensionValues.createFrom(dimensionValues), registry);
+    }
+
+    /**
+     * Sets a value in this or any nested profile. Any missing structure needed to set this will be created.
+     * If this value is already set, this will overwrite the previous value.
+     *
+     * @param name the name of the field, possibly a dotted name which will cause setting of a variable in a subprofile
+     * @param value the value to assign to the name, a primitive wrapper, string or a query profile
+     * @param dimensionValues the dimension values - will be matched by order to the dimensions set in this - if this is
+     *        shorter or longer than the number of dimensions it will be adjusted as needed
+     * @param registry the registry used to resolve query profile references. If null is passed query profile references
+     *        will cause an exception
+     * @throws IllegalArgumentException if the given name is illegal given the types of this or any nested query profile
+     * @throws IllegalStateException if this query profile is frozen
+     */
+    public final void set(String name,Object value,DimensionValues dimensionValues, QueryProfileRegistry registry) {
+        set(new CompoundName(name), value, DimensionBinding.createFrom(getDimensions(), dimensionValues), registry);
+    }
+
+    // ----------------- Misc
+
+    public boolean isExplicit() {
+        return !getId().isAnonymous();
+    }
+
+    /**
+     * Switches this from write-only to read-only mode.
+     * This profile can never be modified again after this method returns.
+     * Calling this on an already frozen profile has no effect.
+     * <p>
+     * Calling this will also freeze any profiles inherited and referenced by this.
+     */
+    // TODO: Remove/simplify as query profiles are not used at query time
+    public synchronized void freeze() {
+        if (isFrozen()) return;
+
+        resolvedDimensions=getDimensions();
+
+        if (variants !=null)
+            variants.freeze();
+
+        if (inherited!=null) {
+            for (QueryProfile inheritedProfile : inherited)
+                inheritedProfile.freeze();
+        }
+
+        content.freeze();
+
+        inherited= inherited==null ? ImmutableList.of() : ImmutableList.copyOf(inherited);
+
+        super.freeze();
+    }
+
+    @Override
+    public String toString() {
+        return "query profile '" + getId()  + "'" + (type!=null ? " of type '" + type.getId() + "'" : "");
+    }
+
+    /**
+     * Returns a clone of this. The clone will not be frozen and will contain copied inherited and content collections
+     * pointing to the same values as this.
+     */
+    @Override
+    public QueryProfile clone() {
+        if (isFrozen()) return this;
+        QueryProfile clone=(QueryProfile)super.clone();
+        if (variants !=null)
+            clone.variants = variants.clone();
+        if (inherited!=null)
+            clone.inherited=new ArrayList<>(inherited);
+
+        if (this.content!=null)
+            clone.content=content.clone();
+
+        return clone;
+    }
+
+    /**
+     * Clones a value of a type which may appear in a query profile if cloning is necessary (i.e if it is
+     * not immutable). Returns the input type otherwise.
+     */
+    static Object cloneIfNecessary(Object object) {
+        if (object instanceof QueryProfile) return ((QueryProfile)object).clone();
+        return object; // Other types are immutable
+    }
+
+    /** Throws IllegalArgumentException if the given string is not a valid query profile name */
+    public static void validateName(String name) {
+        Matcher nameMatcher=namePattern.matcher(name);
+        if ( ! nameMatcher.matches())
+            throw new IllegalArgumentException("Illegal name '" + name + "'");
+    }
+
+    // ----------------- For subclass use --------------------------------------------------------------------
+
+    /** Override this to intercept all writes to this profile (or any nested profiles) */
+    protected void set(CompoundName name, Object value, DimensionBinding binding, QueryProfileRegistry registry) {
+        try {
+            setNode(name, value, null, binding, registry);
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'",e);
+        }
+    }
+
+    /** Returns this value, or its corresponding substitution string if it contains substitutions */
+    protected Object convertToSubstitutionString(Object value) {
+        if (value==null) return value;
+        if (value.getClass()!=String.class) return value;
+        SubstituteString substituteString=SubstituteString.create((String)value);
+        if (substituteString==null) return value;
+        return substituteString;
+    }
+
+    /** Returns the field description of this field, or null if it is not typed */
+    protected FieldDescription getFieldDescription(CompoundName name, DimensionBinding binding) {
+        FieldDescriptionQueryProfileVisitor visitor=new FieldDescriptionQueryProfileVisitor(name.asList());
+        accept(visitor, binding,null);
+        return visitor.result();
+    }
+
+    /**
+     * Returns true if this value is definitely overridable in this (set and not unoverridable),
+     * false if it is declared unoverridable (in instance or type), and null if this profile has no
+     * opinion on the matter because the value is not set in this.
+     */
+    Boolean isLocalOverridable(String localName,DimensionBinding binding) {
+        if (localLookup(localName, binding)==null) return null; // Not set
+        Boolean isLocalInstanceOverridable=isLocalInstanceOverridable(localName);
+        if (isLocalInstanceOverridable!=null)
+            return isLocalInstanceOverridable.booleanValue();
+        if (type!=null) return type.isOverridable(localName);
+        return true;
+    }
+
+    protected Boolean isLocalInstanceOverridable(String localName) {
+        if (overridable==null) return null;
+        return overridable.get(localName);
+    }
+
+    protected Object lookup(CompoundName name,boolean allowQueryProfileResult, DimensionBinding dimensionBinding) {
+        SingleValueQueryProfileVisitor visitor=new SingleValueQueryProfileVisitor(name.asList(),allowQueryProfileResult);
+        accept(visitor,dimensionBinding,null);
+        return visitor.getResult();
+    }
+
+    protected final void accept(QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) {
+        acceptAndEnter("", visitor, dimensionBinding, owner);
+    }
+
+    void acceptAndEnter(String key, QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) {
+        boolean allowContent=visitor.enter(key);
+        accept(allowContent, visitor, dimensionBinding, owner);
+        if (allowContent)
+            visitor.leave(key);
+    }
+
+    /**
+     * Visit the profiles and values referenced from this in order of decreasing precedence
+     *
+     * @param allowContent whether content in this should be visited
+     * @param visitor the visitor
+     * @param dimensionBinding the dimension binding to use
+     */
+    final void accept(boolean allowContent,QueryProfileVisitor visitor, DimensionBinding dimensionBinding, QueryProfile owner) {
+        visitor.onQueryProfile(this, dimensionBinding, owner);
+        if (visitor.isDone()) return;
+
+        visitVariants(allowContent,visitor,dimensionBinding);
+        if (visitor.isDone()) return;
+
+        if (allowContent) {
+            visitContent(visitor,dimensionBinding);
+            if (visitor.isDone()) return;
+        }
+
+        if (visitor.visitInherited())
+            visitInherited(allowContent, visitor, dimensionBinding, owner);
+    }
+
+    protected void visitVariants(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) {
+        if (getVariants()!=null)
+            getVariants().accept(allowContent, getType(), visitor, dimensionBinding);
+    }
+
+    protected void visitInherited(boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding, QueryProfile owner) {
+        if (inherited==null) return;
+        for (QueryProfile inheritedProfile : inherited) {
+            inheritedProfile.accept(allowContent,visitor,dimensionBinding.createFor(inheritedProfile.getDimensions()), owner);
+            if (visitor.isDone()) return;
+        }
+    }
+
+    private void visitContent(QueryProfileVisitor visitor,DimensionBinding dimensionBinding) {
+        String contentKey=visitor.getLocalKey();
+
+        // Visit this' content
+        if (contentKey!=null) { // Get only the content of the current key
+            if (type!=null)
+                contentKey=type.unalias(contentKey);
+            visitor.acceptValue(contentKey, getContent(contentKey), dimensionBinding, this);
+        }
+        else { // get all content in this
+            for (Map.Entry<String,Object> entry : getContent().entrySet()) {
+                visitor.acceptValue(entry.getKey(), entry.getValue(), dimensionBinding, this);
+                if (visitor.isDone()) return;
+            }
+        }
+    }
+
+    /** Returns a value from the content of this, or null if not present */
+    protected Object getContent(String key) {
+        return content.get(key);
+    }
+
+    /** Returns all the content from this as an unmodifiable map */
+    protected Map<String,Object> getContent() {
+        return content.unmodifiableMap();
+    }
+
+    /** Sets the value of a node in <i>this</i> profile - the local name given must not be nested (contain dots) */
+    protected QueryProfile setLocalNode(String localName, Object value,QueryProfileType parentType,
+                                        DimensionBinding dimensionBinding, QueryProfileRegistry registry) {
+        if (parentType!=null && type==null && !isFrozen())
+            type=parentType;
+
+        value=checkAndConvertAssignment(localName, value, registry);
+        localPut(localName,value,dimensionBinding);
+        return this;
+    }
+
+    /**
+     * Combines an existing and a new value for a query property key.
+     * Return the new object to add to the state of the owning profile (/variant), or null if no new value needs to
+     * be added (usually because the new value was added to the existing).
+     */
+    static Object combineValues(Object newValue, Object existingValue) {
+        if (newValue instanceof QueryProfile) {
+            QueryProfile newProfile=(QueryProfile)newValue;
+            if ( existingValue==null || ! (existingValue instanceof QueryProfile)) {
+                if (!isModifiable(newProfile))
+                    newProfile=new BackedOverridableQueryProfile(newProfile); // Make the query profile reference overridable
+                newProfile.value=existingValue;
+                return newProfile;
+            }
+
+            // if both are profiles:
+            return combineProfiles(newProfile,(QueryProfile)existingValue);
+        }
+        else {
+            if (existingValue instanceof QueryProfile) { // we need to set a non-leaf value on a query profile
+                QueryProfile existingProfile=(QueryProfile)existingValue;
+                if (isModifiable(existingProfile)) {
+                    existingProfile.setValue(newValue);
+                    return null;
+                }
+                else {
+                    QueryProfile existingOverridable = new BackedOverridableQueryProfile((QueryProfile)existingValue);
+                    existingOverridable.setValue(newValue);
+                    return existingOverridable;
+                }
+            }
+            else {
+                return newValue;
+            }
+        }
+    }
+
+    private static QueryProfile combineProfiles(QueryProfile newProfile,QueryProfile existingProfile) {
+        QueryProfile returnValue=null;
+        QueryProfile existingModifiable;
+
+        // Ensure the existing profile is modifiable
+        if (existingProfile.getClass()==QueryProfile.class) {
+            existingModifiable = new BackedOverridableQueryProfile(existingProfile);
+            returnValue=existingModifiable;
+        }
+        else { // is an overridable wrapper
+            existingModifiable=existingProfile; // May be used as-is
+        }
+
+        // Make the existing profile inherit the new one
+        if (existingModifiable instanceof BackedOverridableQueryProfile)
+            ((BackedOverridableQueryProfile)existingModifiable).addInheritedHere(newProfile);
+        else
+            existingModifiable.addInherited(newProfile);
+
+        // Remove content from the existing which the new one does not allow overrides of
+        if (existingModifiable.content!=null) {
+            for (String key : existingModifiable.content.unmodifiableMap().keySet()) {
+                if ( ! newProfile.isLocalOverridable(key, null)) {
+                    existingModifiable.content.remove(key);
+                }
+            }
+        }
+
+        return returnValue;
+    }
+
+    /** Returns whether the given profile may be modified from this profile */
+    private static boolean isModifiable(QueryProfile profile) {
+        if (profile.isFrozen()) return false;
+        if ( ! profile.isExplicit()) return true; // Implicitly defined from this - ok to modify then
+        if (! (profile instanceof BackedOverridableQueryProfile)) return false;
+        return true;
+    }
+
+    /**
+     * Converts to the type of the receiving field, if possible and necessary.
+     *
+     * @return the value to be assigned: the original or a converted value
+     * @throws IllegalArgumentException if the assignment is illegal
+     */
+    protected Object checkAndConvertAssignment(String localName, Object value, QueryProfileRegistry registry) {
+        if (type==null) return value; // no type checking
+
+        FieldDescription fieldDescription=type.getField(localName);
+        if (fieldDescription==null) {
+            if (type.isStrict())
+                throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict");
+            return value;
+        }
+
+        if (registry == null && (fieldDescription.getType() instanceof QueryProfileFieldType))
+            throw new IllegalArgumentException("A registry was not passed: Query profile references is not supported");
+        Object convertedValue = fieldDescription.getType().convertFrom(value, registry);
+        if (convertedValue == null)
+            throw new IllegalArgumentException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription());
+        return convertedValue;
+    }
+
+    /**
+     * Looks up all inherited profiles and adds any that matches this name.
+     * This default implementation returns an empty profile.
+     */
+    protected QueryProfile createSubProfile(String name,DimensionBinding dimensionBinding) {
+        QueryProfile queryProfile = new QueryProfile(ComponentId.createAnonymousComponentId(name));
+        return queryProfile;
+    }
+
+    /** Do a variant-aware content lookup in this */
+    protected Object localLookup(String name, DimensionBinding dimensionBinding) {
+        Object node=null;
+        if ( variants!=null && !dimensionBinding.isNull())
+            node=variants.get(name,type,true,dimensionBinding);
+        if (node==null)
+            node=content==null ? null : content.get(name);
+        return node;
+    }
+
+    // ----------------- Private  ----------------------------------------------------------------------------------
+
+    private Boolean isDeclaredOverridable(CompoundName name,DimensionBinding dimensionBinding) {
+        QueryProfile parent= lookupParentExact(name, true, dimensionBinding);
+        if (parent.overridable==null) return null;
+        return parent.overridable.get(name.last());
+    }
+
+    /**
+     * Sets the overridability of a field in this profile,
+     * this overrides the corresponding setting in the type (if any)
+     */
+    private void setOverridable(CompoundName fieldName,boolean overridable,DimensionBinding dimensionBinding) {
+        QueryProfile parent= lookupParentExact(fieldName, true, dimensionBinding);
+        if (parent.overridable==null)
+            parent.overridable=new HashMap<>();
+        parent.overridable.put(fieldName.last(),overridable);
+    }
+
+    /** Sets a value to a (possibly non-local) node. The parent query profile holding the value is returned */
+    private void setNode(CompoundName name, Object value, QueryProfileType parentType,
+                         DimensionBinding dimensionBinding, QueryProfileRegistry registry) {
+        ensureNotFrozen();
+        if (name.isCompound()) {
+            QueryProfile parent= getQueryProfileExact(name.first(), true, dimensionBinding);
+            parent.setNode(name.rest(), value,parentType, dimensionBinding.createFor(parent.getDimensions()), registry);
+        }
+        else {
+            setLocalNode(name.toString(), value,parentType, dimensionBinding, registry);
+        }
+    }
+
+    /**
+     * Looks up and, if necessary, creates, the query profile which should hold the given local name portion of the
+     * given name. If the name contains no dots, this is returned.
+     *
+     * @param name the name of the variable to lookup the parent of
+     * @param create whether or not to create the parent if it is not present
+     * @return the parent, or null if not present and created is false
+     */
+    private QueryProfile lookupParentExact(CompoundName name, boolean create, DimensionBinding dimensionBinding) {
+        CompoundName rest=name.rest();
+        if (rest.isEmpty()) return this;
+
+        QueryProfile topmostParent= getQueryProfileExact(name.first(), create, dimensionBinding);
+        if (topmostParent==null) return null;
+        return topmostParent.lookupParentExact(rest, create, dimensionBinding.createFor(topmostParent.getDimensions()));
+    }
+
+    /**
+     * Returns a query profile from this by name
+     *
+     * @param localName the local name of the profile in this, this is never a compound
+     * @param create whether the profile should be created if missing
+     * @return the created profile, or null if not present, and create is false
+     */
+    private QueryProfile getQueryProfileExact(String localName, boolean create, DimensionBinding dimensionBinding) {
+        Object node=localExactLookup(localName, dimensionBinding);
+        if (node!=null && node instanceof QueryProfile) {
+            return (QueryProfile)node;
+        }
+        if (!create) return null;
+
+        QueryProfile queryProfile=createSubProfile(localName,dimensionBinding);
+        if (type!=null) {
+            Class<?> legalClass=type.getValueClass(localName);
+            if (legalClass==null || ! legalClass.isInstance(queryProfile))
+                throw new RuntimeException("'" + localName + "' is not a legal query profile reference name in " + this);
+            queryProfile.setType(type.getType(localName));
+        }
+        localPut(localName,queryProfile,dimensionBinding);
+        return queryProfile;
+    }
+
+    /** Do a variant-aware content lookup in this - without looking in any wrapped content. But by matching variant bindings exactly only */
+    private Object localExactLookup(String name,DimensionBinding dimensionBinding) {
+        if (dimensionBinding.isNull()) return content==null ? null : content.get(name);
+        if (variants==null) return null;
+        QueryProfileVariant variant=variants.getVariant(dimensionBinding.getValues(),false);
+        if (variant==null) return null;
+        return variant.values().get(name);
+    }
+
+    /** Sets a value directly in this query profile (unless frozen) */
+    private void localPut(String localName,Object value,DimensionBinding dimensionBinding) {
+        ensureNotFrozen();
+
+        if (type!=null)
+            localName=type.unalias(localName);
+
+        validateName(localName);
+        value=convertToSubstitutionString(value);
+
+        if (dimensionBinding.isNull()) {
+            Object combinedValue;
+            if (value instanceof QueryProfile)
+                combinedValue = combineValues(value,content==null ? null : content.get(localName));
+            else
+                combinedValue = combineValues(value, localLookup(localName, dimensionBinding));
+
+            if (combinedValue!=null)
+                content.put(localName,combinedValue);
+        }
+        else {
+            if (variants==null)
+                variants=new QueryProfileVariants(dimensionBinding.getDimensions(), this);
+            variants.set(localName,dimensionBinding.getValues(),value);
+        }
+    }
+
+    private static final Pattern namePattern=Pattern.compile("[$a-zA-Z_/][-$a-zA-Z0-9_/()]*");
+
+    /**
+     * Returns a compiled version of this which produces faster lookup times
+     *
+     * @param registry the registry this will be added to by the caller, or null if none
+     */
+    public CompiledQueryProfile compile(CompiledQueryProfileRegistry registry) {
+        return QueryProfileCompiler.compile(this, registry);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java
new file mode 100644
index 00000000000..795c7655dfb
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.DimensionalMap;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Compile a set of query profiles into compiled profiles.
+ *
+ * @author bratseth
+ */
+public class QueryProfileCompiler {
+
+    private static final Logger log = Logger.getLogger(QueryProfileCompiler.class.getName());
+
+    public static CompiledQueryProfileRegistry compile(QueryProfileRegistry input) {
+        CompiledQueryProfileRegistry output = new CompiledQueryProfileRegistry(input.getTypeRegistry());
+        for (QueryProfile inputProfile : input.allComponents()) {
+            output.register(compile(inputProfile, output));
+        }
+        return output;
+    }
+
+    public static CompiledQueryProfile compile(QueryProfile in, CompiledQueryProfileRegistry registry) {
+        DimensionalMap.Builder<CompoundName, Object> values = new DimensionalMap.Builder<>();
+        DimensionalMap.Builder<CompoundName, QueryProfileType> types = new DimensionalMap.Builder<>();
+        DimensionalMap.Builder<CompoundName, Object> references = new DimensionalMap.Builder<>();
+        DimensionalMap.Builder<CompoundName, Object> unoverridables = new DimensionalMap.Builder<>();
+
+        // Resolve values for each existing variant and combine into a single data structure
+        Set<DimensionBindingForPath> variants = new HashSet<>();
+        collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding, variants);
+        variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants
+        if (log.isLoggable(Level.FINE))
+            log.fine("Compiling " + in.toString() + " having " + variants.size() + " variants");
+        int i = 0;
+        for (DimensionBindingForPath variant : variants) {
+            if (log.isLoggable(Level.FINER))
+                log.finer("    Compiling variant " + i++ + ": " + variant);
+            for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet())
+                values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue());
+            for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet())
+                types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue());
+            for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext()))
+                references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored
+            for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext()))
+                unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored
+        }
+
+        return new CompiledQueryProfile(in.getId(), in.getType(),
+                                        values.build(), types.build(), references.build(), unoverridables.build(),
+                                        registry);
+    }
+
+    /**
+     * Returns all the unique combinations of dimension values which have values set reachable from this profile.
+     *
+     * @param profile the profile we are collecting the variants of
+     * @param currentVariant the variant we must have to arrive at this point in the query profile graph
+     * @param allVariants the set of all variants accumulated so far
+     */
+    private static void collectVariants(CompoundName path, QueryProfile profile, DimensionBinding currentVariant, Set<DimensionBindingForPath> allVariants) {
+        for (QueryProfile inheritedProfile : profile.inherited())
+            collectVariants(path, inheritedProfile, currentVariant, allVariants);
+
+        collectVariantsFromValues(path, profile.getContent(), currentVariant, allVariants);
+
+        collectVariantsInThis(path, profile, currentVariant, allVariants);
+        if (profile instanceof BackedOverridableQueryProfile)
+            collectVariantsInThis(path, ((BackedOverridableQueryProfile) profile).getBacking(), currentVariant, allVariants);
+    }
+
+    private static void collectVariantsInThis(CompoundName path, QueryProfile profile, DimensionBinding currentVariant, Set<DimensionBindingForPath> allVariants) {
+        QueryProfileVariants profileVariants = profile.getVariants();
+        if (profileVariants != null) {
+            for (QueryProfileVariant variant : profile.getVariants().getVariants()) {
+                DimensionBinding combinedVariant =
+                        DimensionBinding.createFrom(profile.getDimensions(), variant.getDimensionValues()).combineWith(currentVariant);
+                if (combinedVariant.isInvalid()) continue; // values at this point in the graph are unreachable
+                collectVariantsFromValues(path, variant.values(), combinedVariant, allVariants);
+                for (QueryProfile variantInheritedProfile : variant.inherited())
+                    collectVariants(path, variantInheritedProfile, combinedVariant, allVariants);
+            }
+        }
+    }
+
+    private static void collectVariantsFromValues(CompoundName path, Map<String, Object> values, DimensionBinding currentVariant, Set<DimensionBindingForPath> allVariants) {
+        if ( ! values.isEmpty())
+            allVariants.add(new DimensionBindingForPath(currentVariant, path)); // there are actual values for this variant
+
+        for (Map.Entry<String, Object> entry : values.entrySet()) {
+            if (entry.getValue() instanceof QueryProfile)
+                collectVariants(path.append(entry.getKey()), (QueryProfile)entry.getValue(), currentVariant, allVariants);
+        }
+    }
+
+    private static class DimensionBindingForPath {
+
+        private final DimensionBinding binding;
+        private final CompoundName path;
+
+        public DimensionBindingForPath(DimensionBinding binding, CompoundName path) {
+            this.binding = binding;
+            this.path = path;
+        }
+
+        public DimensionBinding binding() { return binding; }
+        public CompoundName path() { return path; }
+
+        @Override
+        public boolean equals(Object o) {
+            if ( o == this ) return true;
+            if ( ! (o instanceof DimensionBindingForPath)) return false;
+            DimensionBindingForPath other = (DimensionBindingForPath)o;
+            return other.binding.equals(this.binding) && other.path.equals(this.path);
+        }
+
+        @Override
+        public int hashCode() {
+           return binding.hashCode() + 17*path.hashCode();
+        }
+
+        @Override
+        public String toString() {
+            return binding + " for path " + path;
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
new file mode 100644
index 00000000000..2432cb2ab33
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
@@ -0,0 +1,258 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.processing.request.properties.PropertyMap;
+import com.yahoo.protect.Validator;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
+import com.yahoo.search.query.profile.compiled.DimensionalValue;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Properties backed by a query profile.
+ * This has the scope of one query and is not multithread safe.
+ *
+ * @author bratseth
+ */
+public class QueryProfileProperties extends Properties {
+
+    private final CompiledQueryProfile profile;
+
+    // Note: The priority order is: values has precedence over references
+
+    /** Values which has been overridden at runtime, or null if none */
+    private Map<CompoundName, Object> values = null;
+    /** Query profile references which has been overridden at runtime, or null if none. Earlier values has precedence */
+    private List<Pair<CompoundName, CompiledQueryProfile>> references = null;
+
+    /** Creates an instance from a profile, throws an exception if the given profile is null */
+    public QueryProfileProperties(CompiledQueryProfile profile) {
+        Validator.ensureNotNull("The profile wrapped by this cannot be null", profile);
+        this.profile = profile;
+    }
+
+    /** Returns the query profile backing this, or null if none */
+    public CompiledQueryProfile getQueryProfile() { return profile; }
+
+    /** Gets a value from the query profile, or from the nested profile if the value is null */
+    @Override
+    public Object get(CompoundName name, Map<String,String> context,
+                      com.yahoo.processing.request.Properties substitution) {
+        name = unalias(name, context);
+        Object value = null;
+        if (values != null)
+            value = values.get(name);
+        if (value == null) {
+            Pair<CompoundName, CompiledQueryProfile> reference = findReference(name);
+            if (reference != null)
+                return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // yes; even if null
+        }
+
+        if (value == null)
+            value = profile.get(name, context, substitution);
+        if (value == null)
+            value = super.get(name, context, substitution);
+        return value;
+    }
+
+    /**
+     * Sets a value in this query profile
+     *
+     * @throws IllegalArgumentException if this property cannot be set in the wrapped query profile
+     */
+    @Override
+    public void set(CompoundName name, Object value, Map<String,String> context) {
+        // TODO: Refactor
+        try {
+            name = unalias(name, context);
+
+            if (context == null)
+                context = Collections.emptyMap();
+
+            if ( ! profile.isOverridable(name, context)) return;
+
+            // Check runtime references
+            Pair<CompoundName, CompiledQueryProfile> runtimeReference = findReference(name);
+            if (runtimeReference != null &&  ! runtimeReference.getSecond().isOverridable(name.rest(runtimeReference.getFirst().size()), context))
+                return;
+
+            // Check types
+            if ( ! profile.getTypes().isEmpty()) {
+                for (int i = 0; i<name.size(); i++) {
+                    QueryProfileType type = profile.getType(name.first(i), context);
+                    if (type == null) continue;
+                    String localName = name.get(i);
+                    FieldDescription fieldDescription = type.getField(localName);
+                    if (fieldDescription == null && type.isStrict())
+                        throw new IllegalArgumentException("'" + localName + "' is not declared in " + type + ", and the type is strict");
+
+                    // TODO: In addition to strictness, check legality along the way
+
+                    if (i == name.size()-1 && fieldDescription != null) { // at the end of the path, check the assignment type
+                        value = fieldDescription.getType().convertFrom(value, profile.getRegistry());
+                        if (value == null)
+                            throw new IllegalArgumentException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription());
+                    }
+                }
+            }
+
+            if (value instanceof String && value.toString().startsWith("ref:")) {
+                if (profile.getRegistry() == null)
+                    throw new IllegalArgumentException("Runtime query profile references does not work when the " +
+                                                       "QueryProfileProperties are constructed without a registry");
+                String queryProfileId = value.toString().substring(4);
+                value = profile.getRegistry().findQueryProfile(queryProfileId);
+                if (value == null)
+                    throw new IllegalArgumentException("Query profile '" + queryProfileId + "' is not found");
+            }
+
+            if (value instanceof CompiledQueryProfile) { // this will be due to one of the two clauses above
+                if (references == null)
+                    references = new ArrayList<>();
+                references.add(0, new Pair<>(name, (CompiledQueryProfile)value)); // references set later has precedence - put first
+            }
+            else {
+                if (values == null)
+                    values = new HashMap<>();
+                values.put(name, value);
+            }
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "': " + e.getMessage()); // TODO: Nest instead
+        }
+    }
+
+    @Override
+    public Map<String, Object> listProperties(CompoundName path, Map<String,String> context,
+                                              com.yahoo.processing.request.Properties substitution) {
+        path = unalias(path, context);
+        if (context == null) context = Collections.emptyMap();
+
+        Map<String, Object> properties = profile.listValues(path, context, substitution);
+
+        properties.putAll(super.listProperties(path, context, substitution));
+
+        if (references != null) {
+            for (Pair<CompoundName, CompiledQueryProfile> refEntry : references) {
+                if ( ! refEntry.getFirst().hasPrefix(path.first(Math.min(refEntry.getFirst().size(), path.size())))) continue;
+
+                CompoundName pathInReference;
+                CompoundName prefixToReferenceKeys;
+                if (refEntry.getFirst().size() > path.size()) {
+                    pathInReference = CompoundName.empty;
+                    prefixToReferenceKeys = refEntry.getFirst().rest(path.size());
+                }
+                else {
+                    pathInReference = path.rest(refEntry.getFirst().size());
+                    prefixToReferenceKeys = CompoundName.empty;
+                }
+                for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) {
+                    properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue());
+                }
+            }
+
+        }
+
+        if (values != null) {
+            for (Map.Entry<CompoundName, Object> entry : values.entrySet()) {
+                if (entry.getKey().hasPrefix(path))
+                    properties.put(entry.getKey().rest(path.size()).toString(), entry.getValue());
+            }
+        }
+
+        return properties;
+    }
+
+    public boolean isComplete(StringBuilder firstMissingName, Map<String,String> context) {
+        // Are all types reachable from this complete?
+        if ( ! reachableTypesAreComplete(CompoundName.empty, profile, firstMissingName, context))
+            return false;
+
+        // Are all runtime references in this complete?
+        if (references == null) return true;
+        for (Pair<CompoundName, CompiledQueryProfile> reference : references) {
+            if ( ! reachableTypesAreComplete(reference.getFirst(), reference.getSecond(), firstMissingName, context))
+                return false;
+        }
+
+        return true;
+    }
+
+    private boolean reachableTypesAreComplete(CompoundName prefix, CompiledQueryProfile profile, StringBuilder firstMissingName, Map<String,String> context) {
+        for (Map.Entry<CompoundName, DimensionalValue<QueryProfileType>> typeEntry : profile.getTypes().entrySet()) {
+            QueryProfileType type = typeEntry.getValue().get(context);
+            if (type == null) continue;
+            if ( ! typeIsComplete(prefix.append(typeEntry.getKey()), type, firstMissingName, context))
+                return false;
+        }
+        return true;
+    }
+
+    private boolean typeIsComplete(CompoundName prefix, QueryProfileType type, StringBuilder firstMissingName, Map<String,String> context) {
+        if (type == null) return true;
+        for (FieldDescription field : type.fields().values()) {
+            if ( ! field.isMandatory()) continue;
+
+            CompoundName fieldName = prefix.append(field.getName());
+            if ( get(fieldName, null) != null) continue;
+            if ( hasReference(fieldName)) continue;
+
+            if (profile.getReferences().get(fieldName, context) != null) continue;
+
+            if (firstMissingName != null)
+                firstMissingName.append(fieldName);
+            return false;
+        }
+        return true;
+    }
+
+    private boolean hasReference(CompoundName name) {
+        if (references == null) return false;
+        for (Pair<CompoundName, CompiledQueryProfile> reference : references)
+            if (reference.getFirst().equals(name))
+                return true;
+        return false;
+    }
+
+    private Pair<CompoundName, CompiledQueryProfile> findReference(CompoundName name) {
+        if (references == null) return null;
+        for (Pair<CompoundName, CompiledQueryProfile> entry : references) {
+            if (name.hasPrefix(entry.getFirst())) return entry;
+        }
+        return null;
+    }
+
+    CompoundName unalias(CompoundName name, Map<String,String> context) {
+        if (profile.getTypes().isEmpty()) return name;
+
+        CompoundName unaliasedName = name;
+        for (int i = 0; i<name.size(); i++) {
+            QueryProfileType type = profile.getType(name.first(i), context);
+            if (type == null) continue;
+            if (type.aliases() == null) continue; // TODO: Make never null
+            if (type.aliases().isEmpty()) continue;
+            String localName = name.get(i);
+            String unaliasedLocalName = type.unalias(localName);
+            unaliasedName = unaliasedName.set(i, unaliasedLocalName);
+        }
+        return unaliasedName;
+    }
+
+    @Override
+    public QueryProfileProperties clone() {
+        QueryProfileProperties clone = (QueryProfileProperties)super.clone();
+        if (this.values != null)
+            clone.values = PropertyMap.cloneMap(this.values);
+        return clone;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java
new file mode 100644
index 00000000000..a4bca752d18
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java
@@ -0,0 +1,89 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry;
+
+/**
+ * A set of query profiles. This also holds the query profile types as a dependent registry
+ *
+ * @author bratseth
+ */
+public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> {
+
+    private QueryProfileTypeRegistry queryProfileTypeRegistry = new QueryProfileTypeRegistry();
+
+    /** The current default instance of this registry */
+    private static QueryProfileRegistry instance = new QueryProfileRegistry();
+
+    /** Register this type by its id */
+    public void register(QueryProfile profile) {
+        super.register(profile.getId(), profile);
+    }
+
+    /** Returns a query profile type by name, or null if not found */
+    public QueryProfileType getType(String type) {
+        return queryProfileTypeRegistry.getComponent(type);
+    }
+
+    /** Returns the type registry attached to this */
+    public QueryProfileTypeRegistry getTypeRegistry() { return queryProfileTypeRegistry; }
+
+    /**
+     * <p>Returns a query profile for the given request string, or null if a suitable one is not found.</p>
+     *
+     * The request string must be a valid {@link com.yahoo.component.ComponentId} or null.
+     *
+     * <p>
+     * If the string is null, the profile named "default" is returned, or null if that does not exists.
+     *
+     * <p>
+     * The version part (if any) is matched used the usual component version patching rules.
+     * If the name part matches a query profile name perfectly, that profile is returned.
+     * If not, and the name is a slash-separated path, the profile with the longest matching left sub-path
+     * which has a type which allows path mahting is used. If there is no such profile, null is returned.
+     */
+    public QueryProfile findQueryProfile(String idString) {
+        if (idString==null) return getComponent("default");
+        ComponentSpecification id=new ComponentSpecification(idString);
+        QueryProfile profile=getComponent(id);
+        if (profile!=null) return profile;
+
+        return findPathParentQueryProfile(new ComponentSpecification(idString));
+    }
+
+    private QueryProfile findPathParentQueryProfile(ComponentSpecification id) {
+        // Try the name with "/" appended - should have the same semantics with path matching
+        QueryProfile slashedProfile=getComponent(new ComponentSpecification(id.getName() + "/",id.getVersionSpecification()));
+        if (slashedProfile!=null && slashedProfile.getType()!=null && slashedProfile.getType().getMatchAsPath())
+            return slashedProfile;
+
+        // Extract the parent (if any)
+        int slashIndex=id.getName().lastIndexOf("/");
+        if (slashIndex<1) return null;
+        String parentName=id.getName().substring(0,slashIndex);
+        if (parentName.equals("")) return null;
+
+        ComponentSpecification parentId=new ComponentSpecification(parentName,id.getVersionSpecification());
+
+        QueryProfile pathParentProfile=getComponent(parentId);
+
+        if (pathParentProfile!=null && pathParentProfile.getType()!=null && pathParentProfile.getType().getMatchAsPath())
+            return pathParentProfile;
+        return findPathParentQueryProfile(parentId);
+    }
+
+    /** Freezes this, and all owned query profiles and query profile types */
+    public @Override void freeze() {
+        if (isFrozen()) return;
+        queryProfileTypeRegistry.freeze();
+        for (QueryProfile queryProfile : allComponents())
+            queryProfile.freeze();
+    }
+
+    public CompiledQueryProfileRegistry compile() { return QueryProfileCompiler.compile(this); }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java
new file mode 100644
index 00000000000..42ea4a96d8f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariant.java
@@ -0,0 +1,157 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.*;
+
+/**
+ * A variant of a query profile
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+*/
+public class QueryProfileVariant implements Cloneable, Comparable<QueryProfileVariant> {
+
+    private List<QueryProfile> inherited=null;
+
+    private DimensionValues dimensionValues;
+
+    private Map<String,Object> values;
+
+    private boolean frozen=false;
+
+    private QueryProfile owner;
+
+    public QueryProfileVariant(DimensionValues dimensionValues, QueryProfile owner) {
+        this.dimensionValues=dimensionValues;
+        this.owner = owner;
+    }
+
+    public DimensionValues getDimensionValues() { return dimensionValues; }
+
+    /**
+     * Returns the live reference to the values of this. This may be modified
+     * if this is not frozen.
+     */
+    public Map<String,Object> values() {
+        if (values==null) {
+            if (frozen)
+                return Collections.emptyMap();
+            else
+                values=new HashMap<>();
+        }
+        return values;
+    }
+
+    /**
+     * Returns the live reference to the inherited profiles of this. This may be modified
+     * if this is not frozen.
+     */
+    public List<QueryProfile> inherited() {
+        if (inherited==null) {
+            if (frozen)
+                return Collections.emptyList();
+            else
+                inherited=new ArrayList<>();
+        }
+        return inherited;
+    }
+
+    public void set(String key, Object newValue) {
+        if (values==null)
+            values=new HashMap<>();
+
+        Object oldValue = values.get(key);
+
+        if (oldValue == null) {
+            values.put(key, newValue);
+        } else {
+            Object combinedOrNull = QueryProfile.combineValues(newValue, oldValue);
+            if (combinedOrNull != null) {
+                values.put(key, combinedOrNull);
+            }
+        }
+    }
+
+    public void inherit(QueryProfile profile) {
+        if (inherited==null)
+            inherited=new ArrayList<>(1);
+        inherited.add(profile);
+    }
+
+    /**
+     * Implements the sort order of this which is based on specificity
+     * where dimensions to the left are more significant.
+     * <p>
+     * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions
+     * are <i>set</i>, regardless of what they are set <i>to</i>.
+     */
+    public @Override int compareTo(QueryProfileVariant other) {
+        return this.dimensionValues.compareTo(other.dimensionValues);
+    }
+
+    public boolean matches(DimensionValues givenDimensionValues) {
+        return this.dimensionValues.matches(givenDimensionValues);
+    }
+
+    /** Accepts a visitor to the values of this */
+    public void accept(boolean allowContent,QueryProfileType type,QueryProfileVisitor visitor, DimensionBinding dimensionBinding) {
+        // Visit this
+        if (allowContent) {
+            String key=visitor.getLocalKey();
+            if (key!=null) {
+                if (type!=null)
+                    type.unalias(key);
+
+                visitor.acceptValue(key, values().get(key), dimensionBinding, owner);
+                if (visitor.isDone()) return;
+            }
+            else {
+                for (Map.Entry<String,Object> entry : values().entrySet()) {
+                    visitor.acceptValue(entry.getKey(), entry.getValue(), dimensionBinding, owner);
+                    if (visitor.isDone()) return;
+                }
+            }
+        }
+
+        // Visit inherited
+        for (QueryProfile profile : inherited()) {
+            if (visitor.visitInherited()) {
+                profile.accept(allowContent,visitor,dimensionBinding.createFor(profile.getDimensions()), owner);
+            }
+            if (visitor.isDone()) return;
+        }
+    }
+
+    public void freeze() {
+        if (frozen) return;
+        if (inherited != null)
+            inherited = ImmutableList.copyOf(inherited);
+        if (values != null)
+            values = ImmutableMap.copyOf(values);
+        frozen=true;
+    }
+
+    public QueryProfileVariant clone() {
+        if (frozen) return this;
+       try {
+           QueryProfileVariant clone=(QueryProfileVariant)super.clone();
+           if (this.inherited!=null)
+               clone.inherited=new ArrayList<>(this.inherited); // TODO: Deep clone is more correct, but probably does not matter in practice
+
+           clone.values=CopyOnWriteContent.deepClone(this.values);
+
+           return clone;
+       }
+       catch (CloneNotSupportedException e) {
+           throw new RuntimeException(e);
+       }
+    }
+
+    public @Override String toString() {
+        return "query profile variant for " + dimensionValues;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java
new file mode 100644
index 00000000000..fde851bdc75
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java
@@ -0,0 +1,486 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.provider.Freezable;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.*;
+
+/**
+ * This class represent a set of query profiles virtually - rather
+ * than storing and instantiating each profile this structure represents explicitly only
+ * the values set in the various virtual profiles. The set of virtual profiles are defined by a set of
+ * <i>dimensions</i>. Values may be set for any point in this multi-dimensional space, and may also be set for
+ * any regular hyper-region by setting values for any point in certain of these dimensions.
+ * The set of virtual profiles defined by this consists of all the combinations of dimension points for
+ * which one or more values is set in this, as well as any possible less specified regions.
+ * <p>
+ * A set of virtual profiles are always owned by a single profile, which is also their parent
+ * in the inheritance hierarchy.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class QueryProfileVariants implements Freezable, Cloneable {
+
+    private boolean frozen=false;
+
+    /** Properties indexed by name, to support fast lookup of single values */
+    private Map<String,FieldValues> fieldValuesByName=new HashMap<>();
+
+    /** The inherited profiles for various dimensions settings - a set of fieldvalues of List&lt;QueryProfile&gt; */
+    private FieldValues inheritedProfiles=new FieldValues();
+
+    /**
+     * Field and inherited profiles sorted by specificity used for all-value visiting.
+     * This is the same as how the source data looks (apart from the sorting).
+     */
+    private List<QueryProfileVariant> variants=new ArrayList<>();
+
+    /**
+     * The names of the dimensions (which are possible properties in the context given on lookup) of this.
+     * Order matters - more specific values to the left in this list are more significant than more specific values
+     * to the right
+     */
+    private List<String> dimensions;
+
+    /** The query profile this variants of */
+    private QueryProfile owner;
+
+    /**
+     * Creates a set of virtual query profiles which may return varying values over the set of dimensions given.
+     * Each dimension is a name for which a key-value may be supplied in the context properties
+     * on lookup time to influence the value returned.
+     */
+    public QueryProfileVariants(String[] dimensions, QueryProfile owner) {
+        this(Arrays.asList(dimensions), owner);
+    }
+
+    /**
+     * Creates a set of virtual query profiles which may return varying values over the set of dimensions given.
+     * Each dimension is a name for which a key-value may be supplied in the context properties
+     * on lookup time to influence the value returned.
+     *
+     * @param dimensions the dimension names this may vary over. The list gets owned by this, so it must not be further
+     *        modified from outside). This will not modify the list.
+     */
+    public QueryProfileVariants(List<String> dimensions, QueryProfile owner) {
+        // Note: This is not made unmodifiable (here or in freeze) because we depend on map identity comparisons of this
+        // list (in dimensionBinding) for performance reasons.
+        this.dimensions = dimensions;
+        this.owner = owner;
+    }
+
+    /** Irreversibly prevents any further modifications to this */
+    public void freeze() {
+        if (frozen) return;
+        for (FieldValues fieldValues : fieldValuesByName.values())
+            fieldValues.freeze();
+        fieldValuesByName = ImmutableMap.copyOf(fieldValuesByName);
+        inheritedProfiles.freeze();
+
+        Collections.sort(variants);
+        for (QueryProfileVariant variant : variants)
+            variant.freeze();
+        variants = ImmutableList.copyOf(variants);
+
+        frozen=true;
+    }
+
+    @Override
+    public boolean isFrozen() {
+        return frozen;
+    }
+
+    /** Visits the most specific match to the dimension binding of each variable (or the one named by the visitor) */
+    void accept(boolean allowContent,QueryProfileType type,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) {
+        String contentName=null;
+        if (allowContent)
+            contentName=visitor.getLocalKey();
+
+        if (contentName!=null) {
+            if (type!=null)
+                contentName=type.unalias(contentName);
+            acceptSingleValue(contentName,allowContent,visitor,dimensionBinding); // Special cased for performance
+        }
+        else {
+            acceptAllValues(allowContent,visitor,type,dimensionBinding);
+        }
+    }
+
+    // PERF: 90%
+    void acceptSingleValue(String name,boolean allowContent,QueryProfileVisitor visitor,DimensionBinding dimensionBinding) {
+        FieldValues fieldValues=fieldValuesByName.get(name);
+        if (fieldValues==null || !allowContent)
+            fieldValues=new FieldValues();
+
+        fieldValues.sort();
+        inheritedProfiles.sort();
+
+        int inheritedIndex=0;
+        int fieldIndex=0;
+        // Go through both the fields and the inherited profiles at the same time and try the single must specific pick
+        // from either of the lists at each step
+        while(fieldIndex<fieldValues.size() || inheritedIndex<inheritedProfiles.size()) { // PERF: 8% - fieldValues.size()
+            // Get the next most specific from field and inherited
+            FieldValue fieldValue=fieldValues.getIfExists(fieldIndex); // PERF: 11% - getIfExists
+            FieldValue inheritedProfileValue=inheritedProfiles.getIfExists(inheritedIndex); // PERF: 11% - getIfExists
+
+            // Try the most specific first, then the other
+            if (inheritedProfileValue==null || (fieldValue!=null && fieldValue.compareTo(inheritedProfileValue)<=0)) { // Field is most specific, or both are equally specific
+                if (fieldValue.matches(dimensionBinding.getValues())) { // PERF: 42% - matches, together with the other matches
+                    visitor.acceptValue(name, fieldValue.getValue(), dimensionBinding, owner);
+                }
+                if (visitor.isDone()) return;
+                fieldIndex++;
+            }
+            else if (inheritedProfileValue!=null) { // Inherited is most specific at this point
+                if (inheritedProfileValue.matches(dimensionBinding.getValues())) { // PERF: 42% - matches, together with the other matches
+                    @SuppressWarnings("unchecked")
+                    List<QueryProfile> inheritedProfileList=(List<QueryProfile>)inheritedProfileValue.getValue();
+                    for (QueryProfile inheritedProfile : inheritedProfileList) {
+                        if (visitor.visitInherited()) {
+                            inheritedProfile.accept(allowContent,visitor,dimensionBinding.createFor(inheritedProfile.getDimensions()), owner);
+                        }
+                        if (visitor.isDone()) return;
+                    }
+                }
+                inheritedIndex++;
+            }
+            if (visitor.isDone()) return;
+        }
+    }
+
+    void acceptAllValues(boolean allowContent,QueryProfileVisitor visitor, QueryProfileType type,DimensionBinding dimensionBinding) {
+        if (!frozen)
+            Collections.sort(variants);
+        for (QueryProfileVariant variant : variants) {
+            if (variant.matches(dimensionBinding.getValues()))
+                variant.accept(allowContent,type,visitor,dimensionBinding);
+            if (visitor.isDone()) return;
+        }
+    }
+
+    /**
+     * Returns the most specific matching value of a name for a given set of <b>canonical</b> dimension values.
+     *
+     * @param name the name to return the best matching value of
+     * @param dimensionBinding the dimension bindings to use in this
+     */
+    public Object get(String name, QueryProfileType type, boolean allowQueryProfileResult, DimensionBinding dimensionBinding) {
+        SingleValueQueryProfileVisitor visitor=new SingleValueQueryProfileVisitor(Collections.singletonList(name),allowQueryProfileResult);
+        visitor.enter("");
+        accept(true,type,visitor,dimensionBinding);
+        visitor.leave("");
+        return visitor.getResult();
+    }
+
+    /** Inherits a particular profile in a variant of this */
+    public void inherit(QueryProfile profile,DimensionValues dimensionValues) {
+        ensureNotFrozen();
+
+        // Update variant
+        getVariant(dimensionValues,true).inherit(profile);
+
+        // Update per-variable optimized structure
+        @SuppressWarnings("unchecked")
+        List<QueryProfile> inheritedAtDimensionValues=(List<QueryProfile>)inheritedProfiles.getExact(dimensionValues);
+        if (inheritedAtDimensionValues==null) {
+            inheritedAtDimensionValues=new ArrayList<>();
+            inheritedProfiles.put(dimensionValues,inheritedAtDimensionValues);
+        }
+        inheritedAtDimensionValues.add(profile);
+    }
+
+    /**
+     * Sets a value to this
+     *
+     * @param fieldName the name of the field to set. This cannot be a compound (dotted) name
+     * @param binding the dimension values for which this value applies.
+     *         The dimensions must be canonicalized, and ownership is transferred to this.
+     * @param value the value to set
+     */
+    /**
+     * Sets a value to this
+     *
+     * @param fieldName the name of the field to set. This cannot be a compound (dotted) name
+     * @param dimensionValues the dimension values for which this value applies
+     * @param value the value to set
+     */
+    public void set(String fieldName,DimensionValues dimensionValues,Object value) {
+        ensureNotFrozen();
+
+        // Update variant
+        getVariant(dimensionValues,true).set(fieldName,value);
+
+        // Update per-variable optimized structure
+        FieldValues fieldValues=fieldValuesByName.get(fieldName);
+        if (fieldValues==null) {
+            fieldValues=new FieldValues();
+            fieldValuesByName.put(fieldName,fieldValues);
+        }
+
+        Object combinedValue=QueryProfile.combineValues(value,fieldValues.getExact(dimensionValues));
+        if (combinedValue!=null)
+            fieldValues.put(dimensionValues,combinedValue);
+    }
+
+    /**
+     * Returns the dimensions over which the virtual profiles in this may return different values.
+     * Each dimension is a name for which a key-value may be supplied in the context properties
+     * on lookup time to influence the value returned.
+     * The dimensions may not be modified - the returned list is always read only.
+     */
+    // Note: A performance optimization in DimensionBinding depends on the identity of the list returned from this
+    public List<String> getDimensions() { return dimensions; }
+
+    /** Returns the map of field values of this indexed by field name. */
+    public Map<String,FieldValues> getFieldValues() { return fieldValuesByName; }
+
+    /** Returns the profiles inherited from various variants of this */
+    public FieldValues getInherited() { return inheritedProfiles; }
+
+    /**
+     * Returns all the variants of this, sorted by specificity. This is content as declared.
+     * The returned list is always unmodifiable.
+     */
+    public List<QueryProfileVariant> getVariants() {
+        if (frozen) return variants; // Already unmodifiable
+        return Collections.unmodifiableList(variants);
+    }
+
+    public QueryProfileVariants clone() {
+        try {
+            if (frozen) return this;
+            QueryProfileVariants clone=(QueryProfileVariants)super.clone();
+            clone.inheritedProfiles=inheritedProfiles.clone();
+
+            clone.variants=new ArrayList<>();
+            for (QueryProfileVariant variant : variants)
+                clone.variants.add(variant.clone());
+
+            clone.fieldValuesByName=new HashMap<>();
+            for (Map.Entry<String,FieldValues> entry : fieldValuesByName.entrySet())
+                clone.fieldValuesByName.put(entry.getKey(),entry.getValue().clone(entry.getKey(),clone.variants));
+
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /** Throws an IllegalStateException if this is frozen */
+    protected void ensureNotFrozen() {
+        if (frozen)
+            throw new IllegalStateException(this + " is frozen and cannot be modified");
+    }
+
+    /**
+     * Returns the query profile variant having exactly the given dimensions, and creates it if create is set and
+     * it is missing
+     *
+     * @param  dimensionValues the dimension values
+     * @param  create whether or not to create the variant if missing
+     * @return the profile variant, or null if not found and create is false
+     */
+    public QueryProfileVariant getVariant(DimensionValues dimensionValues,boolean create) {
+        for (QueryProfileVariant profileVariant : variants)
+            if (profileVariant.getDimensionValues().equals(dimensionValues))
+                return profileVariant;
+
+        // Not found
+        if (!create) return null;
+        QueryProfileVariant variant=new QueryProfileVariant(dimensionValues, owner);
+        variants.add(variant);
+        return variant;
+    }
+
+    public static class FieldValues implements Freezable, Cloneable {
+
+        private List<FieldValue> resolutionList=null;
+
+        private boolean frozen=false;
+
+        @Override
+        public void freeze() {
+            if (frozen) return;
+            sort();
+            if (resolutionList != null)
+                resolutionList = ImmutableList.copyOf(resolutionList);
+            frozen = true;
+        }
+
+        @Override
+        public boolean isFrozen() {
+            return frozen;
+        }
+
+        public void put(DimensionValues dimensionValues,Object value) {
+            ensureNotFrozen();
+            if (resolutionList==null) resolutionList=new ArrayList<>();
+            FieldValue fieldValue=getExactFieldValue(dimensionValues);
+            if (fieldValue!=null) // Replace
+                fieldValue.setValue(value);
+            else
+                resolutionList.add(new FieldValue(dimensionValues,value));
+        }
+
+        /** Returns the value having exactly the given dimensions, or null if none */
+        public Object getExact(DimensionValues dimensionValues) {
+            FieldValue value=getExactFieldValue(dimensionValues);
+            if (value==null) return null;
+            return value.getValue();
+        }
+
+        /** Returns the field value having exactly the given dimensions, or null if none */
+        private FieldValue getExactFieldValue(DimensionValues dimensionValues) {
+            for (FieldValue fieldValue : asList())
+                if (fieldValue.getDimensionValues().equals(dimensionValues))
+                    return fieldValue;
+            return null;
+        }
+
+        /** Returns the field values (values for various dimensions) for this field as a read-only list (never null) */
+        public List<FieldValue> asList() {
+            if (resolutionList==null) return Collections.emptyList();
+            return resolutionList;
+        }
+
+        public FieldValue getIfExists(int index) {
+            if (index>=size()) return null;
+            return resolutionList.get(index);
+        }
+
+        public void sort() {
+            if (frozen) return ; // sorted already
+            if (resolutionList!=null)
+                Collections.sort(resolutionList);
+        }
+
+        /** Same as asList().size() */
+        public int size() {
+            if (resolutionList==null) return 0;
+            return resolutionList.size();
+        }
+
+        /** Throws an IllegalStateException if this is frozen */
+        protected void ensureNotFrozen() {
+            if (frozen)
+                throw new IllegalStateException(this + " is frozen and cannot be modified");
+        }
+
+        /** Clone by filling in values from the given variants */
+        public FieldValues clone(String fieldName,List<QueryProfileVariant> clonedVariants) {
+            try {
+                if (frozen) return this;
+                FieldValues clone=(FieldValues)super.clone();
+
+                if (resolutionList!=null) {
+                    clone.resolutionList=new ArrayList<>(resolutionList.size());
+                    for (FieldValue value : resolutionList)
+                        clone.resolutionList.add(value.clone(fieldName,clonedVariants));
+                }
+
+                return clone;
+            }
+            catch (CloneNotSupportedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        public @Override FieldValues clone() {
+            try {
+                if (frozen) return this;
+                FieldValues clone=(FieldValues)super.clone();
+
+                if (resolutionList!=null) {
+                    clone.resolutionList=new ArrayList<>(resolutionList.size());
+                    for (FieldValue value : resolutionList)
+                        clone.resolutionList.add(value.clone());
+                }
+
+                return clone;
+            }
+            catch (CloneNotSupportedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+    }
+
+    public static class FieldValue implements Comparable<FieldValue>, Cloneable {
+
+        private DimensionValues dimensionValues;
+        private Object value;
+
+        public FieldValue(DimensionValues dimensionValues,Object value) {
+            this.dimensionValues=dimensionValues;
+            this.value=value;
+        }
+
+        /**
+         * Returns the dimension values for which this value should be used.
+         * The dimension array is always of the exact size of the dimensions specified by the owning QueryProfileVariants,
+         * and the values appear in the order defined. "Wildcard" dimensions are represented by a null.
+         */
+        public DimensionValues getDimensionValues() { return dimensionValues; }
+
+        /** Returns the value to use for this set of dimension values */
+        public Object getValue() { return value; }
+
+        /** Sets the value to use for this set of dimension values */
+        public void setValue(Object value) { this.value=value; }
+
+        public boolean matches(DimensionValues givenDimensionValues) {
+            return dimensionValues.matches(givenDimensionValues);
+        }
+
+        /**
+         * Implements the sort order of this which is based on specificity
+         * where dimensions to the left are more significant.
+         * <p>
+         * <b>Note:</b> This ordering is not consistent with equals - it returns 0 when the same dimensions
+         * are <i>set</i>, regardless of what they are set <i>to</i>.
+         */
+        public @Override int compareTo(FieldValue other) {
+            return this.dimensionValues.compareTo(other.dimensionValues);
+        }
+
+        /** Clone by filling in the value from the given variants */
+        public FieldValue clone(String fieldName,List<QueryProfileVariant> clonedVariants) {
+            try {
+                FieldValue clone=(FieldValue)super.clone();
+                if (this.value instanceof QueryProfile)
+                    clone.value=lookupInVariants(fieldName,dimensionValues,clonedVariants);
+                // Otherwise the value is immutable, so keep it as-is
+                return clone;
+            }
+            catch (CloneNotSupportedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        public FieldValue clone() {
+            try {
+                FieldValue clone=(FieldValue)super.clone();
+                clone.value=QueryProfile.cloneIfNecessary(this.value);
+                return clone;
+            }
+            catch (CloneNotSupportedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        private Object lookupInVariants(String fieldName,DimensionValues dimensionValues,List<QueryProfileVariant> variants) {
+            for (QueryProfileVariant variant : variants) {
+                if ( ! variant.getDimensionValues().equals(dimensionValues)) continue;
+                return variant.values().get(fieldName);
+            }
+            return null;
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVisitor.java
new file mode 100644
index 00000000000..8cb6bf34021
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVisitor.java
@@ -0,0 +1,87 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+/**
+ * Instances of this is used to visit nodes in a graph of query profiles
+ *
+ * <code>
+ * Visitor are called in the following sequence on each query profile:
+ * enter=enter(referenceName);
+ * onQueryProfile(this)
+ * if (enter) {
+ *   getLocalKey()
+ *     ...calls on nested content found in variants, this and inherited, in that order
+ *   leave(referenceName)
+ * }
+ *
+ * The first enter call will be on the root node, which has an empt reference name.
+ * </code>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+abstract class QueryProfileVisitor {
+
+    /**
+     * Called when a new <b>nested</b> profile in the graph is entered.
+     * This default implementation does nothing but returning true.
+     * If the node is entered (if true is returned from this), a corresponding {@link #leave(String)} call will happen
+     * later.
+     *
+     * @param name the name this profile is nested as, or the empty string if we are entering the root profile
+     * @return whether we should visit the content of this node or not
+     */
+    public boolean enter(String name) { return true; }
+
+    /**
+     * Called when the last {@link #enter(String) entered} nested profile is left.
+     * That is: One leave call is made for each enter call which returns true,
+     * but due to nesting those calls are not necessarily alternating.
+     * This default implementation does nothing.
+     */
+    public void leave(String name) { }
+
+    /**
+     * Called when a value (not a query profile) is encountered.
+     *
+     * @param localName the local name of this value (the full name, if needed, must be reconstructed
+     *        by the information given by the history of {@link #enter(String)} and {@link #leave(String)} calls
+     * @param value the value
+     * @param binding the binding this holds for
+     * @param owner the query profile having this value, or null only when profile is the root profile
+     */
+    public abstract void onValue(String localName, Object value, DimensionBinding binding, QueryProfile owner);
+
+    /**
+     * Called when a query profile is encountered.
+     *
+     * @param profile the query profile reference encountered
+     * @param binding the binding this holds for
+     * @param owner the profile making this reference, or null only when profile is the root profile
+     */
+    public abstract void onQueryProfile(QueryProfile profile, DimensionBinding binding, QueryProfile owner);
+
+    /** Returns whether this visitor is done visiting what it needed to visit at this point */
+    public abstract boolean isDone();
+
+    /** Returns whether we should, at this point, visit inherited profiles. This default implementation returns true */
+    public boolean visitInherited() { return true; }
+
+    /**
+     * Returns the current local key which should be visited in the last {@link #enter(String) entered} sub-profile
+     * (or in the top level profile if none is entered), or null to visit all content
+     */
+    public abstract String getLocalKey();
+
+    /** Calls onValue or onQueryProfile on this and visits the content if it's a profile */
+    final void acceptValue(String key, Object value, DimensionBinding dimensionBinding, QueryProfile owner) {
+        if (value==null) return;
+        if (value instanceof QueryProfile) {
+            QueryProfile queryProfileValue=(QueryProfile)value;
+            queryProfileValue.acceptAndEnter(key, this, dimensionBinding.createFor(queryProfileValue.getDimensions()), owner);
+        }
+        else {
+            onValue(key, value, dimensionBinding, owner);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java
new file mode 100644
index 00000000000..6d5d1b0686a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/SingleValueQueryProfileVisitor.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import java.util.List;
+
+/**
+ * Visitor which stores the first non-query-profile value encountered,
+ * or the first query profile encountered at a stop where we do not have any name components left which can be used to
+ * visit further subprofiles. Hence this may be used both to get the highest prioritized primitive
+ * value, or query profile, whichever is encountered first which matches the name.
+ * <p>
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+final class SingleValueQueryProfileVisitor extends QueryProfileVisitor {
+
+    /** The value found, or null if none */
+    private Object value=null;
+
+    private final List<String> name;
+
+    private int nameIndex=-1;
+
+    private final boolean allowQueryProfileResult;
+
+    private boolean enteringContent=true;
+
+    public SingleValueQueryProfileVisitor(List<String> name,boolean allowQueryProfileResult) {
+        this.name=name;
+        this.allowQueryProfileResult=allowQueryProfileResult;
+    }
+
+    public @Override String getLocalKey() {
+        return name.get(nameIndex);
+    }
+
+    public @Override boolean enter(String name) {
+        if (nameIndex+1<this.name.size()) {
+            nameIndex++;
+            enteringContent=true;
+        }
+        else {
+            enteringContent=false;
+        }
+        return enteringContent;
+    }
+
+    public @Override void leave(String name) {
+        nameIndex--;
+    }
+
+    public @Override void onValue(String key,Object value, DimensionBinding binding, QueryProfile owner) {
+        if (nameIndex==name.size()-1)
+            this.value=value;
+    }
+
+    public @Override void onQueryProfile(QueryProfile profile,DimensionBinding binding, QueryProfile owner) {
+        if (enteringContent) return; // still waiting for content
+        if (allowQueryProfileResult)
+            this.value = profile;
+        else
+            this.value = profile.getValue();
+    }
+
+    public @Override boolean isDone() {
+        return value!=null;
+    }
+
+    /** Returns the value found during visiting, or null if none */
+    public Object getResult() { return value; }
+
+    public @Override String toString() {
+        return "a single value visitor (hash " + hashCode() + ") with current value " + value;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java
new file mode 100644
index 00000000000..59401592378
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/SubstituteString.java
@@ -0,0 +1,127 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile;
+
+import com.yahoo.processing.request.Properties;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A string which contains one or more elements of the form %{name},
+ * where these occurrences are to be replaced by a query profile lookup on name.
+ * <p>
+ * This objects does the analysis on creation and provides a (reasonably) fast method of
+ * performing the actual substitution (at lookup time).
+ * <p>
+ * This is a value object. Lookups in this are thread safe.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class SubstituteString {
+
+    private final List<Component> components;
+    private final String stringValue;
+
+    /**
+     * Returns a new SubstituteString if the given string contains substitutions, null otherwise.
+     */
+    public static SubstituteString create(String value) {
+        int lastEnd=0;
+        int start=value.indexOf("%{");
+        if (start<0) return null; // Shortcut
+        List<Component> components=new ArrayList<>();
+        while (start>=0) {
+            int end=value.indexOf("}",start+2);
+            if (end<0)
+                throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'");
+            String propertyName=value.substring(start+2,end);
+            if (propertyName.indexOf("%{")>=0)
+                throw new IllegalArgumentException("Unterminated value substitution '" + value.substring(start) + "'");
+            components.add(new StringComponent(value.substring(lastEnd,start)));
+            components.add(new PropertyComponent(propertyName));
+            lastEnd=end+1;
+            start=value.indexOf("%{",lastEnd);
+        }
+        components.add(new StringComponent(value.substring(lastEnd,value.length())));
+        return new SubstituteString(components, value);
+    }
+
+    private SubstituteString(List<Component> components, String stringValue) {
+        this.components = components;
+        this.stringValue = stringValue;
+    }
+
+    /**
+     * Perform the substitution in this, by looking up in the given query profile,
+     * and returns the resulting string
+     */
+    public String substitute(Map<String,String> context,Properties substitution) {
+        StringBuilder b=new StringBuilder();
+        for (Component component : components)
+            b.append(component.getValue(context,substitution));
+        return b.toString();
+    }
+
+    @Override
+    public int hashCode() {
+        return stringValue.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) return true;
+        if ( ! (other instanceof SubstituteString)) return false;
+        return this.stringValue.equals(((SubstituteString)other).stringValue);
+    }
+
+    /** Returns this string in original (unsubstituted) form */
+    public @Override String toString() {
+        return stringValue;
+    }
+
+    private abstract static class Component {
+
+        protected abstract String getValue(Map<String,String> context,Properties substitution);
+
+    }
+
+    private final static class StringComponent extends Component {
+
+        private final String value;
+
+        public StringComponent(String value) {
+            this.value=value;
+        }
+
+        public @Override String getValue(Map<String,String> context,Properties substitution) {
+            return value;
+        }
+
+        public @Override String toString() {
+            return value;
+        }
+
+    }
+
+    private final static class PropertyComponent extends Component {
+
+        private final String propertyName;
+
+        public PropertyComponent(String propertyName) {
+            this.propertyName=propertyName;
+        }
+
+        public @Override String getValue(Map<String,String> context,Properties substitution) {
+            Object value=substitution.get(propertyName,context,substitution);
+            if (value==null) return "";
+            return String.valueOf(value);
+        }
+
+        public @Override String toString() {
+            return "%{" + propertyName + "}";
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java
new file mode 100644
index 00000000000..a440365ceba
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.compiled;
+
+import com.yahoo.search.query.profile.DimensionBinding;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An immutable binding of a set of dimensions to values.
+ * This binding is minimal in that it only includes dimensions which actually have values.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class Binding implements Comparable<Binding> {
+
+    private static final int maxDimensions = 31;
+
+    /**
+     * A higher number means this is more general. This accounts for both the number and position of the bindings
+     * in the dimensional space, such that bindings in earlier dimensions are matched before bindings in
+     * later dimensions
+     */
+    private final int generality;
+
+    /** The dimensions of this. Unenforced invariant: Content never changes. */
+    private final String[] dimensions;
+
+    /** The values of those dimensions. Unenforced invariant: Content never changes. */
+    private final String[] dimensionValues;
+
+    private final int hashCode;
+
+    @SuppressWarnings("unchecked")
+    public static final Binding nullBinding= new Binding(Integer.MAX_VALUE, Collections.<String,String>emptyMap());
+
+    public static Binding createFrom(DimensionBinding dimensionBinding) {
+        if (dimensionBinding.getDimensions().size() > maxDimensions)
+            throw new IllegalArgumentException("More than 31 dimensions is not supported");
+
+        int generality = 0;
+        Map<String, String> context = new HashMap<>();
+        if (dimensionBinding.getDimensions() == null || dimensionBinding.getDimensions().isEmpty()) { // TODO: Just have this return the nullBinding
+            generality = Integer.MAX_VALUE;
+        }
+        else {
+            for (int i = 0; i <= maxDimensions; i++) {
+                String value = i < dimensionBinding.getDimensions().size() ? dimensionBinding.getValues().get(i) : null;
+                if (value == null)
+                    generality += Math.pow(2, maxDimensions - i-1);
+                else
+                    context.put(dimensionBinding.getDimensions().get(i), value);
+            }
+        }
+        return new Binding(generality, context);
+    }
+
+    private Binding(int generality, Map<String, String> binding) {
+        this.generality = generality;
+
+        // Map -> arrays to limit memory consumption and speed up evaluation
+        dimensions = new String[binding.size()];
+        dimensionValues = new String[binding.size()];
+
+        int i = 0;
+        int bindingHash = 0;
+        for (Map.Entry<String,String> entry : binding.entrySet()) {
+            dimensions[i] = entry.getKey();
+            dimensionValues[i] = entry.getValue();
+            bindingHash += i * entry.getKey().hashCode() +  11 * i * entry.getValue().hashCode();
+            i++;
+        }
+        this.hashCode = bindingHash;
+    }
+
+    /** Returns true only if this binding is null (contains no values for its dimensions (if any) */
+    public boolean isNull() { return dimensions.length == 0; }
+
+    @Override
+    public String toString() {
+        StringBuilder b = new StringBuilder("Binding[");
+        for (int i = 0; i < dimensions.length; i++)
+            b.append(dimensions[i]).append("=").append(dimensionValues[i]).append(",");
+        if (dimensions.length > 0)
+            b.setLength(b.length()-1);
+        b.append("] (generality " + generality + ")");
+        return b.toString();
+    }
+
+    /** Returns whether the given binding has exactly the same values as this */
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if (! (o instanceof Binding)) return false;
+        Binding other = (Binding)o;
+        return Arrays.equals(this.dimensions, other.dimensions)
+            && Arrays.equals(this.dimensionValues, other.dimensionValues);
+    }
+
+    @Override
+    public int hashCode() { return hashCode; }
+
+    /**
+     * Returns true if all the dimension values in this have the same values
+     * in the given context.
+     */
+    public boolean matches(Map<String,String> context) {
+        for (int i = 0; i < dimensions.length; i++) {
+            if ( ! dimensionValues[i].equals(context.get(dimensions[i]))) return false;
+        }
+        return true;
+    }
+
+    /**
+     * Implements a partial ordering where more specific bindings come before less specific ones,
+     * taking both the number of bindings and their positions into account (earlier dimensions
+     * take precedence over later ones.
+     * <p>
+     * The order is not well defined for bindings in different dimensional spaces.
+     */
+    @Override
+    public int compareTo(Binding other) {
+        return Integer.compare(this.generality, other.generality);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
new file mode 100644
index 00000000000..a4056ee55a2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
@@ -0,0 +1,183 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.compiled;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.processing.request.Properties;
+import com.yahoo.search.query.profile.QueryProfileProperties;
+import com.yahoo.search.query.profile.SubstituteString;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A query profile in a state where it is optimized for fast lookups.
+ *
+ * @author bratseth
+ */
+public class CompiledQueryProfile extends AbstractComponent implements Cloneable {
+
+    private static final Pattern namePattern=Pattern.compile("[$a-zA-Z_/][-$a-zA-Z0-9_/()]*");
+
+    private final CompiledQueryProfileRegistry registry;
+
+    /** The type of this, or null if none */
+    private final QueryProfileType type;
+
+    /** The values of this */
+    private final DimensionalMap<CompoundName, Object> entries;
+
+    /** Keys which have a type in this */
+    private final DimensionalMap<CompoundName, QueryProfileType> types;
+
+    /** Keys which are (typed or untyped) references to other query profiles in this. Used as a set. */
+    private final DimensionalMap<CompoundName, Object> references;
+
+    /** Values which are not overridable in this. Used as a set. */
+    private final DimensionalMap<CompoundName, Object> unoverridables;
+
+    /**
+     * Creates a new query profile from an id.
+     */
+    public CompiledQueryProfile(ComponentId id, QueryProfileType type,
+                                DimensionalMap<CompoundName, Object> entries,
+                                DimensionalMap<CompoundName, QueryProfileType> types,
+                                DimensionalMap<CompoundName, Object> references,
+                                DimensionalMap<CompoundName, Object> unoverridables,
+                                CompiledQueryProfileRegistry registry) {
+        super(id);
+        this.registry = registry;
+        if (type != null)
+            type.freeze();
+        this.type = type;
+        this.entries = entries;
+        this.types = types;
+        this.references = references;
+        this.unoverridables = unoverridables;
+        if ( ! id.isAnonymous())
+            validateName(id.getName());
+    }
+
+    // ----------------- Public API -------------------------------------------------------------------------------
+
+    /** Returns the registry this belongs to, or null if none (in which case runtime profile reference assignment won't work) */
+    public CompiledQueryProfileRegistry getRegistry() { return registry; }
+
+    /** Returns the type of this or null if it has no type */
+    // TODO: Move into below
+    public QueryProfileType getType() { return type; }
+
+    /**
+     * Returns whether or not the given field name can be overridden at runtime.
+     * Attempts to override values which cannot be overridden will not fail but be ignored.
+     * Default: true.
+     *
+     * @param name the name of the field to check
+     * @param context the context in which to check, or null if none
+     */
+    public final boolean isOverridable(CompoundName name, Map<String, String> context) {
+        return unoverridables.get(name, context) == null;
+    }
+
+    /** Returns the type of a given prefix reachable from this profile, or null if none */
+    public final QueryProfileType getType(CompoundName name, Map<String, String> context) {
+        return types.get(name, context);
+    }
+
+    /** Returns the types reachable from this, or an empty map (never null) if none */
+    public DimensionalMap<CompoundName, QueryProfileType> getTypes() { return types; }
+
+    /** Returns the references reachable from this, or an empty map (never null) if none */
+    public DimensionalMap<CompoundName, Object> getReferences() { return references; }
+
+    /**
+     * Return all objects that start with the given prefix path using no context. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public final Map<String, Object> listValues(final CompoundName prefix) {  return listValues(prefix, Collections.<String,String>emptyMap()); }
+    public final Map<String, Object> listValues(final String prefix) { return listValues(new CompoundName(prefix)); }
+    /**
+     * Return all objects that start with the given prefix path. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public final Map<String, Object> listValues(final String prefix,Map<String,String> context) {
+        return listValues(new CompoundName(prefix), context);
+    }
+    /**
+     * Return all objects that start with the given prefix path. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public final Map<String, Object> listValues(final CompoundName prefix,Map<String,String> context) {
+        return listValues(prefix, context, null);
+    }
+    /**
+     * Adds all objects that start with the given path prefix to the given value map. Use "" to list all.
+     * <p>
+     * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
+     * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
+     */
+    public Map<String, Object> listValues(CompoundName prefix, Map<String,String> context, Properties substitution) {
+        Map<String, Object> values = new HashMap<>();
+        for (Map.Entry<CompoundName, DimensionalValue<Object>> entry : entries.entrySet()) {
+            if ( entry.getKey().size() <= prefix.size()) continue;
+            if ( ! entry.getKey().hasPrefix(prefix)) continue;
+
+            Object value = entry.getValue().get(context);
+            if (value == null) continue;
+
+            value = substitute(value, context, substitution);
+            CompoundName suffixName = entry.getKey().rest(prefix.size());
+            values.put(suffixName.toString(), value);
+        }
+        return values;
+    }
+
+    public final Object get(String name) {
+        return get(name, Collections.<String,String>emptyMap());
+    }
+    public final Object get(String name, Map<String,String> context) {
+        return get(name, context, new QueryProfileProperties(this));
+    }
+    public final Object get(String name, Map<String,String> context, Properties substitution) {
+        return get(new CompoundName(name), context, substitution);
+    }
+    public final Object get(CompoundName name, Map<String, String> context, Properties substitution) {
+        return substitute(entries.get(name, context), context, substitution);
+    }
+
+    private Object substitute(Object value, Map<String,String> context, Properties substitution) {
+        if (value == null) return value;
+        if (substitution == null) return value;
+        if (value.getClass() != SubstituteString.class) return value;
+        return ((SubstituteString)value).substitute(context, substitution);
+    }
+
+    /** Throws IllegalArgumentException if the given string is not a valid query profile name */
+    private static void validateName(String name) {
+        Matcher nameMatcher=namePattern.matcher(name);
+        if ( ! nameMatcher.matches())
+            throw new IllegalArgumentException("Illegal name '" + name + "'");
+    }
+
+    @Override
+    public CompiledQueryProfile clone() {
+        return this; // immutable
+    }
+
+    @Override
+    public String toString() {
+        return "query profile '" + getId()  + "'" + (type!=null ? " of type '" + type.getId() + "'" : "");
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java
new file mode 100644
index 00000000000..91a81888267
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java
@@ -0,0 +1,76 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.compiled;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry;
+
+/**
+ * A set of compiled query profiles.
+ *
+ * @author bratseth
+ */
+public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQueryProfile> {
+
+    private final QueryProfileTypeRegistry typeRegistry;
+
+    /** Creates a compiled query profile registry with no types */
+    public CompiledQueryProfileRegistry() {
+        this(QueryProfileTypeRegistry.emptyFrozen());
+    }
+
+    public CompiledQueryProfileRegistry(QueryProfileTypeRegistry typeRegistry) {
+        this.typeRegistry = typeRegistry;
+    }
+
+    /** Registers a type by its id */
+    public void register(CompiledQueryProfile profile) {
+        super.register(profile.getId(), profile);
+    }
+
+    public QueryProfileTypeRegistry getTypeRegistry() { return typeRegistry; }
+
+    /**
+     * <p>Returns a query profile for the given request string, or null if a suitable one is not found.</p>
+     *
+     * The request string must be a valid {@link com.yahoo.component.ComponentId} or null.<br>
+     * If the string is null, the profile named "default" is returned, or null if that does not exists.
+     *
+     * <p>
+     * The version part (if any) is matched used the usual component version patching rules.
+     * If the name part matches a query profile name perfectly, that profile is returned.
+     * If not, and the name is a slash-separated path, the profile with the longest matching left sub-path
+     * which has a type which allows path matching is used. If there is no such profile, null is returned.
+     */
+    public CompiledQueryProfile findQueryProfile(String idString) {
+        if (idString==null || idString.isEmpty()) return getComponent("default");
+        ComponentSpecification id=new ComponentSpecification(idString);
+        CompiledQueryProfile profile=getComponent(id);
+        if (profile!=null) return profile;
+
+        return findPathParentQueryProfile(new ComponentSpecification(idString));
+    }
+
+    private CompiledQueryProfile findPathParentQueryProfile(ComponentSpecification id) {
+        // Try the name with "/" appended - should have the same semantics with path matching
+        CompiledQueryProfile slashedProfile=getComponent(new ComponentSpecification(id.getName() + "/",id.getVersionSpecification()));
+        if (slashedProfile!=null && slashedProfile.getType()!=null && slashedProfile.getType().getMatchAsPath())
+            return slashedProfile;
+
+        // Extract the parent (if any)
+        int slashIndex=id.getName().lastIndexOf("/");
+        if (slashIndex<1) return null;
+        String parentName=id.getName().substring(0,slashIndex);
+        if (parentName.equals("")) return null;
+
+        ComponentSpecification parentId=new ComponentSpecification(parentName,id.getVersionSpecification());
+
+        CompiledQueryProfile pathParentProfile=getComponent(parentId);
+
+        if (pathParentProfile!=null && pathParentProfile.getType()!=null && pathParentProfile.getType().getMatchAsPath())
+            return pathParentProfile;
+        return findPathParentQueryProfile(parentId);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java
new file mode 100644
index 00000000000..b82939fa4ac
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalMap.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.compiled;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.search.query.profile.DimensionBinding;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A map which may return different values depending on the values given in a context
+ * supplied with the key on all operations.
+ * <p>
+ * Dimensional maps are immutable and created through a DimensionalMap.Builder
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DimensionalMap<KEY, VALUE> {
+
+    private final Map<KEY, DimensionalValue<VALUE>> values;
+
+    private DimensionalMap(Map<KEY, DimensionalValue<VALUE>> values) {
+        this.values = ImmutableMap.copyOf(values);
+    }
+
+    /** Returns the value for this key matching a context, or null if none */
+    public VALUE get(KEY key, Map<String, String> context) {
+        DimensionalValue<VALUE> variants = values.get(key);
+        if (variants == null) return null;
+        return variants.get(context);
+    }
+
+    /** Returns the set of dimensional entries across all contexts. */
+    public Set<Map.Entry<KEY, DimensionalValue<VALUE>>> entrySet() {
+        return values.entrySet();
+    }
+
+    /** Returns true if this is empty for all contexts. */
+    public boolean isEmpty() {
+        return values.isEmpty();
+    }
+
+    public static class Builder<KEY, VALUE> {
+
+        private Map<KEY, DimensionalValue.Builder<VALUE>> entries = new HashMap<>();
+
+        // TODO: DimensionBinding -> Binding?
+        public void put(KEY key, DimensionBinding binding, VALUE value) {
+            DimensionalValue.Builder<VALUE> entry = entries.get(key);
+            if (entry == null) {
+                entry = new DimensionalValue.Builder<>();
+                entries.put(key, entry);
+            }
+            entry.add(value, binding);
+        }
+
+        public DimensionalMap<KEY, VALUE> build() {
+            Map<KEY, DimensionalValue<VALUE>> map = new HashMap<>();
+            for (Map.Entry<KEY, DimensionalValue.Builder<VALUE>> entry : entries.entrySet()) {
+                map.put(entry.getKey(), entry.getValue().build());
+            }
+            return new DimensionalMap<>(map);
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java
new file mode 100644
index 00000000000..0112928ada6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java
@@ -0,0 +1,159 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.compiled;
+
+import com.yahoo.search.query.profile.DimensionBinding;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Contains the values a given key in a DimensionalMap may take for different dimensional contexts.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DimensionalValue<VALUE> {
+
+    private final List<Value<VALUE>> values;
+
+    /** Create a set of variants which is a single value regardless of dimensions */
+    public DimensionalValue(Value<VALUE> value) {
+        this.values = Collections.singletonList(value);
+    }
+
+    public DimensionalValue(List<Value<VALUE>> valueVariants) {
+        if (valueVariants.size() == 1) { // special cased for efficiency
+            this.values = Collections.singletonList(valueVariants.get(0));
+        }
+        else {
+            this.values = new ArrayList<>(valueVariants);
+            Collections.sort(this.values);
+        }
+    }
+
+    /** Returns the value matching this context, or null if none */
+    public VALUE get(Map<String, String> context) {
+        if (context == null)
+            context = Collections.emptyMap();
+        for (Value<VALUE> value : values) {
+            if (value.matches(context))
+                return value.value();
+        }
+        return null;
+    }
+
+    public boolean isEmpty() { return values.isEmpty(); }
+
+    @Override
+    public String toString() {
+        return values.toString();
+    }
+
+    public static class Builder<VALUE> {
+
+        /** The minimal set of variants needed to capture all values at this key */
+        private Map<VALUE, Value.Builder<VALUE>> buildableVariants = new HashMap<>();
+
+        public void add(VALUE value, DimensionBinding variantBinding) {
+            // Note: We know we can index by the value because its possible types are constrained
+            // to what query profiles allow: String, primitives and query profiles
+            Value.Builder variant = buildableVariants.get(value);
+            if (variant == null) {
+                variant = new Value.Builder<>(value);
+                buildableVariants.put(value, variant);
+            }
+            variant.addVariant(variantBinding);
+        }
+
+        public DimensionalValue<VALUE> build() {
+            List<Value> variants = new ArrayList<>();
+            for (Value.Builder buildableVariant : buildableVariants.values()) {
+                variants.addAll(buildableVariant.build());
+            }
+            return new DimensionalValue(variants);
+        }
+
+    }
+
+    /** A value for a particular binding */
+    private static class Value<VALUE> implements Comparable<Value> {
+
+        private VALUE value = null;
+
+        /** The minimal binding this holds for */
+        private Binding binding = null;
+
+        public Value(VALUE value, Binding binding) {
+            this.value = value;
+            this.binding = binding;
+        }
+
+        /** Returns the value at this entry or null if none */
+        public VALUE value() { return value; }
+
+        /** Returns the binding that must match for this to be a valid entry, or Binding.nullBinding if none */
+        public Binding binding() {
+            if (binding == null) return Binding.nullBinding;
+            return binding;
+        }
+
+        public boolean matches(Map<String, String> context) {
+            return binding.matches(context);
+        }
+
+        @Override
+        public int compareTo(Value other) {
+            return this.binding.compareTo(other.binding);
+        }
+
+        @Override
+        public String toString() {
+            return " value '" + value + "' for " + binding;
+        }
+
+        /**
+         * A single value with the minimal set of dimension combinations it holds for.
+         */
+        private static class Builder<VALUE> {
+
+            private final VALUE value;
+
+            /**
+             * The set of bindings this value is for.
+             * Some of these are more general versions of others.
+             * We need to keep both to allow interleaving a different value with medium generality.
+             */
+            private Set<DimensionBinding> variants = new HashSet<>();
+
+            public Builder(VALUE value) {
+                this.value = value;
+            }
+
+            /** Add a binding this holds for */
+            public void addVariant(DimensionBinding binding) {
+                variants.add(binding);
+            }
+
+            /** Build a separate value object for each dimension combination which has this value */
+            public List<Value<VALUE>> build() {
+                // Shortcut for efficiency of the normal case
+                if (variants.size()==1)
+                    return Collections.singletonList(new Value<>(value, Binding.createFrom(variants.iterator().next())));
+
+                List<Value<VALUE>> values = new ArrayList<>(variants.size());
+                for (DimensionBinding variant : variants)
+                    values.add(new Value<>(value, Binding.createFrom(variant)));
+                return values;
+            }
+
+            public Object value() {
+                return value;
+            }
+
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java
new file mode 100644
index 00000000000..5770665e3a1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java
@@ -0,0 +1,227 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.config;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.subscription.ConfigSubscriber;
+import com.yahoo.search.query.profile.DimensionValues;
+import com.yahoo.search.query.profile.QueryProfile;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry;
+import com.yahoo.text.BooleanParser;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author bratseth
+ */
+public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber<QueryProfilesConfig> {
+
+    private final ConfigSubscriber subscriber = new ConfigSubscriber();
+
+    private volatile QueryProfileRegistry currentRegistry;
+
+    public QueryProfileConfigurer(String configId) {
+        subscriber.subscribe(this, QueryProfilesConfig.class, configId);
+    }
+
+    /** Returns the registry created by the last occurring call to configure */
+    public QueryProfileRegistry getCurrentRegistry() { return currentRegistry; }
+
+    private void setCurrentRegistry(QueryProfileRegistry registry) {
+        this.currentRegistry=registry;
+    }
+
+    public void configure(QueryProfilesConfig config) {
+        QueryProfileRegistry registry = createFromConfig(config);
+        setCurrentRegistry(registry);
+    }
+
+    public static QueryProfileRegistry createFromConfig(QueryProfilesConfig config) {
+        QueryProfileRegistry registry=new QueryProfileRegistry();
+
+        // Pass 1: Create all profiles and profile types
+        for (QueryProfilesConfig.Queryprofiletype profileTypeConfig : config.queryprofiletype()) {
+            createProfileType(profileTypeConfig,registry.getTypeRegistry());
+        }
+        for (QueryProfilesConfig.Queryprofile profileConfig : config.queryprofile()) {
+            createProfile(profileConfig,registry);
+        }
+
+        // Pass 2: Resolve references and add content
+        for (QueryProfilesConfig.Queryprofiletype profileTypeConfig : config.queryprofiletype()) {
+            fillProfileType(profileTypeConfig,registry.getTypeRegistry());
+        }
+
+        // To ensure topological sorting, using DPS. This will _NOT_ detect cycles (but it will not fail if they
+        // exist either)
+        Set<ComponentId> filled = new HashSet<>();
+        for (QueryProfilesConfig.Queryprofile profileConfig : config.queryprofile()) {
+            fillProfile(profileConfig, config, registry, filled);
+        }
+
+        registry.freeze();
+        return registry;
+    }
+
+    /** Stop subscribing from this configurer */
+    public void shutdown() {
+        subscriber.close();
+    }
+
+    private static void createProfile(QueryProfilesConfig.Queryprofile config,QueryProfileRegistry registry) {
+        QueryProfile profile=new QueryProfile(config.id());
+        try {
+            String typeId=config.type();
+            if (typeId!=null && !typeId.isEmpty())
+                profile.setType(registry.getType(typeId));
+
+            if (config.dimensions().size()>0) {
+                String[] dimensions=new String[config.dimensions().size()];
+                for (int i=0; i<config.dimensions().size(); i++)
+                    dimensions[i]=config.dimensions().get(i);
+                profile.setDimensions(dimensions);
+            }
+
+            registry.register(profile);
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Invalid " + profile,e);
+        }
+    }
+
+    private static void createProfileType(QueryProfilesConfig.Queryprofiletype config, QueryProfileTypeRegistry registry) {
+        QueryProfileType type=new QueryProfileType(config.id());
+        type.setStrict(config.strict());
+        type.setMatchAsPath(config.matchaspath());
+        registry.register(type);
+    }
+
+    private static void fillProfile(QueryProfilesConfig.Queryprofile config,
+                                    QueryProfilesConfig queryProfilesConfig,
+                                    QueryProfileRegistry registry,
+                                    Set<ComponentId> filled) {
+        QueryProfile profile=registry.getComponent(new ComponentSpecification(config.id()).toId());
+        if (filled.contains(profile.getId())) return;
+        filled.add(profile.getId());
+        try {
+            for (String inheritedId : config.inherit()) {
+                QueryProfile inherited=registry.getComponent(inheritedId);
+                if (inherited==null)
+                    throw new IllegalArgumentException("Inherited query profile '" + inheritedId + "' in " + profile + " was not found");
+                fillProfile(inherited, queryProfilesConfig, registry, filled);
+                profile.addInherited(inherited);
+            }
+
+            for (QueryProfilesConfig.Queryprofile.Reference referenceConfig : config.reference()) {
+                QueryProfile referenced=registry.getComponent(referenceConfig.value());
+                if (referenced==null)
+                    throw new IllegalArgumentException("Query profile '" + referenceConfig.value() + "' referenced as '" +
+                            referenceConfig.name() + "' in " + profile + " was not found");
+                profile.set(referenceConfig.name(),referenced, registry);
+                if (referenceConfig.overridable()!=null && !referenceConfig.overridable().isEmpty())
+                    profile.setOverridable(referenceConfig.name(),BooleanParser.parseBoolean(referenceConfig.overridable()),null);
+            }
+
+            for (QueryProfilesConfig.Queryprofile.Property propertyConfig : config.property()) {
+                profile.set(propertyConfig.name(),propertyConfig.value(), registry);
+                if (propertyConfig.overridable()!=null && !propertyConfig.overridable().isEmpty())
+                    profile.setOverridable(propertyConfig.name(),BooleanParser.parseBoolean(propertyConfig.overridable()),null);
+            }
+
+            for (QueryProfilesConfig.Queryprofile.Queryprofilevariant variantConfig : config.queryprofilevariant()) {
+                String[] forDimensionValueArray=new String[variantConfig.fordimensionvalues().size()];
+                for (int i=0; i<variantConfig.fordimensionvalues().size(); i++) {
+                    forDimensionValueArray[i]=variantConfig.fordimensionvalues().get(i).trim();
+                    if ("*".equals(forDimensionValueArray[i]))
+                        forDimensionValueArray[i]=null;
+                }
+                DimensionValues forDimensionValues=DimensionValues.createFrom(forDimensionValueArray);
+
+                for (String inheritedId : variantConfig.inherit()) {
+                    QueryProfile inherited=registry.getComponent(inheritedId);
+                    if (inherited==null)
+                        throw new IllegalArgumentException("Inherited query profile '" + inheritedId + "' in " + profile +
+                                                           " for '" + forDimensionValues + "' was not found");
+                    fillProfile(inherited, queryProfilesConfig, registry, filled);
+                    profile.addInherited(inherited, forDimensionValues);
+                }
+
+                for (QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference referenceConfig : variantConfig.reference()) {
+                    QueryProfile referenced=registry.getComponent(referenceConfig.value());
+                    if (referenced==null)
+                        throw new IllegalArgumentException("Query profile '" + referenceConfig.value() + "' referenced as '" +
+                                referenceConfig.name() + "' in " + profile + " for '" + forDimensionValues + "' was not found");
+                    profile.set(referenceConfig.name(), referenced, forDimensionValues, registry);
+                }
+
+                for (QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property propertyConfig : variantConfig.property()) {
+                    profile.set(propertyConfig.name(), propertyConfig.value(), forDimensionValues, registry);
+                }
+
+            }
+
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Invalid " + profile,e);
+        }
+    }
+
+    /** Fill a given profile by locating its config */
+    private static void fillProfile(QueryProfile inherited,
+                             QueryProfilesConfig queryProfilesConfig,
+                             QueryProfileRegistry registry,
+                             Set<ComponentId> visited) {
+        for (QueryProfilesConfig.Queryprofile inheritedConfig : queryProfilesConfig.queryprofile()) {
+            if (inherited.getId().stringValue().equals(inheritedConfig.id())) {
+                fillProfile(inheritedConfig, queryProfilesConfig, registry, visited);
+            }
+        }
+    }
+
+    private static void fillProfileType(QueryProfilesConfig.Queryprofiletype config,QueryProfileTypeRegistry registry) {
+        QueryProfileType type=registry.getComponent(new ComponentSpecification(config.id()).toId());
+        try {
+
+            for (String inheritedId : config.inherit()) {
+                QueryProfileType inherited=registry.getComponent(inheritedId);
+                if (inherited==null)
+                    throw new IllegalArgumentException("Inherited query profile type '" + inheritedId + "' in " + type + " was not found");
+                else
+                    type.inherited().add(inherited);
+
+            }
+
+            for (QueryProfilesConfig.Queryprofiletype.Field fieldConfig : config.field())
+                instantiateFieldDescription(fieldConfig,type,registry);
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Invalid " + type,e);
+        }
+    }
+
+    private static void instantiateFieldDescription(QueryProfilesConfig.Queryprofiletype.Field fieldConfig,
+                                             QueryProfileType type,
+                                             QueryProfileTypeRegistry registry) {
+        try {
+            FieldType fieldType=FieldType.fromString(fieldConfig.type(),registry);
+            FieldDescription field=new FieldDescription(
+                    fieldConfig.name(),
+                    fieldType,
+                    fieldConfig.alias(),
+                    fieldConfig.mandatory(),
+                    fieldConfig.overridable()
+            );
+            type.addField(field, registry);
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Invalid field '" + fieldConfig.name() + "' in " + type,e);
+        }
+    }
+
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java
new file mode 100644
index 00000000000..97e3fb90dc9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java
@@ -0,0 +1,366 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.config;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.query.profile.DimensionValues;
+import com.yahoo.search.query.profile.QueryProfile;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry;
+import com.yahoo.text.XML;
+import org.w3c.dom.Element;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * A class which imports query profiles and types from XML files
+ *
+ * @author bratseth
+ */
+public class QueryProfileXMLReader {
+
+    private static Logger logger=Logger.getLogger(QueryProfileXMLReader.class.getName());
+
+    /**
+     * Reads all query profile xml files in a given directory,
+     * and all type xml files from the immediate subdirectory "types/" (if any)
+     *
+     * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML
+     */
+    public QueryProfileRegistry read(String directory) {
+        List<NamedReader> queryProfileReaders=new ArrayList<>();
+        List<NamedReader> queryProfileTypeReaders=new ArrayList<>();
+        try {
+            File dir=new File(directory);
+            if ( !dir.isDirectory() ) throw new IllegalArgumentException("Could not read query profiles: '" +
+                                                                         directory + "' is not a valid directory.");
+
+            for (File file : sortFiles(dir)) {
+                if ( ! file.getName().endsWith(".xml")) continue;
+                queryProfileReaders.add(new NamedReader(file.getName(),new FileReader(file)));
+            }
+            File typeDir=new File(dir,"types");
+            if (typeDir.isDirectory()) {
+                for (File file : sortFiles(typeDir)) {
+                    if ( ! file.getName().endsWith(".xml")) continue;
+                    queryProfileTypeReaders.add(new NamedReader(file.getName(),new FileReader(file)));
+                }
+            }
+
+            return read(queryProfileTypeReaders,queryProfileReaders);
+        }
+        catch (IOException e) {
+            throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'",e);
+        }
+        finally {
+            closeAll(queryProfileReaders);
+            closeAll(queryProfileTypeReaders);
+        }
+    }
+
+    private List<File> sortFiles(File dir) {
+        ArrayList<File> files = new ArrayList<>();
+        files.addAll(Arrays.asList(dir.listFiles()));
+        Collections.sort(files);
+        return files;
+    }
+
+    private void closeAll(List<NamedReader> readers) {
+        for (NamedReader reader : readers) {
+            try { reader.close(); } catch (IOException e) { }
+        }
+    }
+
+    /**
+     * Read the XML file readers into a registry. This does not close the readers.
+     * This method is used directly from the admin system.
+     */
+    public QueryProfileRegistry read(List<NamedReader> queryProfileTypeReaders,List<NamedReader> queryProfileReaders) {
+        QueryProfileRegistry registry=new QueryProfileRegistry();
+
+        // Phase 1
+        List<Element> queryProfileTypeElements=createQueryProfileTypes(queryProfileTypeReaders,registry.getTypeRegistry());
+        List<Element> queryProfileElements=createQueryProfiles(queryProfileReaders,registry);
+
+        // Phase 2
+        fillQueryProfileTypes(queryProfileTypeElements,registry.getTypeRegistry());
+        fillQueryProfiles(queryProfileElements,registry);
+        return registry;
+    }
+
+    public List<Element> createQueryProfileTypes(List<NamedReader> queryProfileTypeReaders, QueryProfileTypeRegistry registry) {
+        List<Element> queryProfileTypeElements=new ArrayList<>(queryProfileTypeReaders.size());
+        for (NamedReader reader : queryProfileTypeReaders) {
+            Element root=XML.getDocument(reader).getDocumentElement();
+            if ( ! root.getNodeName().equals("query-profile-type")) {
+                logger.info("Ignoring '" + reader.getName() +
+                            "': Expected XML root element 'query-profile-type' but was '" + root.getNodeName() + "'");
+                continue;
+            }
+
+            String idString=root.getAttribute("id");
+            if (idString==null || idString.equals(""))
+                throw new IllegalArgumentException("'" + reader.getName() + "' has no 'id' attribute in the root element");
+            ComponentId id=new ComponentId(idString);
+            validateFileNameToId(reader.getName(),id,"query profile type");
+            QueryProfileType type=new QueryProfileType(id);
+            type.setMatchAsPath(XML.getChild(root,"match") != null);
+            type.setStrict(XML.getChild(root,"strict") != null);
+            registry.register(type);
+            queryProfileTypeElements.add(root);
+        }
+        return queryProfileTypeElements;
+    }
+
+    public List<Element> createQueryProfiles(List<NamedReader> queryProfileReaders, QueryProfileRegistry registry) {
+        List<Element> queryProfileElements=new ArrayList<>(queryProfileReaders.size());
+        for (NamedReader reader : queryProfileReaders) {
+            Element root=XML.getDocument(reader).getDocumentElement();
+            if ( ! root.getNodeName().equals("query-profile")) {
+                logger.info("Ignoring '" + reader.getName() +
+                            "': Expected XML root element 'query-profile' but was '" + root.getNodeName() + "'");
+                continue;
+            }
+
+            String idString=root.getAttribute("id");
+            if (idString==null || idString.equals(""))
+                throw new IllegalArgumentException("Query profile '" + reader.getName() + "' has no 'id' attribute in the root element");
+            ComponentId id=new ComponentId(idString);
+            validateFileNameToId(reader.getName(),id,"query profile");
+
+            QueryProfile queryProfile=new QueryProfile(id);
+            String typeId=root.getAttribute("type");
+            if (typeId!=null && ! typeId.equals("")) {
+                QueryProfileType type=registry.getType(typeId);
+                if (type==null)
+                    throw new IllegalArgumentException("Query profile '" + reader.getName() + "': Type id '" + typeId + "' can not be resolved");
+                queryProfile.setType(type);
+            }
+
+            Element dimensions=XML.getChild(root,"dimensions");
+            if (dimensions!=null)
+                queryProfile.setDimensions(toArray(XML.getValue(dimensions)));
+
+            registry.register(queryProfile);
+            queryProfileElements.add(root);
+        }
+        return queryProfileElements;
+    }
+
+    /** Throws an exception if the name is not corresponding to the id */
+    private void validateFileNameToId(final String actualName,ComponentId id,String artifactName) {
+        String expectedCanonicalFileName=id.toFileName();
+        String expectedAlternativeFileName=id.stringValue().replace(":","-").replace("/","_"); // legacy
+        String fileName=new File(actualName).getName();
+        fileName=stripXmlEnding(fileName);
+        String canonicalFileName=ComponentId.fromFileName(fileName).toFileName();
+        if ( ! canonicalFileName.equals(expectedCanonicalFileName) && ! canonicalFileName.equals(expectedAlternativeFileName))
+            throw new IllegalArgumentException("The file name of " + artifactName + " '" + id +
+                                               "' must be '" + expectedCanonicalFileName + ".xml' but was '" + actualName + "'");
+    }
+
+    private String stripXmlEnding(String fileName) {
+        if (!fileName.endsWith(".xml"))
+            throw new IllegalArgumentException("'" + fileName + "' should have a .xml ending");
+        else
+            return fileName.substring(0,fileName.length()-4);
+    }
+
+    private String[] toArray(String csv) {
+        String[] array=csv.split(",");
+        for (int i=0; i<array.length; i++)
+            array[i]=array[i].trim();
+        return array;
+    }
+
+    public void fillQueryProfileTypes(List<Element> queryProfileTypeElements, QueryProfileTypeRegistry registry) {
+        for (Element element : queryProfileTypeElements) {
+            QueryProfileType type=registry.getComponent(new ComponentSpecification(element.getAttribute("id")).toId());
+            try {
+                readInheritedTypes(element,type,registry);
+                readFieldDefinitions(element,type,registry);
+            }
+            catch (RuntimeException e) {
+                throw new IllegalArgumentException("Error reading " + type,e);
+            }
+        }
+    }
+
+    private void readInheritedTypes(Element element,QueryProfileType type,QueryProfileTypeRegistry registry) {
+        String inheritedString=element.getAttribute("inherits");
+        if (inheritedString==null || inheritedString.equals("")) return;
+        for (String inheritedId : inheritedString.split(" ")) {
+            inheritedId=inheritedId.trim();
+            if (inheritedId.equals("")) continue;
+            QueryProfileType inheritedType=registry.getComponent(inheritedId);
+            if (inheritedType==null) throw new IllegalArgumentException("Could not resolve inherited query profile type '" + inheritedId);
+            type.inherited().add(inheritedType);
+        }
+    }
+
+    private void readFieldDefinitions(Element element,QueryProfileType type,QueryProfileTypeRegistry registry) {
+        for (Element field : XML.getChildren(element,"field")) {
+            String name=field.getAttribute("name");
+            if (name==null || name.equals("")) throw new IllegalArgumentException("A field has no 'name' attribute");
+            try {
+                String fieldTypeName=field.getAttribute("type");
+                if (fieldTypeName==null) throw new IllegalArgumentException("Field '" + field + "' has no 'type' attribute");
+                FieldType fieldType=FieldType.fromString(fieldTypeName,registry);
+                type.addField(new FieldDescription(name,fieldType,field.getAttribute("alias"),
+                              getBooleanAttribute("mandatory",false,field),getBooleanAttribute("overridable",true,field)), registry);
+            }
+            catch(RuntimeException e) {
+                throw new IllegalArgumentException("Invalid field '" + name + "'",e);
+            }
+        }
+    }
+
+    public void fillQueryProfiles(List<Element> queryProfileElements, QueryProfileRegistry registry) {
+        for (Element element : queryProfileElements) {
+            // Lookup by exact id
+            QueryProfile profile=registry.getComponent(new ComponentSpecification(element.getAttribute("id")).toId());
+            try {
+                readInherited(element,profile,registry,null,profile.toString());
+                readFields(element,profile,registry,null,profile.toString());
+                readVariants(element,profile,registry);
+            }
+            catch (RuntimeException e) {
+                throw new IllegalArgumentException("Error reading " + profile,e);
+            }
+        }
+    }
+
+    private void readInherited(Element element,QueryProfile profile,QueryProfileRegistry registry,DimensionValues dimensionValues,String sourceDescription) {
+        String inheritedString=element.getAttribute("inherits");
+        if (inheritedString==null || inheritedString.equals("")) return;
+        for (String inheritedId : inheritedString.split(" ")) {
+            inheritedId=inheritedId.trim();
+            if (inheritedId.equals("")) continue;
+            QueryProfile inheritedProfile=registry.getComponent(inheritedId);
+            if (inheritedProfile==null) throw new IllegalArgumentException("Could not resolve inherited query profile '" + inheritedId + "' in " + sourceDescription);
+            profile.addInherited(inheritedProfile,dimensionValues);
+        }
+    }
+
+    private void readFields(Element element,QueryProfile profile,QueryProfileRegistry registry,DimensionValues dimensionValues,String sourceDescription) {
+        List<KeyValue> references=new ArrayList<>();
+        List<KeyValue> properties=new ArrayList<>();
+        for (Element field : XML.getChildren(element,"field")) {
+            String name=field.getAttribute("name");
+            if (name==null || name.equals("")) throw new IllegalArgumentException("A field in " + sourceDescription + " has no 'name' attribute");
+            try {
+                Boolean overridable=getBooleanAttribute("overridable",null,field);
+                if (overridable!=null)
+                    profile.setOverridable(name,overridable,null);
+
+                Object fieldValue=readFieldValue(field,name,sourceDescription,registry);
+                if (fieldValue instanceof QueryProfile)
+                    references.add(new KeyValue(name,fieldValue));
+                else
+                    properties.add(new KeyValue(name,fieldValue));
+            }
+            catch (IllegalArgumentException e) {
+                throw new IllegalArgumentException("Invalid field '" + name + "' in " + sourceDescription,e);
+            }
+        }
+        // Must set references before properties
+        for (KeyValue keyValue : references)
+            profile.set(keyValue.getKey() ,keyValue.getValue(), dimensionValues, registry);
+        for (KeyValue keyValue : properties)
+            profile.set(keyValue.getKey(), keyValue.getValue(), dimensionValues, registry);
+
+    }
+
+    private Object readFieldValue(Element field,String name,String targetDescription,QueryProfileRegistry registry) {
+        Element ref=XML.getChild(field,"ref");
+        if (ref!=null) {
+            String referencedName=XML.getValue(ref);
+            QueryProfile referenced=registry.getComponent(referencedName);
+            if (referenced==null)
+                throw new IllegalArgumentException("Could not find query profile '" + referencedName + "' referenced as '" +
+                                                   name + "' in " + targetDescription);
+            return referenced;
+        }
+        else {
+            return XML.getValue(field);
+        }
+    }
+
+    private void readVariants(Element element,QueryProfile profile,QueryProfileRegistry registry) {
+        for (Element queryProfileVariantElement : XML.getChildren(element,"query-profile")) { // A "virtual" query profile contained inside another
+            List<String> dimensions=profile.getDimensions();
+            if (dimensions==null)
+                throw new IllegalArgumentException("Cannot create a query profile variant in " + profile +
+                                                   ", as it has not declared any variable dimensions");
+            String dimensionString=queryProfileVariantElement.getAttribute("for");
+            String[] dimensionValueArray=makeStarsNull(toArray(dimensionString));
+            if (dimensions.size()<dimensionValueArray.length)
+                throw new IllegalArgumentException("Cannot create a query profile variant for '" + dimensionString +
+                                                   "' as only " + dimensions.size() + " dimensions has been defined");
+            DimensionValues dimensionValues=DimensionValues.createFrom(dimensionValueArray);
+
+            String description="variant '" + dimensionString + "' in " + profile.toString();
+            readInherited(queryProfileVariantElement,profile,registry,dimensionValues,description);
+            readFields(queryProfileVariantElement,profile,registry,dimensionValues,description);
+        }
+    }
+
+    private String[] makeStarsNull(String[] strings) {
+        for (int i=0; i<strings.length; i++)
+            if (strings[i].equals("*"))
+                strings[i]=null;
+        return strings;
+    }
+
+    /**
+     * Returns true if the string is "true".<br>
+     * Returns false if the string is "false".<br>
+     * Returns <code>default</code> if the string is null or empty (this parameter may be null)<br>
+     * @throws IllegalArgumentException if the string has any other value
+     */
+    private Boolean asBoolean(String s,Boolean defaultValue) {
+        if (s==null) return defaultValue;
+        if (s.isEmpty()) return defaultValue;
+        if ("true".equals(s)) return true;
+        if ("false".equals(s)) return false;
+        throw new IllegalArgumentException("Expected 'true' or 'false' but was'" + s + "'");
+    }
+
+    /** Returns the given attribute as a boolean, using the semantics of {@link #asBoolean} */
+    private Boolean getBooleanAttribute(String attributeName,Boolean defaultValue,Element from) {
+        try {
+            return asBoolean(from.getAttribute(attributeName),defaultValue);
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Attribute '" + attributeName,e);
+        }
+    }
+
+    private static class KeyValue {
+
+        private String key;
+        private Object value;
+
+        public KeyValue(String key,Object value) {
+            this.key=key;
+            this.value=value;
+        }
+
+        public String getKey() { return key; }
+
+        public Object getValue() { return value; }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/package-info.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/package-info.java
new file mode 100644
index 00000000000..8ea4e887661
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.search.query.profile.config;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/package-info.java b/container-search/src/main/java/com/yahoo/search/query/profile/package-info.java
new file mode 100644
index 00000000000..df3f4ac45ab
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/package-info.java
@@ -0,0 +1,12 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Query Profiles provide nested sets of named (and optionally typed) key-values which can be referenced in a Query
+ * to proviode initial values of Query properties. Values in nested query profiles can be looked up from
+ * the query properties by dotting the names. Query profiles supports inheritance to allow variations
+ * for, e.g different buckets, client types, markets etc. */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.profile;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
new file mode 100644
index 00000000000..c522ec04023
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
@@ -0,0 +1,148 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.QueryProfile;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A field description of a query profile type. Immutable.
+ * Field descriptions can be sorted by name.
+ *
+ * @author bratseth
+ */
+public class FieldDescription implements Comparable<FieldDescription> {
+
+    private final CompoundName name;
+    private final FieldType type;
+    private final List<String> aliases;
+
+    /** If true, this value must be provided either in the query profile or in the search request */
+    private final boolean mandatory;
+
+    /** If true, assignments to this value from outside will be ignored */
+    private final boolean overridable;
+
+    public FieldDescription(String name, FieldType type) {
+        this(name,type,false);
+    }
+
+    public FieldDescription(String name, String type) {
+        this(name,FieldType.fromString(type,null));
+    }
+
+    public FieldDescription(String name, FieldType type, boolean mandatory) {
+        this(name, type, mandatory, true);
+    }
+
+    public FieldDescription(String name, String type, String aliases) {
+        this(name,type,aliases,false,true);
+    }
+
+    public FieldDescription(String name, FieldType type, String aliases) {
+        this(name, type, aliases, false, true);
+    }
+
+    /**
+     * Creates a field description
+     *
+     * @param name the name of the field
+     * @param typeString the type of the field represented as a string - see {@link com.yahoo.search.query.profile.types.FieldType}
+     * @param aliases a space-separated list of alias names of this field name. Aliases are not following dotted
+     *        (meaning they are global, not that they cannot contain dots) and are case insensitive. Null is permissible
+     *        if there are no aliases
+     * @param mandatory whether it is mandatory to provide a value for this field. default: false
+     * @param overridable whether this can be overridden when first set in a profile. Default: true
+     */
+    public FieldDescription(String name, String typeString, String aliases, boolean mandatory, boolean overridable) {
+        this(name,FieldType.fromString(typeString,null),aliases,mandatory,overridable);
+    }
+
+    public FieldDescription(String name, FieldType type, boolean mandatory, boolean overridable) {
+        this(name, type, null, mandatory, overridable);
+    }
+
+    public FieldDescription(String name, FieldType type, String aliases, boolean mandatory, boolean overridable) {
+        this(new CompoundName(name), type, aliases, mandatory, overridable);
+    }
+
+    /**
+     * Creates a field description from a list where the aliases are represented as a comma-separated string
+     */
+    public FieldDescription(CompoundName name, FieldType type, String aliases, boolean mandatory, boolean overridable) {
+        this(name, type, toList(aliases), mandatory, overridable);
+    }
+
+    /**
+     * Creates a field description
+     *
+     * @param name the name of the field
+     * @param type the type of the field represented as a string - see {@link com.yahoo.search.query.profile.types.FieldType}
+     * @param aliases a list of aliases, never null. Aliases are not following dotted
+     *        (meaning they are global, not that they cannot contain dots) and are case insensitive.
+     * @param mandatory whether it is mandatory to provide a value for this field. default: false
+     * @param overridable whether this can be overridden when first set in a profile. Default: true
+     */
+    public FieldDescription(CompoundName name, FieldType type, List<String> aliases, boolean mandatory, boolean overridable) {
+        if (name.isEmpty())
+            throw new IllegalArgumentException("Illegal name ''");
+        for (String nameComponent : name.asList())
+            QueryProfile.validateName(nameComponent);
+        this.name = name;
+        this.type = type;
+
+        // Forbidden until we can figure out the right semantics
+        if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases is not allowed with compound names");
+
+        this.aliases = ImmutableList.copyOf(aliases);
+        this.mandatory = mandatory;
+        this.overridable = overridable;
+    }
+
+    private static List<String> toList(String string) {
+        if (string == null || string.isEmpty()) return ImmutableList.of();
+        return ImmutableList.copyOf(Arrays.asList(string.split(" ")));
+    }
+
+    /** Returns the full name of this as a string */
+    public String getName() { return name.toString(); }
+
+    /** Returns the full name of this as a compound name */
+    public CompoundName getCompoundName() { return name; }
+
+    public FieldType getType() { return type; }
+
+    /** Returns a unmodifiable list of the aliases of this. An empty list (never null) if there are none. */
+    public List<String> getAliases() { return aliases; }
+
+    /** Returns whether this field must be provided in the query profile or the search definition. Default: false */
+    public boolean isMandatory() { return mandatory; }
+
+    /** Returns false if overrides to values for this field from the outside should be ignored. Default: true */
+    public boolean isOverridable() { return overridable; }
+
+    public int compareTo(FieldDescription other) {
+        return name.toString().compareTo(other.name.toString());
+    }
+
+    /** Returns a copy of this with the name set to the argument name */
+    public FieldDescription withName(CompoundName name) {
+        return new FieldDescription(name, type, aliases, mandatory, overridable);
+    }
+
+    /** Returns a copy of this with the type set to the argument type */
+    public FieldDescription withType(FieldType type) {
+        return new FieldDescription(name, type, aliases, mandatory, overridable);
+    }
+
+    @Override
+    public String toString() {
+        return "field '" + name + "' type " + type.stringValue() + "" +
+                (mandatory?" (mandatory)":"") + (!overridable?" (not overridable)":"");
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java
new file mode 100644
index 00000000000..abe3c4425ae
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.search.query.profile.QueryProfile;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.yql.YqlQuery;
+import com.yahoo.tensor.Tensor;
+
+import java.util.Optional;
+
+/**
+ * Superclass of query type field types.
+ * Field types are immutable.
+ *
+ * @author bratseth
+ */
+@SuppressWarnings("rawtypes")
+public abstract class FieldType {
+
+    public static final PrimitiveFieldType stringType = new PrimitiveFieldType(String.class);
+    public static final PrimitiveFieldType integerType = new PrimitiveFieldType(Integer.class);
+    public static final PrimitiveFieldType longType = new PrimitiveFieldType(Long.class);
+    public static final PrimitiveFieldType floatType = new PrimitiveFieldType(Float.class);
+    public static final PrimitiveFieldType doubleType = new PrimitiveFieldType(Double.class);
+    public static final PrimitiveFieldType booleanType = new PrimitiveFieldType(Boolean.class);
+    public static final TensorFieldType genericTensorType = new TensorFieldType(Optional.empty());
+    public static final QueryFieldType queryType = new QueryFieldType();
+    public static final QueryProfileFieldType genericQueryProfileType = new QueryProfileFieldType();
+
+    /** Returns the class of instance values of this field type */
+    public abstract Class getValueClass();
+
+    /** Returns a string representation of this type which can be converted back to a type class by {@link #fromString} */
+    public abstract String stringValue();
+
+    public abstract String toString();
+
+    /** Returns a string describing possible instances of this type, suitable for user error messages */
+    public abstract String toInstanceDescription();
+
+    /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */
+    public abstract Object convertFrom(Object o, QueryProfileRegistry registry);
+
+    /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */
+    public abstract Object convertFrom(Object o, CompiledQueryProfileRegistry registry);
+
+    /**
+     * Returns the field type for a given string name.
+     *
+     * @param  typeString a type string - a primitive name, "query-profile" or "query-profile:profile-name"
+     * @param  registry the registry in which query profile references are resolved when the last form above is used,
+     *         or null in which case that form cannot be used
+     * @throws IllegalArgumentException if the string does not resolve to a type
+     */
+    public static FieldType fromString(String typeString, QueryProfileTypeRegistry registry) {
+        if ("string".equals(typeString))
+            return stringType;
+        if ("integer".equals(typeString))
+            return integerType;
+        if ("long".equals(typeString))
+            return longType;
+        if ("float".equals(typeString))
+            return floatType;
+        if ("double".equals(typeString))
+            return doubleType;
+        if ("boolean".equals(typeString))
+            return booleanType;
+        if ("query".equals(typeString))
+            return queryType;
+        if (typeString.startsWith("tensor"))
+            return TensorFieldType.fromTypeString(typeString);
+        if ("query-profile".equals(typeString))
+            return genericQueryProfileType;
+        if (typeString.startsWith("query-profile:"))
+            return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()),registry);
+        throw new IllegalArgumentException("Unknown type '" + typeString + "'");
+    }
+
+    /** Returns the field type from a value class, or null if there is no type for it */
+    public static FieldType fromClass(Class clazz) {
+        if (clazz == String.class) return stringType;
+        if (clazz == Integer.class) return integerType;
+        if (clazz == Long.class) return longType;
+        if (clazz == Float.class) return floatType;
+        if (clazz == Double.class) return doubleType;
+        if (clazz == Boolean.class) return booleanType;
+        if (clazz == Tensor.class) return genericTensorType;
+        if (clazz == YqlQuery.class) return queryType;
+        if (clazz == QueryProfile.class) return genericQueryProfileType;
+        return null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java
new file mode 100644
index 00000000000..76b3f78ac2f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * Represents a query field type which is a primitive - String, Integer, Float, Double or Long.
+ *
+ * @author bratseth
+ */
+@SuppressWarnings("rawtypes")
+public class PrimitiveFieldType extends FieldType {
+
+    private Class primitiveClass;
+
+    PrimitiveFieldType(Class primitiveClass) {
+        this.primitiveClass=primitiveClass;
+    }
+
+    public @Override Class getValueClass() { return primitiveClass; }
+
+    public @Override String stringValue() {
+        return toLowerCase(primitiveClass.getSimpleName());
+    }
+
+    public @Override String toString() { return "field type " + stringValue(); }
+
+    public @Override String toInstanceDescription() {
+        return toLowerCase(primitiveClass.getSimpleName());
+    }
+
+    @Override
+    public Object convertFrom(Object object, CompiledQueryProfileRegistry registry) {
+        return convertFrom(object, (QueryProfileRegistry)null);
+    }
+
+    public @Override Object convertFrom(Object object, QueryProfileRegistry registry) {
+        if (primitiveClass == object.getClass()) return object;
+
+        if (object.getClass() == String.class) return convertFromString((String)object);
+        if (object instanceof Number) return convertFromNumber((Number)object);
+
+        return null;
+    }
+
+    private Object convertFromString(String string) {
+        try {
+            if (primitiveClass==Integer.class) return Integer.valueOf(string);
+            if (primitiveClass==Double.class) return Double.valueOf(string);
+            if (primitiveClass==Float.class) return Float.valueOf(string);
+            if (primitiveClass==Long.class) return Long.valueOf(string);
+            if (primitiveClass==Boolean.class) return Boolean.valueOf(string);
+        }
+        catch (NumberFormatException e) {
+            return null; // Handled in caller
+        }
+        throw new RuntimeException("Programming error");
+    }
+
+    private Object convertFromNumber(Number number) {
+        if (primitiveClass==Integer.class) return number.intValue();
+        if (primitiveClass==Double.class) return number.doubleValue();
+        if (primitiveClass==Float.class) return number.floatValue();
+        if (primitiveClass==Long.class) return number.longValue();
+        if (primitiveClass==String.class) return String.valueOf(number);
+        throw new RuntimeException("Programming error: Input type is " + number.getClass() +
+                                   " primitiveClass is " + primitiveClass);
+    }
+
+    @Override
+    public int hashCode() {
+        return primitiveClass.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof PrimitiveFieldType)) return false;
+        PrimitiveFieldType other = (PrimitiveFieldType)o;
+        return other.primitiveClass.equals(this.primitiveClass);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java
new file mode 100644
index 00000000000..a0982fdf0f6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.yql.YqlQuery;
+import com.yahoo.tensor.MapTensor;
+import com.yahoo.tensor.Tensor;
+
+/**
+ * A YQL query template field type in a query profile
+ *
+ * @author bratseth
+ */
+public class QueryFieldType extends FieldType {
+
+    @Override
+    public Class getValueClass() { return YqlQuery.class; }
+
+    @Override
+    public String stringValue() { return "query"; }
+
+    @Override
+    public String toString() { return "field type " + stringValue(); }
+
+    @Override
+    public String toInstanceDescription() { return "a YQL query template"; }
+
+    @Override
+    public Object convertFrom(Object o, QueryProfileRegistry registry) {
+        if (o instanceof YqlQuery) return o;
+        if (o instanceof String) return YqlQuery.from((String)o);
+        return null;
+    }
+
+    @Override
+    public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) {
+        return convertFrom(o, (QueryProfileRegistry)null);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
new file mode 100644
index 00000000000..df52e78c6ef
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
@@ -0,0 +1,100 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.search.query.profile.QueryProfile;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+
+/**
+ * Represents a query profile field type which is a reference to a query profile.
+ * The reference may optionally specify the type of the referred query profile.
+ *
+ * @author bratseth
+ */
+public class QueryProfileFieldType extends FieldType {
+
+    private final QueryProfileType type;
+
+    public static QueryProfileFieldType fromString(String queryProfileName, QueryProfileTypeRegistry registry) {
+        if (queryProfileName==null || queryProfileName.equals(""))
+            return new QueryProfileFieldType(null);
+
+        if (registry==null)
+            throw new IllegalArgumentException("Can not resolve query profile type '" + queryProfileName +
+                                               "' because no registry is provided");
+        QueryProfileType queryProfileType=registry.getComponent(queryProfileName);
+        if (queryProfileType==null)
+            throw new IllegalArgumentException("Could not resolve query profile type '" + queryProfileName + "'");
+        return new QueryProfileFieldType(registry.getComponent(queryProfileName));
+    }
+
+    public QueryProfileFieldType() { this(null); }
+
+    public QueryProfileFieldType(QueryProfileType type) {
+        this.type = type;
+    }
+
+    /** Returns the query profile type of this, or null if any type works */
+    public QueryProfileType getQueryProfileType() { return type; }
+
+    public @Override Class<?> getValueClass() { return QueryProfile.class; }
+
+    public @Override String stringValue() {
+        return "query-profile" + (type!=null ? ":" + type.getId().getName() : "");
+    }
+
+    public @Override String toString() {
+        return "field type " + stringValue();
+    }
+
+    public @Override String toInstanceDescription() {
+        return "reference to a query profile" + (type!=null ? " of type '" + type.getId().getName() + "'" : "");
+    }
+
+    @Override
+    public CompiledQueryProfile convertFrom(Object object, CompiledQueryProfileRegistry registry) {
+        String profileId = object.toString();
+        if (profileId.startsWith("ref:"))
+            profileId = profileId.substring("ref:".length());
+        CompiledQueryProfile profile = registry.getComponent(profileId);
+        if (profile == null) return null;
+        if (type != null && ! type.equals(profile.getType())) return null;
+        return profile;
+    }
+
+    @Override
+    public QueryProfile convertFrom(Object object, QueryProfileRegistry registry) {
+        QueryProfile profile;
+        if (object instanceof String)
+            profile = registry.getComponent((String)object);
+        else if (object instanceof QueryProfile)
+            profile = (QueryProfile)object;
+        else
+            return null;
+
+        // Verify its type as well
+        if (type!=null && type!=profile.getType()) return null;
+        return profile;
+    }
+
+    @Override
+    public int hashCode() {
+        if (type == null) return 17;
+        return type.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof QueryProfileFieldType)) return false;
+        QueryProfileFieldType other = (QueryProfileFieldType)o;
+        return equals(this.type.getId(), other.type.getId());
+    }
+
+    private boolean equals(Object o1, Object o2) {
+        if (o1 == null) return o2 == null;
+        return o1.equals(o2);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
new file mode 100644
index 00000000000..ecf60f8723d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
@@ -0,0 +1,355 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.provider.FreezableSimpleComponent;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.QueryProfile;
+
+import java.util.*;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * Defines a kind of query profiles
+ *
+ * @author bratseth
+ */
+public class QueryProfileType extends FreezableSimpleComponent {
+
+    /** The fields of this query profile type */
+    private Map<String, FieldDescription> fields = new HashMap<>();
+
+    /** The query profile types this inherits */
+    private List<QueryProfileType> inherited = new ArrayList<>();
+
+    /** If this is true, keys which are not declared in this type cannot be set in instances */
+    private boolean strict = false;
+
+    /** True if the name of instances of this profile should be matched as path names, see QueryProfileRegistry */
+    private boolean matchAsPath = false;
+
+    private boolean builtin = false;
+
+    /** Aliases *from* any strings *to* field names. Aliases are case insensitive */
+    private Map<String, String> aliases = null;
+
+    public QueryProfileType(String idString) {
+        this(new ComponentId(idString));
+    }
+
+    public QueryProfileType(ComponentId id) {
+        super(id);
+        QueryProfile.validateName(id.getName());
+    }
+
+    private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited,
+                            boolean strict, boolean matchAsPath, boolean builtin, Map<String,String> aliases) {
+        super(id);
+        this.fields = new HashMap<>(fields);
+        this.inherited = new ArrayList<>(inherited);
+        this.strict = strict;
+        this.matchAsPath = matchAsPath;
+        this.builtin = builtin;
+        this.aliases = aliases == null ? null : new HashMap<>(aliases);
+    }
+
+    /** Return this is it is not frozen, returns a modifiable deeply unfrozen copy otherwise */
+    public QueryProfileType unfrozen() {
+        if ( ! isFrozen()) return this;
+
+        // Unfreeze inherited query profile references
+        List<QueryProfileType> unfrozenInherited = new ArrayList<>();
+        for (QueryProfileType inheritedType : inherited) {
+            unfrozenInherited.add(inheritedType.unfrozen());
+        }
+
+        // Unfreeze nested query profile references
+        Map<String, FieldDescription> unfrozenFields = new HashMap<>();
+        for (Map.Entry<String, FieldDescription> field : fields.entrySet()) {
+            FieldDescription unfrozenFieldValue = field.getValue();
+            if (field.getValue().getType() instanceof QueryProfileFieldType) {
+                QueryProfileFieldType queryProfileFieldType = (QueryProfileFieldType)field.getValue().getType();
+                if (queryProfileFieldType.getQueryProfileType() != null) {
+                    QueryProfileFieldType unfrozenType =
+                            new QueryProfileFieldType(queryProfileFieldType.getQueryProfileType().unfrozen());
+                    unfrozenFieldValue = field.getValue().withType(unfrozenType);
+                }
+            }
+            unfrozenFields.put(field.getKey(), unfrozenFieldValue);
+        }
+
+        return new QueryProfileType(getId(), unfrozenFields, unfrozenInherited, strict, matchAsPath, builtin, aliases);
+    }
+
+    /** Mark this type as built into the system. Do not use */
+    public void setBuiltin(boolean builtin) { this.builtin=builtin; }
+
+    /** Returns whether this type is built into the system */
+    public boolean isBuiltin() { return builtin; }
+
+    /**
+     * Returns the query profile types inherited from this (never null).
+     * If this profile type is not frozen, this list can be modified to change the set of inherited types.
+     * If it is frozen, the returned list is immutable.
+     */
+    public List<QueryProfileType> inherited() { return inherited; }
+
+    /**
+     * Returns the fields declared in this (i.e not including those inherited) as an immutable map.
+     *
+     * @throws IllegalStateException if this is frozen
+     */
+    public Map<String,FieldDescription> declaredFields() {
+        ensureNotFrozen();
+        return Collections.unmodifiableMap(fields);
+    }
+
+    /**
+     * Returns true if <i>this</i> is declared strict.
+     * @throws IllegalStateException if this is frozen
+     */
+    public boolean isDeclaredStrict() {
+        ensureNotFrozen();
+        return strict;
+    }
+
+    /**
+     * Returns true if <i>this</i> is declared as match as path.
+     * @throws IllegalStateException if this is frozen
+     */
+    public boolean getDeclaredMatchAsPath() {
+        ensureNotFrozen();
+        return matchAsPath;
+    }
+
+    /** Set whether nondeclared fields are permissible. Throws an exception if this is frozen. */
+    public void setStrict(boolean strict) {
+        ensureNotFrozen();
+        this.strict=strict;
+    }
+
+    /** Returns whether field not declared in this type is permissible in instances. Default is false: Additional values are allowed */
+    public boolean isStrict() {
+        if (isFrozen()) return strict;
+
+        // Check if any of this or an inherited is true
+        if (strict) return true;
+        for (QueryProfileType inheritedType : inherited)
+            if (inheritedType.isStrict()) return true;
+        return false;
+    }
+
+    /** Returns whether instances of this should be matched as path names. Throws if this is frozen. */
+    public void setMatchAsPath(boolean matchAsPath) {
+        ensureNotFrozen();
+        this.matchAsPath=matchAsPath;
+    }
+
+    /** Returns whether instances of this should be matched as path names. Default is false: Use exact name matching. */
+    public boolean getMatchAsPath() {
+        if (isFrozen()) return matchAsPath;
+
+        // Check if any of this or an inherited is true
+        if (matchAsPath) return true;
+        for (QueryProfileType inheritedType : inherited)
+            if (inheritedType.getMatchAsPath()) return true;
+        return false;
+    }
+
+    public void freeze() {
+        if (isFrozen()) return;
+        // Flatten the inheritance hierarchy into this to facilitate faster lookup
+        for (QueryProfileType inheritedType : inherited) {
+            for (FieldDescription field : inheritedType.fields().values())
+                if ( ! fields.containsKey(field.getName()))
+                    fields.put(field.getName(),field);
+        }
+        fields = ImmutableMap.copyOf(fields);
+        inherited = ImmutableList.copyOf(inherited);
+        strict = isStrict();
+        matchAsPath = getMatchAsPath();
+        super.freeze();
+    }
+
+    /**
+     * Returns whether the given field name is overridable in this type.
+     * Default: true (so all non-declared fields returns true)
+     */
+    public boolean isOverridable(String fieldName) {
+        FieldDescription field=getField(fieldName);
+        if (field==null) return true;
+        return field.isOverridable();
+    }
+
+    /**
+     * Returns the permissible class for the value of the given name in this type
+     *
+     * @return the permissible class for a value, <code>Object</code> if all types are legal,
+     *         null if no types are legal (i.e if the name is not legal)
+     */
+    public Class<?> getValueClass(String name) {
+        FieldDescription fieldDescription=getField(name);
+        if (fieldDescription==null) {
+            if (strict)
+                return null; // Undefined -> Not legal
+            else
+                return Object.class; // Undefined -> Anything is legal
+        }
+        return fieldDescription.getType().getValueClass();
+    }
+
+    /** Returns the type of the given query profile type declared as a field in this */
+    public QueryProfileType getType(String localName) {
+        FieldDescription fieldDescription=getField(localName);
+        if (fieldDescription ==null) return null;
+        if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) return null;
+        return ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType();
+    }
+
+    /**
+     * Returns the description of the field with the given name in this type or an inherited type
+     * (depth first left to right search). Returns null if the field is not defined in this or an inherited profile.
+     */
+    public FieldDescription getField(String name) {
+        FieldDescription field=fields.get(name);
+        if ( field!=null ) return field;
+
+        if ( isFrozen() ) return null; // Inherited are collapsed into this
+
+        for (QueryProfileType inheritedType : this.inherited() ) {
+            field=inheritedType.getField(name);
+            if (field!=null) return field;
+        }
+
+        return null;
+    }
+
+    /**
+     * Removes a field from this (not from any inherited profile)
+     *
+     * @return the removed field or null if none
+     * @throws IllegalStateException if this is frozen
+     */
+    public FieldDescription removeField(String fieldName) {
+        ensureNotFrozen();
+        return fields.remove(fieldName);
+    }
+
+    /**
+     * Adds a field to this, without associating with a type registry; field descriptions with compound
+     * is not be supported.
+     *
+     * @throws IllegalStateException if this is frozen
+     */
+    public void addField(FieldDescription fieldDescription) {
+        // Compound names translates to new types, which must be added to a supplied registry
+        if (fieldDescription.getCompoundName().isCompound())
+            throw new IllegalArgumentException("Adding compound names is only legal when supplying a registry");
+        addField(fieldDescription, null);
+    }
+
+    /**
+     * Adds a field to this
+     *
+     * @throws IllegalStateException if this is frozen
+     */
+    public void addField(FieldDescription fieldDescription, QueryProfileTypeRegistry registry) {
+        CompoundName name = fieldDescription.getCompoundName();
+        if (name.isCompound()) {
+            // Add (/to) a query profile type containing the rest of the name.
+            // (we do not need the field description settings for intermediate query profile types
+            // as the leaf entry will enforce them)
+            QueryProfileType type = getOrCreateQueryProfileType(name.first(), registry);
+            type.addField(fieldDescription.withName(name.rest()), registry);
+        }
+        else {
+            ensureNotFrozen();
+            fields.put(fieldDescription.getName(), fieldDescription);
+        }
+
+        for (String alias : fieldDescription.getAliases())
+            addAlias(alias, fieldDescription.getName());
+    }
+
+    private QueryProfileType getOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) {
+        FieldDescription fieldDescription = getField(name);
+        if (fieldDescription != null) {
+            if ( ! ( fieldDescription.getType() instanceof QueryProfileFieldType))
+                throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " +
+                                                   "already a " + fieldDescription.getType());
+            QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType();
+            QueryProfileType type = fieldType.getQueryProfileType();
+            if (type == null) { // an as-yet untyped reference; add type
+                type = new QueryProfileType(name);
+                registry.register(type.getId(), type);
+                fields.put(name, fieldDescription.withType(new QueryProfileFieldType(type)));
+            }
+            return type;
+        }
+        else {
+            QueryProfileType type = new QueryProfileType(name);
+            registry.register(type.getId(), type);
+            fields.put(name, new FieldDescription(name, new QueryProfileFieldType(type)));
+            return type;
+        }
+    }
+
+    private void addAlias(String alias,String field) {
+        ensureNotFrozen();
+        if (aliases==null)
+            aliases=new HashMap<>();
+        aliases.put(toLowerCase(alias),field);
+    }
+
+    /** Returns all the fields of this profile type and all types it inherits as a read-only map */
+    public Map<String,FieldDescription> fields() {
+        if (isFrozen()) return fields;
+        if (inherited().size()==0) return Collections.unmodifiableMap(fields);
+
+        // Collapse inherited
+        Map<String,FieldDescription> allFields=new HashMap<>(fields);
+        for (QueryProfileType inheritedType : inherited)
+            allFields.putAll(inheritedType.fields());
+        return Collections.unmodifiableMap(allFields);
+    }
+
+    /**
+     * Returns the alias to field mapping of this type as a read-only map. This is never null.
+     * Note that all keys are lower-cased because aliases are case-insensitive
+     */
+    public Map<String,String> aliases() {
+        if (isFrozen()) return aliases;
+        if (aliases == null) return Collections.emptyMap();
+        return Collections.unmodifiableMap(aliases);
+    }
+
+    /** Returns the field name of an alias or field name */
+    public String unalias(String aliasOrField) {
+        if (aliases==null || aliases.isEmpty()) return aliasOrField;
+        String field=aliases.get(toLowerCase(aliasOrField));
+        if (field!=null) return field;
+        return aliasOrField;
+    }
+
+    @Override
+    public int hashCode() {
+        return getId().hashCode();
+    }
+
+    /** Two types are equal if they have the same id */
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof QueryProfileType)) return false;
+        QueryProfileType other = (QueryProfileType)o;
+        return other.getId().equals(this.getId());
+    }
+
+    public String toString() {
+        return "query profile type '" + getId() + "'";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java
new file mode 100644
index 00000000000..3f64caa7ab1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileTypeRegistry.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+
+/**
+ * A registry of query profile types
+ *
+ * @author bratseth
+ */
+public class QueryProfileTypeRegistry extends ComponentRegistry<QueryProfileType> {
+
+    public QueryProfileTypeRegistry() {
+        Query.addNativeQueryProfileTypesTo(this);
+    }
+
+    /** Register this type by its id */
+    public void register(QueryProfileType type) {
+        super.register(type.getId(), type);
+    }
+
+    @Override
+    public void freeze() {
+        if (isFrozen()) return;
+        for (QueryProfileType queryProfileType : allComponents())
+            queryProfileType.freeze();
+    }
+
+    public static QueryProfileTypeRegistry emptyFrozen() {
+        QueryProfileTypeRegistry registry = new QueryProfileTypeRegistry();
+        registry.freeze();
+        return registry;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
new file mode 100644
index 00000000000..747cf73acb3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
@@ -0,0 +1,59 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.tensor.MapTensor;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
+
+import java.util.Optional;
+
+/**
+ * A tensor field type in a query profile
+ *
+ * @author bratseth
+ */
+public class TensorFieldType extends FieldType {
+
+    private final Optional<TensorType> type;
+
+    /** Creates a tensor field type with optional information about the kind of tensor this will hold */
+    public TensorFieldType(Optional<TensorType> type) {
+        this.type = type;
+    }
+
+    /** Returns information about the type of tensor this will hold, or empty to allow any kind of tensor */
+    public Optional<TensorType> type() { return type; }
+
+    @Override
+    public Class getValueClass() { return Tensor.class; }
+
+    @Override
+    public String stringValue() { return "tensor"; }
+
+    @Override
+    public String toString() { return "field type " + stringValue(); }
+
+    @Override
+    public String toInstanceDescription() { return "a tensor"; }
+
+    @Override
+    public Object convertFrom(Object o, QueryProfileRegistry registry) {
+        if (o instanceof Tensor) return o;
+        if (o instanceof String) return MapTensor.from((String)o);
+        return null;
+    }
+
+    @Override
+    public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) {
+        return convertFrom(o, (QueryProfileRegistry)null);
+    }
+
+    public static TensorFieldType fromTypeString(String s) {
+        if (s.equals("tensor")) return genericTensorType;
+        return new TensorFieldType(Optional.of(TensorType.fromSpec(s)));
+    }
+
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/package-info.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/package-info.java
new file mode 100644
index 00000000000..1f9fa7a1fb4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/package-info.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Query profile types defines the set of fields a query profile may, can or must have. Query profile
+ * types may be inherited in a type hierarchy.
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java
new file mode 100644
index 00000000000..01c861b879e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.properties;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.Map;
+
+/**
+ * Default values for properties that are meant to be customized in query profiles.
+ * @author tonytv
+ */
+public final class DefaultProperties extends Properties  {
+    public static final CompoundName MAX_OFFSET = new CompoundName("maxOffset");
+    public static final CompoundName MAX_HITS = new CompoundName("maxHits");
+
+
+    public static final QueryProfileType argumentType = new QueryProfileType("DefaultProperties");
+    static {
+        argumentType.setBuiltin(true);
+
+        argumentType.addField(new FieldDescription(MAX_OFFSET.toString(), "integer"));
+        argumentType.addField(new FieldDescription(MAX_HITS.toString(), "integer"));
+
+        argumentType.freeze();
+    }
+
+    @Override
+    public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) {
+        if (MAX_OFFSET.equals(name)) {
+            return 1000;
+        } else if (MAX_HITS.equals(name)) {
+            return 400;
+        } else {
+            return super.get(name, context, substitution);
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
new file mode 100644
index 00000000000..cc2c08c5504
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
@@ -0,0 +1,58 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.properties;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.Properties;
+
+import java.util.Map;
+
+/**
+ * A properties implementation which translates the incoming name to its standard name
+ * if it is a registered alias.
+ * <p>
+ * Aliases are case insensitive. One standard name may have multiple aliases.
+ * <p>
+ * This is multithread safe or not depending on the status of the passed map of aliases.
+ * Cloning will not deep copy the set of aliases.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class PropertyAliases extends Properties {
+
+    /** A map from aliases to standard names */
+    private final Map<String,CompoundName> aliases;
+
+    /**
+     * Creates an instance with a set of aliases. The given aliases will be used directly by this class.
+     * To make this class immutable and thread safe, relinquish ownership of the parameter map.
+     */
+    public PropertyAliases(Map<String,CompoundName> aliases) {
+        this.aliases=aliases;
+    }
+
+    /**
+     * Returns the standard name for an alias, or the given name if it is not a registered alias
+     *
+     * @param nameOrAlias the name to check if is an alias
+     * @return the real name if an alias or the input name itself
+     */
+    protected CompoundName unalias(CompoundName nameOrAlias) {
+        CompoundName properName = aliases.get(nameOrAlias.getLowerCasedName());
+        return (properName != null) ? properName : nameOrAlias;
+    }
+
+    public @Override Map<String, Object> listProperties(CompoundName property,Map<String,String> context,
+                                                        com.yahoo.processing.request.Properties substitution) {
+        return super.listProperties(unalias(property),context,substitution);
+    }
+
+    public @Override Object get(CompoundName name,Map<String,String> context,
+                                com.yahoo.processing.request.Properties substitution) {
+        return super.get(unalias(name),context,substitution);
+    }
+
+    public @Override void set(CompoundName name,Object value,Map<String,String> context) {
+        super.set(unalias(name),value,context);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
new file mode 100644
index 00000000000..820c4fc8ea3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
@@ -0,0 +1,132 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.properties;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.result.Hit;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * A Map backing of Properties.
+ * <p>
+ * When this is cloned it will deep copy not only the model object map, but also each
+ * clonable member inside the map.
+ * <p>
+ * Subclassing is supported, a hook can be implemented to provide conditional inclusion in the map.
+ * By default - all properties are accepted, so set is never propagated.
+ * <p>
+ * This class is not multithread safe.
+ *
+ * @author bratseth
+ */
+public class PropertyMap extends Properties {
+
+    private static Logger log=Logger.getLogger(PropertyMap.class.getName());
+
+    /** The properties of this */
+    private Map<CompoundName, Object> properties = new LinkedHashMap<>();
+
+    public void set(CompoundName name, Object value, Map<String,String> context) {
+        if (shouldSet(name, value))
+            properties.put(name, value);
+        else
+            super.set(name, value, context);
+    }
+
+    /**
+     * Return true if this value should be set in this map, false if the set should be propagated instead
+     * This default implementation always returns true.
+     */
+    protected boolean shouldSet(CompoundName name,Object value) { return true; }
+
+    public @Override Object get(CompoundName name, Map<String,String> context,
+                                com.yahoo.processing.request.Properties substitution) {
+        if ( ! properties.containsKey(name)) return super.get(name,context,substitution);
+        return properties.get(name);
+    }
+
+    /**
+     * Returns a direct reference to the map containing the properties set in this instance.
+     */
+    public Map<CompoundName, Object> propertyMap() {
+        return properties;
+    }
+
+    public @Override PropertyMap clone() {
+        PropertyMap clone = (PropertyMap)super.clone();
+        clone.properties = new HashMap<>();
+        for (Map.Entry<CompoundName, Object> entry : this.properties.entrySet()) {
+            Object cloneValue = clone(entry.getValue());
+            if (cloneValue == null)
+                cloneValue = entry.getValue(); // Shallow copy objects which does not support cloning
+            clone.properties.put(entry.getKey(), cloneValue);
+        }
+        return clone;
+    }
+
+    /** Clones this object if it is clonable, and the clone is public. Returns null if not */
+    public static Object clone(Object object) {
+        if (object==null) return null;
+        if (! ( object instanceof Cloneable) ) return null;
+        if (object  instanceof Object[])
+            return arrayClone((Object[])object);
+        else
+            return objectClone(object);
+    }
+
+    private static Object arrayClone(Object[] object) {
+        Object[] arrayClone= Arrays.copyOf(object, object.length);
+        // deep clone
+        for (int i=0; i<arrayClone.length; i++) {
+            Object elementClone=clone(arrayClone[i]);
+            if (elementClone!=null)
+                arrayClone[i]=elementClone;
+        }
+        return arrayClone;
+    }
+
+    private static Object objectClone(Object object) {
+        if (object instanceof Hit) {
+            return ((Hit) object).clone();
+        } else if (object instanceof LinkedList) {
+            return ((LinkedList) object).clone();
+        }
+        try {
+            Method cloneMethod=object.getClass().getMethod("clone");
+            return cloneMethod.invoke(object);
+        }
+        catch (NoSuchMethodException e) {
+            log.warning("'" + object + "' is Cloneable, but has no clone method - will use the same instance in all requests");
+            return null;
+        }
+        catch (IllegalAccessException e) {
+            log.warning("'" + object + "' is Cloneable, but clone method cannot be accessed - will use the same instance in all requests");
+            return null;
+        }
+        catch (InvocationTargetException e) {
+            throw new RuntimeException("Exception cloning '" + object + "'",e);
+        }
+    }
+
+    @Override
+    public Map<String, Object> listProperties(CompoundName path, Map<String, String> context, com.yahoo.processing.request.Properties substitution) {
+        Map<String, Object> map = super.listProperties(path, context, substitution);
+
+        for (Map.Entry<CompoundName, Object> entry : properties.entrySet()) {
+            if ( ! entry.getKey().hasPrefix(path)) continue;
+            CompoundName propertyName = entry.getKey().rest(path.size());
+            if (propertyName.isEmpty()) continue;
+            map.put(propertyName.toString(), entry.getValue());
+        }
+        return map;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
new file mode 100644
index 00000000000..cd4e02dc768
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -0,0 +1,296 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.properties;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.*;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry;
+import com.yahoo.search.query.ranking.Diversity;
+import com.yahoo.search.query.ranking.MatchPhase;
+import com.yahoo.tensor.Tensor;
+
+import java.util.Map;
+
+/**
+ * Maps between the query model and text properties.
+ * This can be done simpler by using reflection but the performance penalty was not worth it,
+ * especially since we should be conservative in adding things to the query model.
+ *
+ * @author bratseth
+ */
+public class QueryProperties extends Properties {
+
+    private static final String MODEL_PREFIX = Model.MODEL + ".";
+    private static final String RANKING_PREFIX = Ranking.RANKING + ".";
+    private static final String PRESENTATION_PREFIX = Presentation.PRESENTATION + ".";
+
+    public static final CompoundName[] PER_SOURCE_QUERY_PROPERTIES = new CompoundName[] {
+            new CompoundName(MODEL_PREFIX + Model.QUERY_STRING),
+            new CompoundName(MODEL_PREFIX + Model.TYPE),
+            new CompoundName(MODEL_PREFIX + Model.FILTER),
+            new CompoundName(MODEL_PREFIX + Model.DEFAULT_INDEX),
+            new CompoundName(MODEL_PREFIX + Model.LANGUAGE),
+            new CompoundName(MODEL_PREFIX + Model.ENCODING),
+            new CompoundName(MODEL_PREFIX + Model.SOURCES),
+            new CompoundName(MODEL_PREFIX + Model.SEARCH_PATH),
+            new CompoundName(MODEL_PREFIX + Model.RESTRICT),
+            new CompoundName(RANKING_PREFIX + Ranking.LOCATION),
+            new CompoundName(RANKING_PREFIX + Ranking.PROFILE),
+            new CompoundName(RANKING_PREFIX + Ranking.SORTING),
+            new CompoundName(RANKING_PREFIX + Ranking.FRESHNESS),
+            new CompoundName(RANKING_PREFIX + Ranking.QUERYCACHE),
+            new CompoundName(RANKING_PREFIX + Ranking.LIST_FEATURES),
+            new CompoundName(PRESENTATION_PREFIX + Presentation.BOLDING),
+            new CompoundName(PRESENTATION_PREFIX + Presentation.SUMMARY),
+            new CompoundName(PRESENTATION_PREFIX + Presentation.REPORT_COVERAGE),
+            new CompoundName(PRESENTATION_PREFIX + Presentation.FORMAT),
+            new CompoundName(PRESENTATION_PREFIX + Presentation.SUMMARY_FIELDS),
+            Query.HITS,
+            Query.OFFSET,
+            Query.TRACE_LEVEL,
+            Query.TIMEOUT,
+            Query.NO_CACHE,
+            Query.GROUPING_SESSION_CACHE };
+
+    private Query query;
+    private final CompiledQueryProfileRegistry profileRegistry;
+
+    public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry) {
+        this.query = query;
+        this.profileRegistry = profileRegistry;
+    }
+
+    public void setParentQuery(Query query) {
+        this.query=query;
+        super.setParentQuery(query);
+    }
+
+    @SuppressWarnings("deprecation")
+    @Override
+    public Object get(final CompoundName key, Map<String,String> context,
+                                com.yahoo.processing.request.Properties substitution) {
+        if (key.size()==2 && key.first().equals(Model.MODEL)) {
+            if (key.last().equals(Model.QUERY_STRING)) return query.getModel().getQueryString();
+            if (key.last().equals(Model.TYPE)) return query.getModel().getType();
+            if (key.last().equals(Model.FILTER)) return query.getModel().getFilter();
+            if (key.last().equals(Model.DEFAULT_INDEX)) return query.getModel().getDefaultIndex();
+            if (key.last().equals(Model.LANGUAGE)) return query.getModel().getLanguage();
+            if (key.last().equals(Model.ENCODING)) return query.getModel().getEncoding();
+            if (key.last().equals(Model.SOURCES)) return query.getModel().getSources();
+            if (key.last().equals(Model.SEARCH_PATH)) return query.getModel().getSearchPath();
+            if (key.last().equals(Model.RESTRICT)) return query.getModel().getRestrict();
+        }
+        else if (key.first().equals(Ranking.RANKING)) {
+            if (key.size()==2) {
+                if (key.last().equals(Ranking.LOCATION)) return query.getRanking().getLocation();
+                if (key.last().equals(Ranking.PROFILE)) return query.getRanking().getProfile();
+                if (key.last().equals(Ranking.SORTING)) return query.getRanking().getSorting();
+                if (key.last().equals(Ranking.FRESHNESS)) return query.getRanking().getFreshness();
+                if (key.last().equals(Ranking.QUERYCACHE)) return query.getRanking().getQueryCache();
+                if (key.last().equals(Ranking.LIST_FEATURES)) return query.getRanking().getListFeatures();
+            }
+            else if (key.size()>=3 && key.get(1).equals(Ranking.MATCH_PHASE)) {
+                if (key.size() == 3) {
+                    MatchPhase matchPhase = query.getRanking().getMatchPhase();
+                    if (key.last().equals(MatchPhase.ATTRIBUTE)) return matchPhase.getAttribute();
+                    if (key.last().equals(MatchPhase.ASCENDING)) return matchPhase.getAscending();
+                    if (key.last().equals(MatchPhase.MAX_HITS)) return matchPhase.getMaxHits();
+                    if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) return matchPhase.getMaxFilterCoverage();
+                } else if (key.size() >= 4 && key.get(2).equals(Ranking.DIVERSITY)) {
+                    Diversity diversity = query.getRanking().getMatchPhase().getDiversity();
+                    if (key.size() == 4) {
+                        if (key.last().equals(Diversity.ATTRIBUTE)) return diversity.getAttribute();
+                        if (key.last().equals(Diversity.MINGROUPS)) return diversity.getMinGroups();
+                    } else if ((key.size() == 5)  && key.get(3).equals(Diversity.CUTOFF)) {
+                        if (key.last().equals(Diversity.FACTOR)) return diversity.getCutoffFactor();
+                        if (key.last().equals(Diversity.STRATEGY)) return diversity.getCutoffStrategy();
+                    }
+                }
+            }
+            else if (key.size()>2) {
+                // pass the portion after "ranking.features/properties" down
+                if (key.get(1).equals(Ranking.FEATURES)) return query.getRanking().getFeatures().getObject(key.rest().rest().toString());
+                if (key.get(1).equals(Ranking.PROPERTIES)) return query.getRanking().getProperties().get(key.rest().rest().toString());
+            }
+        }
+        else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) {
+            if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding();
+            if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary();
+            if (key.last().equals(Presentation.REPORT_COVERAGE)) return query.getPresentation().getReportCoverage();
+            if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat();
+            if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming();
+            if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields();
+        }
+        else if (key.first().equals("rankfeature") || key.first().equals("featureoverride")) { // featureoverride is deprecated
+            return query.getRanking().getFeatures().getObject(key.rest().toString());
+        } else if (key.first().equals("rankproperty")) {
+            return query.getRanking().getProperties().get(key.rest().toString());
+        } else if (key.size()==1) {
+            if (key.equals(Query.HITS)) return query.getHits();
+            if (key.equals(Query.OFFSET)) return query.getOffset();
+            if (key.equals(Query.TRACE_LEVEL)) return query.getTraceLevel();
+            if (key.equals(Query.TIMEOUT)) return query.getTimeout();
+            if (key.equals(Query.NO_CACHE)) return query.getNoCache();
+            if (key.equals(Query.GROUPING_SESSION_CACHE)) return query.getGroupingSessionCache();
+            if (key.toString().equals(Model.MODEL)) return query.getModel();
+            if (key.toString().equals(Ranking.RANKING)) return query.getRanking();
+            if (key.toString().equals(Presentation.PRESENTATION)) return query.getPresentation();
+        }
+        return super.get(key,context,substitution);
+    }
+
+    @SuppressWarnings("deprecation")
+    @Override
+    public void set(final CompoundName key,Object value,Map<String,String> context) {
+        // Note: The defaults here are never used
+        try {
+            if (key.size()==2 && key.first().equals(Model.MODEL)) {
+                if (key.last().equals(Model.QUERY_STRING))
+                    query.getModel().setQueryString(asString(value, ""));
+                else if (key.last().equals(Model.TYPE))
+                    query.getModel().setType(asString(value, "ANY"));
+                else if (key.last().equals(Model.FILTER))
+                    query.getModel().setFilter(asString(value, ""));
+                else if (key.last().equals(Model.DEFAULT_INDEX))
+                    query.getModel().setDefaultIndex(asString(value, ""));
+                else if (key.last().equals(Model.LANGUAGE))
+                    query.getModel().setLanguage(asString(value, ""));
+                else if (key.last().equals(Model.ENCODING))
+                    query.getModel().setEncoding(asString(value,""));
+                else if (key.last().equals(Model.SEARCH_PATH))
+                    query.getModel().setSearchPath(asString(value,""));
+                else if (key.last().equals(Model.SOURCES))
+                    query.getModel().setSources(asString(value,""));
+                else if (key.last().equals(Model.RESTRICT))
+                    query.getModel().setRestrict(asString(value,""));
+                else
+                    throwIllegalParameter(key.last(),Model.MODEL);
+            }
+            else if (key.first().equals(Ranking.RANKING)) {
+                if (key.size()==2) {
+                    if (key.last().equals(Ranking.LOCATION))
+                        query.getRanking().setLocation(asString(value,""));
+                    else if (key.last().equals(Ranking.PROFILE))
+                        query.getRanking().setProfile(asString(value,""));
+                    else if (key.last().equals(Ranking.SORTING))
+                        query.getRanking().setSorting(asString(value,""));
+                    else if (key.last().equals(Ranking.FRESHNESS))
+                        query.getRanking().setFreshness(asString(value, ""));
+                    else if (key.last().equals(Ranking.QUERYCACHE))
+                        query.getRanking().setQueryCache(asBoolean(value, false));
+                    else if (key.last().equals(Ranking.LIST_FEATURES))
+                        query.getRanking().setListFeatures(asBoolean(value,false));
+                }
+                else if (key.size()>=3 && key.get(1).equals(Ranking.MATCH_PHASE)) {
+                    if (key.size() == 3) {
+                        MatchPhase matchPhase = query.getRanking().getMatchPhase();
+                        if (key.last().equals(MatchPhase.ATTRIBUTE)) {
+                            matchPhase.setAttribute(asString(value, null));
+                        } else if (key.last().equals(MatchPhase.ASCENDING)) {
+                            matchPhase.setAscending(asBoolean(value, false));
+                        } else if (key.last().equals(MatchPhase.MAX_HITS)) {
+                            matchPhase.setMaxHits(asLong(value, null));
+                        } else if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) {
+                            matchPhase.setMaxFilterCoverage(asDouble(value, 1.0));
+                        }
+                    } else if (key.size() > 3 && key.get(2).equals(Ranking.DIVERSITY)) {
+                        Diversity diversity = query.getRanking().getMatchPhase().getDiversity();
+                        if (key.last().equals(Diversity.ATTRIBUTE)) {
+                            diversity.setAttribute(asString(value, null));
+                        } else if (key.last().equals(Diversity.MINGROUPS)) {
+                            diversity.setMinGroups(asLong(value, null));
+                        } else if ((key.size() > 4) && key.get(3).equals(Diversity.CUTOFF)) {
+                            if (key.last().equals(Diversity.FACTOR)) {
+                                diversity.setCutoffFactor(asDouble(value, 10.0));
+                            } else if (key.last().equals(Diversity.STRATEGY)) {
+                                diversity.setCutoffStrategy(asString(value, "loose"));
+                            }
+                        }
+                    }
+                }
+                else if (key.size()>2) {
+                    String restKey = key.rest().rest().toString();
+                    if (key.get(1).equals(Ranking.FEATURES))
+                        setRankingFeature(query, restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("features")));
+                    else if (key.get(1).equals(Ranking.PROPERTIES))
+                        query.getRanking().getProperties().put(restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("properties")));
+                    else
+                        throwIllegalParameter(key.rest().toString(),Ranking.RANKING);
+                }
+            }
+            else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) {
+                if (key.last().equals(Presentation.BOLDING))
+                    query.getPresentation().setBolding(asBoolean(value, true));
+                else if (key.last().equals(Presentation.SUMMARY))
+                    query.getPresentation().setSummary(asString(value, ""));
+                else if (key.last().equals(Presentation.REPORT_COVERAGE))
+                    query.getPresentation().setReportCoverage(asBoolean(value,true));
+                else if (key.last().equals(Presentation.FORMAT))
+                    query.getPresentation().setFormat(asString(value,""));
+                else if (key.last().equals(Presentation.TIMING))
+                    query.getPresentation().setTiming(asBoolean(value, true));
+                else if (key.last().equals(Presentation.SUMMARY_FIELDS))
+                    query.getPresentation().setSummaryFields(asString(value,""));
+                else
+                    throwIllegalParameter(key.last(), Presentation.PRESENTATION);
+            }
+            else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated
+                setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features")));
+            } else if (key.first().equals("rankproperty")) {
+                query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("properties")));
+            } else if (key.size()==1) {
+                if (key.equals(Query.HITS))
+                    query.setHits(asInteger(value,10));
+                else if (key.equals(Query.OFFSET))
+                    query.setOffset(asInteger(value,0));
+                else if (key.equals(Query.TRACE_LEVEL))
+                    query.setTraceLevel(asInteger(value,0));
+                else if (key.equals(Query.TIMEOUT))
+                    query.setTimeout(value.toString());
+                else if (key.equals(Query.NO_CACHE))
+                    query.setNoCache(asBoolean(value,false));
+                else if (key.equals(Query.GROUPING_SESSION_CACHE))
+                    query.setGroupingSessionCache(asBoolean(value, false));
+                else
+                    super.set(key,value,context);
+            }
+            else
+                super.set(key,value,context);
+        }
+        catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation
+            if (e.getMessage().startsWith("Could not set"))
+                throw e;
+            else
+                throw new IllegalArgumentException("Could not set '" + key + "' to '" + value + "'", e);
+        }
+    }
+
+    private void setRankingFeature(Query query, String key, Object value) {
+        if (value instanceof Tensor)
+            query.getRanking().getFeatures().put(key, (Tensor)value);
+        else
+            query.getRanking().getFeatures().put(key, asString(value, ""));
+    }
+
+    private Object toSpecifiedType(String key, Object value, QueryProfileType type) {
+        if ( ! ( value instanceof String)) return value; // already typed
+        if (type == null) return value; // no type info -> keep as string
+        FieldDescription field = type.getField(key);
+        if (field == null) return value; // ditto
+        return field.getType().convertFrom(value, profileRegistry);
+    }
+
+    private void throwIllegalParameter(String key,String namespace) {
+        throw new IllegalArgumentException("'" + key + "' is not a valid property in '" + namespace +
+                                           "'. See the search api for valid keys starting by '" + namespace + "'.");
+    }
+
+    @Override
+    public final Query getParentQuery() {
+        return query;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryPropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryPropertyAliases.java
new file mode 100644
index 00000000000..15544e8ff4c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryPropertyAliases.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.properties;
+
+import com.yahoo.processing.request.CompoundName;
+
+import java.util.Map;
+
+/**
+ * Property aliases which contains some hardcoded unaliasing of prefixes of
+ * rankfeature and rankproperty maps.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class QueryPropertyAliases extends PropertyAliases {
+
+    /**
+     * Creates an instance with a set of aliases. The given aliases will be used directly by this class.
+     * To make this class immutable and thread safe, relinquish ownership of the parameter map.
+     */
+    public QueryPropertyAliases(Map<String,CompoundName> aliases) {
+        super(aliases);
+    }
+
+    @Override
+    protected CompoundName unalias(CompoundName nameOrAlias) {
+        if (nameOrAlias.first().equalsIgnoreCase("rankfeature"))
+            return nameOrAlias.rest().prepend("ranking", "features");
+        else if (nameOrAlias.first().equalsIgnoreCase("rankproperty"))
+            return nameOrAlias.rest().prepend("ranking", "properties");
+        return super.unalias(nameOrAlias);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
new file mode 100644
index 00000000000..c97f4daf6d4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.properties;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.Properties;
+
+import java.util.Map;
+
+/**
+ * Turns get(name) into get(name,request) using the request given at construction time.
+ * This is used to allow the query's request to be supplied to all property requests
+ * without forcing users of the query.properties() to supply this explicitly.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class RequestContextProperties extends Properties {
+
+    private final Map<String,String> requestMap;
+
+    public RequestContextProperties(Map<String, String> properties) {
+        this.requestMap=properties;
+    }
+
+    @Override
+    public Object get(CompoundName name,Map<String,String> context,
+                      com.yahoo.processing.request.Properties substitution) {
+        return super.get(name,context==null ? requestMap : context,substitution);
+    }
+
+    @Override
+    public void set(CompoundName name,Object value,Map<String,String> context) {
+        super.set(name,value,context==null ? requestMap : context);
+    }
+
+    @Override
+    public Map<String, Object> listProperties(CompoundName path,Map<String,String> context,
+                                              com.yahoo.processing.request.Properties substitution) {
+        return super.listProperties(path,context==null ? requestMap : context,substitution);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java
new file mode 100644
index 00000000000..7f5c2ec2558
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.properties;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.processing.request.Properties;
+
+import java.util.Map;
+
+/**
+ * A wrapper around a chain of property objects that prefixes all gets/sets with a given path
+ *
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ */
+public class SubProperties extends com.yahoo.search.query.Properties {
+
+    final private CompoundName pathPrefix;
+    final private Properties parent;
+
+    public SubProperties(String pathPrefix, Properties properties) {
+        this(new CompoundName(pathPrefix),properties);
+    }
+
+    public SubProperties(CompoundName pathPrefix, Properties properties) {
+        this.pathPrefix = pathPrefix;
+        this.parent = properties;
+    }
+
+    @Override
+    public Object get(CompoundName key, Map<String,String> context,
+                      com.yahoo.processing.request.Properties substitution) {
+        if(key == null) return null;
+        Object result = parent.get(getPathPrefix() + "." + key,context,substitution);
+        if(result == null) {
+            return super.get(key,context,substitution);
+        } else {
+            return result;
+        }
+    }
+
+    @Override
+    public void set(CompoundName key, Object obj, Map<String,String> context) {
+        if(key == null) return;
+        parent.set(getPathPrefix() + "." + key, obj, context);
+    }
+
+    @Override
+    public Map<String, Object> listProperties(CompoundName path,Map<String,String> context,
+                                              com.yahoo.processing.request.Properties substitution) {
+        Map<String, Object> map = super.listProperties(path,context,substitution);
+        if(path.isEmpty()) {
+            map.putAll(parent.listProperties(getPathPrefix(),context,substitution));
+        } else {
+            map.putAll(parent.listProperties(getPathPrefix() + "." + path,context,substitution));
+        }
+        return map;
+    }
+
+    public CompoundName getPathPrefixCompound() {
+        return pathPrefix;
+    }
+
+    /** Returns getPatchPrefixCompound.toString() */
+    public String getPathPrefix() {
+        return getPathPrefixCompound().toString();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/package-info.java b/container-search/src/main/java/com/yahoo/search/query/properties/package-info.java
new file mode 100644
index 00000000000..047a5494e53
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.properties;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Diversity.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Diversity.java
new file mode 100644
index 00000000000..b1865ad9d75
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Diversity.java
@@ -0,0 +1,127 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.Objects;
+
+/**
+ * <p>The diversity settings during match phase of a query.
+ * These are the same settings for diversity during match phase that can be set in a rank profile
+ * and is used for achieving guaranteed diversity at the cost of slightly higher cost as more hits must be
+ * considered compared to plain match-phase.</p>
+ *
+ * <p>You specify an additional attribute to be the diversifier and also min diversity needed.</p>
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+public class Diversity implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+
+    public static final String ATTRIBUTE = "attribute";
+    public static final String MINGROUPS = "minGroups";
+    public static final String CUTOFF = "cutoff";
+    public static final String FACTOR = "factor";
+    public static final String STRATEGY = "strategy";
+
+
+    static {
+        argumentType =new QueryProfileType(Ranking.DIVERSITY);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(ATTRIBUTE, "string"));
+        argumentType.addField(new FieldDescription(MINGROUPS, "long"));
+        argumentType.freeze();
+    }
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    public enum CutoffStrategy {loose, strict};
+    private String attribute = null;
+    private Long minGroups = null;
+    private Double cutoffFactor = null;
+    private CutoffStrategy cutoffStrategy= null;
+
+    /**
+     * Sets the attribute field which will be used to guarantee diversity.
+     * Set to null (default) to disable diversification.
+     * <p>
+     * If this is set, make sure to also set the maxGroups value.
+     * <p>
+     * This attribute must be singlevalue.
+     */
+    public void setAttribute(String attribute) { this.attribute = attribute; }
+
+    /** Returns the attribute to use for diversity, or null if none */
+    public String getAttribute() { return attribute; }
+
+    /**
+     * Sets the max hits to aim for producing in the match phase.
+     * This must be set if an attribute value is set.
+     * It should be set to a reasonable fraction of the total documents on each partition.
+     */
+    public void setMinGroups(long minGroups) { this.minGroups = minGroups; }
+
+    /** Returns the max hits to aim for producing in the match phase on each content node, or null if not set */
+    public Long getMinGroups() { return minGroups; }
+
+    public void setCutoffFactor(double cutoffFactor) { this.cutoffFactor = cutoffFactor; }
+    public Double getCutoffFactor() { return cutoffFactor; }
+    public void setCutoffStrategy(String cutoffStrategy) { this.cutoffStrategy = CutoffStrategy.valueOf(cutoffStrategy); }
+    public CutoffStrategy getCutoffStrategy() { return cutoffStrategy; }
+
+    /** Internal operation - DO NOT USE */
+    public void prepare(RankProperties rankProperties) {
+        if (attribute == null && minGroups == null) return;
+
+        if (attribute != null && !attribute.isEmpty()) {
+            rankProperties.put("vespa.matchphase.diversity.attribute", attribute);
+        }
+        if (minGroups != null) {
+            rankProperties.put("vespa.matchphase.diversity.mingroups", String.valueOf(minGroups));
+        }
+        if (cutoffFactor != null) {
+            rankProperties.put("vespa.matchphase.diversity.cutoff.factor", String.valueOf(cutoffFactor));
+        }
+        if (cutoffStrategy != null) {
+            rankProperties.put("vespa.matchphase.diversity.cutoff.strategy", cutoffStrategy);
+        }
+    }
+
+    @Override
+    public Diversity clone() {
+        try {
+            return (Diversity)super.clone();
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Won't happen", e);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        if (attribute != null) hash += 11 * attribute.hashCode();
+        if (minGroups != null) hash += 13 * minGroups.hashCode();
+        if (cutoffFactor != null) hash += 17 * cutoffFactor.hashCode();
+        if (cutoffStrategy != null) hash += 19 * cutoffStrategy.hashCode();
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof Diversity)) return false;
+
+        Diversity other = (Diversity)o;
+        if ( ! Objects.equals(this.attribute, other.attribute)) return false;
+        if ( ! Objects.equals(this.minGroups, other.minGroups)) return false;
+        if ( ! Objects.equals(this.cutoffFactor, other.cutoffFactor)) return false;
+        if ( ! Objects.equals(this.cutoffStrategy, other.cutoffStrategy)) return false;
+        return true;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java
new file mode 100644
index 00000000000..ba25ddbe7e6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/MatchPhase.java
@@ -0,0 +1,153 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+
+import java.util.Objects;
+
+/**
+ * The match phase ranking settings of this query.
+ * These are the same settings for match phase that can be set in a rank profile
+ * and is used for achieving reasonable query behavior given a query which causes too many matches:
+ * The engine will fall back to retrieving the best values according to the attribute given here
+ * during matching.
+ * <p>
+ * For this feature to work well, the order given by the attribute should correlate reasonably with the order
+ * of results produced if full evaluation is performed.
+ *
+ * @author bratseth
+ */
+public class MatchPhase implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+
+    public static final String ATTRIBUTE = "attribute";
+    public static final String ASCENDING = "ascending";
+    public static final String MAX_HITS = "maxHits";
+    public static final String MAX_FILTER_COVERAGE = "maxFilterCoverage";
+
+    static {
+        argumentType =new QueryProfileType(Ranking.MATCH_PHASE);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(ATTRIBUTE, "string"));
+        argumentType.addField(new FieldDescription(ASCENDING, "boolean"));
+        argumentType.addField(new FieldDescription(MAX_HITS, "long"));
+        argumentType.addField(new FieldDescription(MAX_FILTER_COVERAGE, "double"));
+        argumentType.addField(new FieldDescription(Ranking.DIVERSITY, "query-profile", "diversity"));
+        argumentType.freeze();
+    }
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    private String attribute = null;
+    private boolean ascending = false;
+    private Long maxHits = null;
+    private Double maxFilterCoverage = 1.0;
+    private Diversity diversity = new Diversity();
+
+    /**
+     * Sets the attribute field which will be used to decide the best matches after it has been determined
+     * during matching that this query is going to cause too many matches.
+     * Set to null (default) to disable degradation.
+     * <p>
+     * If this is set, make sure to also set the maxHits value.
+     * Otherwise, the attribute setting is ignored.
+     * <p>
+     * This attribute should have fast-search turned on.
+     */
+    public void setAttribute(String attribute) { this.attribute = attribute; }
+
+    /** Returns the attribute to use for degradation, or null if none */
+    public String getAttribute() { return attribute; }
+
+    /**
+     * Set to true to sort by the attribute in ascending order when this is in use during the match phase,
+     * false (default) to use descending order.
+     */
+    public void setAscending(boolean ascending) { this.ascending = ascending; }
+
+    /**
+     * Returns the order to sort the attribute during the path phase when this takes effect.
+     */
+    public boolean getAscending() { return ascending; }
+
+    /**
+     * Sets the max hits to aim for producing in the match phase.
+     * This must be set if an attribute value is set.
+     * It should be set to a reasonable fraction of the total documents on each partition.
+     */
+    public void setMaxHits(long maxHits) { this.maxHits = maxHits; }
+
+    public void setMaxFilterCoverage(double maxFilterCoverage) {
+        if ((maxFilterCoverage < 0.0) || (maxFilterCoverage > 1.0)) {
+            throw new IllegalArgumentException("maxFilterCoverage must be in the range [0.0, 1.0]. It is " + maxFilterCoverage);
+        }
+        this.maxFilterCoverage = maxFilterCoverage;
+    }
+
+    /** Returns the max hits to aim for producing in the match phase on each content node, or null if not set */
+    public Long getMaxHits() { return maxHits; }
+
+    public Double getMaxFilterCoverage() { return maxFilterCoverage; }
+
+    public Diversity getDiversity() { return diversity; }
+
+    public void setDiversity(Diversity diversity) {
+        this.diversity = diversity;
+    }
+
+    /** Internal operation - DO NOT USE */
+    public void prepare(RankProperties rankProperties) {
+        if (attribute == null || maxHits == null) return;
+
+        rankProperties.put("vespa.matchphase.degradation.attribute", attribute);
+        if (ascending) { // backend default is descending
+            rankProperties.put("vespa.matchphase.degradation.ascendingorder", "true");
+        }
+        rankProperties.put("vespa.matchphase.degradation.maxhits", String.valueOf(maxHits));
+        rankProperties.put("vespa.matchphase.degradation.maxfiltercoverage", String.valueOf(maxFilterCoverage));
+        diversity.prepare(rankProperties);
+    }
+
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        hash += 13 * Boolean.hashCode(ascending);
+        hash += 19 * diversity.hashCode();
+        if (attribute != null) hash += 11 * attribute.hashCode();
+        if (maxHits != null) hash += 17 * maxHits.hashCode();
+        hash += 23 * maxFilterCoverage.hashCode();
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof MatchPhase)) return false;
+
+        MatchPhase other = (MatchPhase)o;
+        if ( this.ascending != other.ascending) return false;
+        if ( ! Objects.equals(this.attribute, other.attribute)) return false;
+        if ( ! Objects.equals(this.maxHits, other.maxHits)) return false;
+        if ( ! Objects.equals(this.diversity, other.diversity)) return false;
+        if ( ! Objects.equals(this.maxFilterCoverage, other.maxFilterCoverage)) return false;
+        return true;
+    }
+
+    @Override
+    public MatchPhase clone() {
+        try {
+            MatchPhase clone = (MatchPhase)super.clone();
+            clone.diversity = diversity.clone();
+            return clone;
+        }
+        catch (CloneNotSupportedException e) {
+            throw new RuntimeException("Won't happen", e);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java
new file mode 100644
index 00000000000..1bcd548882c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java
@@ -0,0 +1,130 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.fs4.MapEncoder;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.text.JSON;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Contains the rank features of a query.
+ *
+ * @author bratseth
+ */
+public class RankFeatures implements Cloneable {
+
+    private final Map<String, Object> features;
+
+    public RankFeatures() {
+        this(new LinkedHashMap<>());
+    }
+
+    private RankFeatures(Map<String, Object> features) {
+        this.features = features;
+    }
+
+    /** Sets a rank feature by full name to a value */
+    public void put(String name, String value) {
+        features.put(name, value);
+    }
+
+    /** Sets a tensor rank feature */
+    public void put(String name, Tensor value) {
+        features.put(name, value);
+    }
+
+    /** Returns a rank feature as a string by full name or null if not set */
+    public String get(String name) {
+        Object value = features.get(name);
+        if (value == null) return null;
+        return value.toString();
+    }
+
+    /** Returns this value as whatever type it was stored as. Returns null if the value is not set. */
+    public Object getObject(String name) {
+        return features.get(name);
+    }
+
+    /**
+     * Returns a tensor rank feature, or empty if there is no value with this name.
+     *
+     * @throws IllegalArgumentException if the value is set but is not a tensor
+     */
+    public Optional<Tensor> getTensor(String name) {
+        Object feature = features.get(name);
+        if (feature == null) return Optional.empty();
+        if (feature instanceof Tensor) return Optional.of((Tensor)feature);
+        throw new IllegalArgumentException("Expected a tensor value of '" + name + "' but has " + feature);
+    }
+
+    /**
+     * Returns the map holding the features of this.
+     * This map may be modified to change the rank features of the query.
+     */
+    public Map<String, Object> asMap() { return features; }
+
+    public boolean isEmpty() {
+        return features.isEmpty();
+    }
+
+    /**
+     * Prepares this for encoding, not for external use. See encode on Query for details.
+     * <p>
+     * If the query feature is found in the rank feature set,
+     * remove all these entries and insert them into the rank property set instead.
+     * We want to hide from the user that the query feature value is sent down as a rank property
+     * and picked up by the query feature executor in the backend.
+     */
+    public void prepare(RankProperties rankProperties) {
+        if (isEmpty()) return;
+
+        List<String> featuresToRemove = new ArrayList<>();
+        List<String> propertiesToInsert = new ArrayList<>();
+        for (String key : features.keySet()) {
+            if (key.startsWith("query(") && key.endsWith(")")) {
+                featuresToRemove.add(key);
+                propertiesToInsert.add(key.substring("query(".length(), key.length() - 1));
+            } else if (key.startsWith("$")) {
+                featuresToRemove.add(key);
+                propertiesToInsert.add(key.substring(1));
+            }
+        }
+        for (int i = 0; i < featuresToRemove.size(); ++i) {
+            rankProperties.put(propertiesToInsert.get(i), features.remove(featuresToRemove.get(i)));
+        }
+    }
+
+    public int encode(ByteBuffer buffer) {
+        return MapEncoder.encodeMap("feature", features, buffer);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) return true;
+        if ( ! (other instanceof RankFeatures)) return false;
+
+        return this.features.equals(((RankFeatures)other).features);
+    }
+
+    @Override
+    public int hashCode() {
+        return features.hashCode();
+    }
+
+    @Override
+    public RankFeatures clone() {
+        return new RankFeatures(new LinkedHashMap<>(features));
+    }
+
+    @Override
+    public String toString() {
+        return JSON.encode(features);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java
new file mode 100644
index 00000000000..eccb8bac2d4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankProperties.java
@@ -0,0 +1,114 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.fs4.GetDocSumsPacket;
+import com.yahoo.fs4.MapEncoder;
+import com.yahoo.text.JSON;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Contains the properties properties of a query.
+ * This is a multimap: Multiple properties may be set for the same key.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class RankProperties implements Cloneable {
+
+    private Map<String, List<Object>> properties = new LinkedHashMap<>();
+
+    public RankProperties() {
+        this(new LinkedHashMap<>());
+    }
+
+    private RankProperties(Map<String, List<Object>> properties) {
+        this.properties = properties;
+    }
+
+    public void put(String name, String value) {
+        put(name, (Object)value);
+    }
+
+    /** Adds a property by full name to a value */
+    public void put(String name, Object value) {
+        List<Object> list = properties.get(name);
+        if (list == null) {
+            list = new ArrayList<>();
+            properties.put(name, list);
+        }
+        list.add(value);
+    }
+
+    /**
+     * Returns a read-only list of properties properties by full name.
+     * If this is not set, null is returned. If this is explicitly set to
+     * have no values, and empty list is returned.
+     */
+    public List<String> get(String name) {
+        List<Object> values = properties.get(name);
+        if (values == null) return null;
+        if (values.isEmpty()) return Collections.<String>emptyList();
+
+        // Compatibility ...
+        List<String> stringValues = new ArrayList<>(values.size());
+        for (Object value : values)
+            stringValues.add(value.toString());
+        return Collections.unmodifiableList(stringValues);
+    }
+
+    /** Removes all properties properties for a given name */
+    public void remove(String name) {
+        properties.remove(name);
+    }
+
+    public boolean isEmpty() {
+        return properties.isEmpty();
+    }
+
+    /** Returns a modifiable map of the properties of this */
+    public Map<String, List<Object>> asMap() { return properties; }
+
+    /** Encodes this in a binary internal representation and returns the number of property maps encoded (0 or 1) */
+    public int encode(ByteBuffer buffer, boolean encodeQueryData) {
+        if (encodeQueryData) {
+            return MapEncoder.encodeObjectMultiMap("rank", properties, buffer);
+        }
+        else {
+            List<Object> sessionId = properties.get(GetDocSumsPacket.sessionIdKey);
+            if (sessionId == null) return 0;
+            return MapEncoder.encodeSingleValue("rank", GetDocSumsPacket.sessionIdKey, sessionId.get(0), buffer);
+        }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) return true;
+        if ( ! (other instanceof RankProperties)) return false;
+
+        return this.properties.equals(((RankProperties)other).properties);
+    }
+
+    @Override
+    public int hashCode() {
+        return properties.hashCode();
+    }
+
+    @Override
+    public RankProperties clone() {
+        Map<String, List<Object>> clone = new LinkedHashMap<>();
+        for (Map.Entry<String, List<Object>> entry : properties.entrySet())
+            clone.put(entry.getKey(), new ArrayList<>(entry.getValue()));
+        return new RankProperties(clone);
+    }
+
+    @Override
+    public String toString() {
+        return JSON.encode(properties);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/package-info.java b/container-search/src/main/java/com/yahoo/search/query/ranking/package-info.java
new file mode 100644
index 00000000000..f254b327f96
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.ranking;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java
new file mode 100644
index 00000000000..bb76c1006f2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/QueryRewriteSearcher.java
@@ -0,0 +1,423 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite;
+
+import com.google.inject.Inject;
+import com.yahoo.search.*;
+import com.yahoo.config.*;
+import com.yahoo.search.query.rewrite.RewritesConfig.FsaDict;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.fsa.FSA;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.component.ComponentId;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * <p>A template class for all rewriters</p>
+ *
+ * <p>All rewriters extending this class would need to implement the
+ * rewrite method which contains the rewriter's main logic,
+ * getSkipRewriterIfRewritten method which indicates whether this
+ * rewriter should be skipped if the query has been rewritten,
+ * getRewriterName method which returns the name of the rewriter used
+ * in query profile, configure method which contains any instance
+ * creation time configuration besides the default FSA loading, and
+ * getDefaultDicts method which return the pair of dictionary name
+ * and filename.</p>
+ *
+ * <p>Common rewrite features are in RewriterFeatures.java.
+ * Common rewriter utils are in RewriterUtils.java.</p>
+ *
+ * @author Karen Sze Wing Lee
+ */
+public abstract class QueryRewriteSearcher extends Searcher {
+
+    // Indicate whether rewriter is properly initiated
+    private boolean isOk = false;
+
+    protected final Logger logger = Logger.getLogger(QueryRewriteSearcher.class.getName());
+
+    // HashMap which store the rewriter dicts
+    // It has the following format:
+    // HashMap<String(e.g. dictionary name, etc),
+    //         Object(e.g. FSA, etc)>>
+    protected HashMap<String, Object> rewriterDicts = new HashMap<>();
+
+    /**
+     * Constructor for this rewriter.
+     * Prepare the data needed by the rewriter
+     * @param id Component ID (see vespa's search container doc for more detail)
+     * @param fileAcquirer Required param for retrieving file type config
+     *                     (see vespa's search container doc for more detail)
+     * @param config Config from vespa-services.xml (see vespa's search
+     *               container doc for more detail)
+     */
+    @Inject
+    protected QueryRewriteSearcher(ComponentId id,
+                                   FileAcquirer fileAcquirer,
+                                   RewritesConfig config) {
+        super(id);
+        RewriterUtils.log(logger, "In QueryRewriteSearcher(ComponentId id, " +
+                                  "FileAcquirer fileAcquirer, " +
+                                  "RewritesConfig config)");
+        isOk = loadFSADicts(fileAcquirer, config, null);
+        isOk = isOk && configure(fileAcquirer, config, null);
+        if(isOk) {
+            RewriterUtils.log(logger, "Rewriter is configured properly");
+        } else {
+            RewriterUtils.log(logger, "Rewriter is not configured properly");
+        }
+    }
+
+    /**
+     * Constructor for unit test.
+     * Prepare the data needed by the rewriter
+     * @param config Config from vespa-services.xml (see vespa's search
+     *               container doc for more detail)
+     * @param fileList pairs of file name and file handler for unit tests
+     */
+    protected QueryRewriteSearcher(RewritesConfig config,
+                                   HashMap<String, File> fileList) {
+        RewriterUtils.log(logger, "In QueryRewriteSearcher(RewritesConfig config, " +
+                                  "HashMap<String, File> fileList)");
+        isOk = loadFSADicts(null, config, fileList);
+        isOk = isOk && configure(null, config, fileList);
+        if(isOk) {
+            RewriterUtils.log(logger, "Rewriter is configured properly");
+        } else {
+            RewriterUtils.log(logger, "Rewriter is not configured properly");
+        }
+    }
+
+    /**
+     * Empty constructor.
+     * Do nothing at instance creation time
+     */
+    protected QueryRewriteSearcher(ComponentId id) {
+        super(id);
+        RewriterUtils.log(logger, "In QueryRewriteSearcher(Component id)");
+        RewriterUtils.log(logger, "Configuring rewriter: " + getRewriterName());
+        isOk = true;
+        RewriterUtils.log(logger, "Rewriter is configured properly");
+    }
+
+    /**
+     * Empty constructor for unit test.
+     * Do nothing at instance creation time
+     */
+    protected QueryRewriteSearcher() {
+        RewriterUtils.log(logger, "In QueryRewriteSearcher()");
+        RewriterUtils.log(logger, "Configuring rewriter: " + getRewriterName());
+        isOk = true;
+        RewriterUtils.log(logger, "Rewriter is configured properly");
+    }
+
+    /**
+     * Load the dicts specified in vespa-services.xml
+     *
+     * @param fileAcquirer Required param for retrieving file type config
+     *                     (see vespa's search container doc for more detail)
+     * @param config Config from vespa-services.xml (see vespa's search
+     *               container doc for more detail)
+     * @param fileList pairs of file name and file handler for unit tests
+     * @return boolean true if loaded successfully, false otherwise
+     */
+    private boolean loadFSADicts(FileAcquirer fileAcquirer,
+                                 RewritesConfig config,
+                                 HashMap<String, File> fileList)
+                                 throws RuntimeException {
+
+        // Check if getRewriterName method is properly implemented
+        String rewriterName = getRewriterName();
+        if(rewriterName==null) {
+            RewriterUtils.error(logger, "Rewriter required method is not properly implemented: ");
+            return false;
+        }
+
+        RewriterUtils.log(logger, "Configuring rewriter: " + rewriterName);
+
+        // Check if there's no config need to be loaded
+        if(config==null || (fileAcquirer==null && fileList==null)) {
+            RewriterUtils.log(logger, "No FSA dictionary file need to be loaded");
+            return true;
+        }
+
+        // Check if config contains the FSADict param
+        if(config.fsaDict()==null) {
+            RewriterUtils.error(logger, "FSADict is not properly set in config");
+            return false;
+        }
+
+        RewriterUtils.log(logger, "Loading rewriter dictionaries");
+
+        // Retrieve FSA names and paths
+        ListIterator<FsaDict> fsaList = config.fsaDict().listIterator();
+
+        // Load default dictionaries if no user dictionaries is configured
+        if(!fsaList.hasNext()) {
+            RewriterUtils.log(logger, "Loading default dictionaries");
+            HashMap<String, String> defaultFSAs = getDefaultFSAs();
+
+            if(defaultFSAs==null) {
+                RewriterUtils.log(logger, "No default FSA dictionary is configured");
+                return true;
+            }
+            Iterator<Map.Entry<String, String>> defaultFSAList = defaultFSAs.entrySet().iterator();
+            while(defaultFSAList.hasNext()) {
+                try{
+                    Map.Entry<String, String> currFSA = defaultFSAList.next();
+                    String fsaName = currFSA.getKey();
+                    String fsaPath = currFSA.getValue();
+
+                    RewriterUtils.log(logger,
+                                      "FSA file location for " + fsaName + ": " + fsaPath);
+
+                    // Load FSA
+                    FSA fsa = RewriterUtils.loadFSA(RewriterConstants.DEFAULT_DICT_DIR + fsaPath, null);
+
+                    // Store FSA into dictionary map
+                    rewriterDicts.put(fsaName, fsa);
+                } catch (IOException e) {
+                    RewriterUtils.error(logger, "Error loading FSA dictionary: " +
+                                        e.getMessage());
+                    return false;
+                }
+            }
+        } else {
+            // Load user configured dictionaries
+            while(fsaList.hasNext()) {
+                try{
+                    FsaDict currFSA = fsaList.next();
+                    // fsaName and fsaPath are not null
+                    // or else vespa config server would not have been
+                    // able to start up
+                    String fsaName = currFSA.name();
+                    FileReference fsaPath = currFSA.path();
+
+                    RewriterUtils.log(logger,
+                                      "FSA file location for " + fsaName + ": " + fsaPath);
+
+                    // Retrieve FSA File handler
+                    File fsaFile = null;
+                    if(fileAcquirer!=null) {
+                        fsaFile = fileAcquirer.waitFor(fsaPath, 5, TimeUnit.MINUTES);
+                    } else if(fileList!=null) {
+                        fsaFile = fileList.get(fsaName);
+                    }
+
+                    if(fsaFile==null) {
+                        RewriterUtils.error(logger, "Error loading FSA dictionary file handler");
+                        return false;
+                    }
+
+                    // Load FSA
+                    FSA fsa = RewriterUtils.loadFSA(fsaFile, null);
+
+                    // Store FSA into dictionary map
+                    rewriterDicts.put(fsaName, fsa);
+                } catch (InterruptedException e1) {
+                    RewriterUtils.error(logger, "Error loading FSA dictionary file handler: " +
+                                        e1.getMessage());
+                    return false;
+                } catch (IOException e2) {
+                    RewriterUtils.error(logger, "Error loading FSA dictionary: " +
+                                        e2.getMessage());
+                    return false;
+                }
+            }
+        }
+        RewriterUtils.log(logger, "Successfully loaded rewriter dictionaries");
+        return true;
+    }
+
+    /**
+     * Perform instance creation time configuration besides the
+     * default FSA loading
+     *
+     * @param fileAcquirer Required param for retrieving file type config
+     *                     (see vespa's search container doc for more detail)
+     * @param config Config from vespa-services.xml (see vespa's search
+     *               container doc for more detail)
+     * @param fileList pairs of file name and file handler for unit tests
+     * @return boolean true if loaded successfully, false otherwise
+     */
+    public abstract boolean configure(FileAcquirer fileAcquirer,
+                                      RewritesConfig config,
+                                      HashMap<String, File> fileList)
+                                      throws RuntimeException;
+
+    /**
+     * Perform main rewrite logics for this searcher<br>
+     * - Skip to next rewriter if query is previously
+     *   rewritten and getSkipRewriterIfRewritten() is
+     *   true for this rewriter<br>
+     * - Execute rewriter's main rewrite logic<br>
+     * - Pass to the next rewriter the query to be used
+     *   for dictionary retrieval<br>
+     */
+    public @Override Result search(Query query, Execution execution) {
+        RewriterUtils.log(logger, query, "Executing " + getRewriterName());
+
+        // Check if rewriter is properly initialized
+        if(!isOk) {
+            RewriterUtils.error(logger, query, "Rewriter is not properly initialized");
+            return execution.search(query);
+        }
+
+        RewriterUtils.log(logger, query, "Original query: " + query.toDetailString());
+
+        // Retrieve metadata passed by previous rewriter
+        HashMap<String, Object> rewriteMeta = RewriterUtils.getRewriteMeta(query);
+
+        // This key would be updated by each rewriter to specify
+        // the key to be used for dict retrieval in next
+        // rewriter downstream. This controls whether the
+        // next rewriter should use the rewritten query or the
+        // original query for dict retrieval. e.g. rewriters
+        // following misspell rewriter should use the rewritten
+        // query by misspell rewriter for dict retrieval
+        String prevDictKey = (String)rewriteMeta.get(RewriterConstants.DICT_KEY);
+
+        // Whether the query has been rewritten
+        Boolean prevRewritten = (Boolean)rewriteMeta.get(RewriterConstants.REWRITTEN);
+
+        // Check if rewriter should be skipped if the query
+        // has been rewritten
+        if(prevRewritten && getSkipRewriterIfRewritten()) {
+            RewriterUtils.log(logger, query, "Skipping rewriter since the " +
+                              "query has been rewritten");
+            return execution.search(query);
+        }
+
+        // Store rewriter result
+        HashMap<String, Object> rewriterResult = null;
+        Query originalQueryObj = query.clone();
+
+        try {
+            // Execute rewriter's main rewrite logic
+            rewriterResult = rewrite(query, prevDictKey);
+
+        } catch (RuntimeException e) {
+            RewriterUtils.error(logger, originalQueryObj, "Error executing this rewriter, " +
+                                "skipping to next rewriter: " + e.getMessage());
+            return execution.search(originalQueryObj);
+        }
+
+        // Check if rewriter result is set properly
+        if(rewriterResult==null) {
+            RewriterUtils.error(logger, originalQueryObj, "Rewriter result are not set properly, " +
+                                "skipping to next rewriter");
+            return execution.search(originalQueryObj);
+        }
+
+        // Retrieve results from rewriter
+        Boolean rewritten = (Boolean)rewriterResult.get(RewriterConstants.REWRITTEN);
+        String dictKey = (String)rewriterResult.get(RewriterConstants.DICT_KEY);
+
+        if(rewritten==null || dictKey==null) {
+            RewriterUtils.error(logger, originalQueryObj, "Rewriter result are not set properly, " +
+                                "skipping to next rewriter");
+            return execution.search(originalQueryObj);
+        }
+
+        // Retrieve results from rewriter
+        rewriteMeta.put(RewriterConstants.REWRITTEN, (rewritten || prevRewritten));
+        rewriteMeta.put(RewriterConstants.DICT_KEY, dictKey);
+
+        // Pass metadata to the next rewriter
+        RewriterUtils.setRewriteMeta(query, rewriteMeta);
+
+        RewriterUtils.log(logger, query, "Final query: " + query.toDetailString());
+
+        return execution.search(query);
+    }
+
+    /**
+     * Perform the main rewrite logic
+     *
+     * @param query Query object from searcher
+     * @param dictKey the key passed from previous rewriter
+     *                to be treated as "original query from user"
+     *                For example, if previous is misspell rewriter,
+     *                it would pass the corrected query as the
+     *                "original query from user". For other rewriters which
+     *                add variants, abbr, etc to the query, the original
+     *                query should be passed as a key. This rewriter could
+     *                still choose to ignore this key. This key
+     *                is not the rewritten query itself. For example,
+     *                if original query is (willl smith) and the
+     *                rewritten query is (willl smith) OR (will smith)
+     *                the key to be passed could be (will smith)
+     * @return HashMap which contains the key value pairs:<br>
+     *         - whether this query has been rewritten by this
+     *           rewriter<br>
+     *           key: rewritten<br>
+     *           value: true or false<br>
+     *         - the key to be treated as "original query from user" in next
+     *           rewriter downstream, for example, misspell rewriter
+     *           would pass the corrected query as the "original query from
+     *           user" to the next rewriter. For other rewriters which
+     *           add variants, abbr, etc to the query, the original
+     *           query should be passed as a key. This key is not necessarily
+     *           consumed by the next rewriter. The next rewriter
+     *           can still choose to ignore this key.<br>
+     *           key: newDictKey<br>
+     *           value: new dict key<br>
+     */
+    protected abstract HashMap<String, Object> rewrite(Query query,
+                                                       String dictKey) throws RuntimeException;
+
+    /**
+     * Check whether rewriter should be skipped if
+     * the query has been rewritten by other rewriter
+     *
+     * @return boolean Whether rewriter should be skipped
+     */
+    protected abstract boolean getSkipRewriterIfRewritten();
+
+    /**
+     * Retrieve rewriter name
+     * It should match the name used in query profile
+     *
+     * @return Name of the rewriter
+     */
+    public abstract String getRewriterName();
+
+   /**
+    * Get default FSA dictionary names
+    *
+    * @return Pair of FSA dictionary name and filename
+    */
+   public abstract HashMap<String, String> getDefaultFSAs();
+
+    /**
+     * Get config parameter value set in query profile
+     *
+     * @param query Query object from the searcher
+     * @param paramName parameter to be retrieved
+     * @return parameter value or null if not found
+     */
+    protected String getQPConfig(Query query,
+                                 String paramName) {
+       return RewriterUtils.getQPConfig(query, getRewriterName(), paramName);
+    }
+
+    /**
+     * Retrieve rewrite from FSA given the original query
+     *
+     * @param query Query object from searcher
+     * @param dictName FSA dictionary name
+     * @param key The original query used to retrieve rewrite
+     *            from the dictionary
+     * @return String The retrieved rewrites, null if query
+     *         doesn't exist
+     */
+    protected String getRewriteFromFSA(Query query,
+                                       String dictName,
+                                       String key) throws RuntimeException {
+        return RewriterUtils.getRewriteFromFSA(query, rewriterDicts, dictName, key);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java
new file mode 100644
index 00000000000..45ce08de9d5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.vespa.defaults.Defaults;
+
+/**
+ * Contains common constant strings used by rewriters
+ *
+ * @author Karen Sze Wing Lee
+ */
+public class RewriterConstants {
+
+    /** Config flag for addUnitToOriginalQuery */
+    public static final String ORIGINAL_AS_UNIT = "OriginalAsUnit";
+
+    /** Config flag for addUnitEquivToOriginalQuery */
+    public static final String ORIGINAL_AS_UNIT_EQUIV = "OriginalAsUnitEquiv";
+
+    /** Config flag for addRewritesAsEquiv(false) */
+    public static final String REWRITES_AS_EQUIV = "RewritesAsEquiv";
+
+    /** Config flag for addRewritesAsEquiv(true) */
+    public static final String REWRITES_AS_UNIT_EQUIV = "RewritesAsUnitEquiv";
+
+    /** Config flag for addExpansions */
+    public static final String PARTIAL_PHRASE_MATCH = "PartialPhraseMatch";
+
+    /** Config flag for max number of rewrites added per rewriter */
+    public static final String MAX_REWRITES = "MaxRewrites";
+
+    /** Config flag for considering QSS Rewrite in spell correction */
+    public static final String QSS_RW = "QSSRewrite";
+
+    /** Config flag for considering QSS Suggest in spell correction */
+    public static final String QSS_SUGG = "QSSSuggest";
+
+    /** Config flag for expansion index name */
+    public static final String EXPANSION_INDEX = "ExpansionIndex";
+
+    /** Name for market chain retrieval from user param */
+    public static final String REWRITER_CHAIN = "QRWChain";
+
+    /** Name for rewrite metadata retrieval from query properties */
+    public static final CompoundName REWRITE_META = new CompoundName("RewriteMeta");
+
+    /** Name for rewritten field retrieval from query properties */
+    public static final String REWRITTEN = "Rewritten";
+
+    /** Name for new dictionary key field retrieval from query properties */
+    public static final String DICT_KEY = "DictKey";
+
+    /** Default dictionaries dir */
+    public static final String DEFAULT_DICT_DIR = Defaults.getDefaults().vespaHome() + "share/qrw_data/";
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterFeatures.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterFeatures.java
new file mode 100644
index 00000000000..0a5110dbd7e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterFeatures.java
@@ -0,0 +1,651 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.parser.CustomParser;
+import com.yahoo.search.*;
+import com.yahoo.search.query.*;
+import com.yahoo.prelude.query.*;
+import com.yahoo.prelude.querytransform.PhraseMatcher;
+import com.yahoo.prelude.querytransform.PhraseMatcher.Phrase;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+
+/**
+ * Contains commonly used rewriter features
+ *
+ * @author Karen Sze Wing Lee
+ */
+public class RewriterFeatures {
+
+    private static final Logger logger = Logger.getLogger(RewriterFeatures.class.getName());
+
+    /**
+     * <p>Add proximity boosting to original query by modifying
+     * the query tree directly</p>
+     * e.g. original Query Tree: (AND aa bb)<br>
+     *      if keepOriginalQuery: true<br>
+     *      new Query tree: (OR (AND aa bb) "aa bb")<br>
+     *      if keepOriginalQuery: false<br>
+     *      new Query Tree: "aa bb"<br><br>
+     *
+     *      original Query Tree: (OR (AND aa bb) (AND cc dd))<br>
+     *      boostingQuery: cc dd<br>
+     *      if keepOriginalQuery: true<br>
+     *      new Query Tree: (OR (AND aa bb) (AND cc dd) "cc dd")<br>
+     *      if keepOriginalQuery: false<br>
+     *      new Query Tree: (OR (AND aa bb) "cc dd") <br>
+     *
+     * @param query Query object from searcher
+     * @param boostingQuery query to be boosted
+     * @param keepOriginalQuery whether to keep original unboosted query as equiv
+     * @return Modified Query object, return original query object
+     *          on error
+     */
+    public static Query addUnitToOriginalQuery(Query query, String boostingQuery,
+                                               boolean keepOriginalQuery)
+                                               throws RuntimeException {
+        RewriterUtils.log(logger, query, "Adding proximity boosting to [" + boostingQuery + "]");
+
+        Model queryModel = query.getModel();
+        QueryTree qTree = queryModel.getQueryTree();
+        Item oldRoot = qTree.getRoot();
+
+        if (oldRoot == null) {
+            RewriterUtils.error(logger, query, "Error retrieving query tree root");
+            throw new RuntimeException("Error retrieving query tree root");
+        }
+
+        // Convert original query to query tree item
+        Item origQueryItem = convertStringToQTree(query, boostingQuery);
+
+        // Boost proximity by phrasing the original query
+        // query tree structure: (AND aa bb)
+        if (oldRoot instanceof AndItem &&
+            oldRoot.equals(origQueryItem)) {
+            PhraseItem phrase = convertAndToPhrase((AndItem)oldRoot);
+
+            if(!keepOriginalQuery) {
+                qTree.setRoot(phrase);
+            } else {
+                OrItem newRoot = new OrItem();
+                newRoot.addItem(oldRoot);
+                newRoot.addItem(phrase);
+                qTree.setRoot(newRoot);
+                queryModel.setType(Query.Type.ADVANCED); //set type=adv
+            }
+            RewriterUtils.log(logger, query, "Added proximity boosting successfully");
+            return query;
+
+        // query tree structure: (OR (AND aa bb) (AND cc dd))
+        } else if (oldRoot instanceof OrItem &&
+                   ((OrItem)oldRoot).getItemIndex(origQueryItem)!=-1 &&
+                   origQueryItem instanceof AndItem) {
+
+            // Remove original unboosted query
+            if(!keepOriginalQuery)
+                ((OrItem)oldRoot).removeItem(origQueryItem);
+
+            // Check if the tree already contained the phrase item
+            PhraseItem pI = convertAndToPhrase((AndItem)origQueryItem);
+            if(((OrItem)oldRoot).getItemIndex(pI)==-1) {
+                ((OrItem)oldRoot).addItem(convertAndToPhrase((AndItem)origQueryItem));
+                RewriterUtils.log(logger, query, "Added proximity boosting successfully");
+                return query;
+            }
+        }
+        RewriterUtils.log(logger, query, "No proximity boosting added");
+        return query;
+    }
+
+    /**
+     * <p>Add query expansion to the query tree</p>
+     * e.g. origQuery: aa bb<br>
+     *      matchingStr: aa bb<br>
+     *      rewrite: cc dd, ee ff<br>
+     *      if addUnitToRewrites: false<br>
+     *      new query tree: (OR (AND aa bb) (AND cc dd) (AND ee ff))<br>
+     *      if addUnitToRewrites: true<br>
+     *      new query tree: (OR (AND aa bb) "cc dd" "ee ff") <br>
+     *
+     * @param query Query object from searcher
+     * @param matchingStr string used to retrieve the rewrite
+     * @param rewrites The rewrite string retrieved from
+     *                 dictionary
+     * @param addUnitToRewrites Whether to add unit to rewrites
+     * @param maxNumRewrites Max number of rewrites to be added,
+     *                       0 if no limit
+     * @return Modified Query object, return original query object
+     *         on error
+     */
+    public static Query addRewritesAsEquiv(Query query, String matchingStr,
+                                           String rewrites,
+                                           boolean addUnitToRewrites,
+                                           int maxNumRewrites) throws RuntimeException {
+        String normalizedQuery = RewriterUtils.getNormalizedOriginalQuery(query);
+
+        RewriterUtils.log(logger, query,
+                          "Adding rewrites [" + rewrites +
+                          "] to the query [" + normalizedQuery + "]");
+        if (rewrites.equalsIgnoreCase(normalizedQuery) || rewrites.equalsIgnoreCase("n/a")) {
+            RewriterUtils.log(logger, query, "No rewrite added");
+            return query;
+        }
+
+        Model queryModel = query.getModel();
+        QueryTree qTree = queryModel.getQueryTree();
+        Item oldRoot = qTree.getRoot();
+
+        if (oldRoot == null) {
+            RewriterUtils.error(logger, query, "Error retrieving query tree root");
+            throw new RuntimeException("Error retrieving query tree root");
+        }
+
+        StringTokenizer rewrite_list = new StringTokenizer(rewrites, "\t");
+        Item rI = null;
+
+        // Convert matching string to query tree item
+        Item matchingStrItem = convertStringToQTree(query, matchingStr);
+        PhraseItem matchingStrPhraseItem = null;
+        if(matchingStrItem instanceof AndItem) {
+            matchingStrPhraseItem = convertAndToPhrase(((AndItem)matchingStrItem));
+        }
+
+        // Add rewrites as OR item to the query tree
+        // Only should rewrite in this case:
+        // - origQuery: (OR (AND aa bb) (AND cc dd))
+        // - matchingStr: (AND aa bb)
+        // Or in this case:
+        // - origQuery: (AND aa bb)
+        // - matching Str: (AND aa bb)
+        // Should not rewrite in this case:
+        // - origQuery: (OR (AND cc (OR dd (AND aa bb)) ee)
+        // - matchingStr: (AND aa bb)
+        // - for this case, should use getNonOverlappingMatches instead
+        OrItem newRoot;
+        if(oldRoot instanceof OrItem) {
+            if(((OrItem)oldRoot).getItemIndex(matchingStrItem)==-1) {
+                RewriterUtils.log(logger, query, "Whole query matching is used, skipping rewrite");
+                return query;
+            }
+            newRoot = (OrItem)oldRoot;
+        } else if(oldRoot.equals(matchingStrItem) || oldRoot.equals(matchingStrPhraseItem)) {
+            newRoot = new OrItem();
+            newRoot.addItem(oldRoot);
+        } else {
+            RewriterUtils.log(logger, query, "Whole query matching is used, skipping rewrite");
+            return query;
+        }
+        int numRewrites = 0;
+        while(rewrite_list.hasMoreTokens() &&
+              (maxNumRewrites==0 || numRewrites < maxNumRewrites)) {
+            rI = convertStringToQTree(query, rewrite_list.nextToken());
+            if(addUnitToRewrites && rI instanceof AndItem) {
+                rI = convertAndToPhrase((AndItem)rI);
+            }
+            if(newRoot.getItemIndex(rI)==-1) {
+                newRoot.addItem(rI);
+                numRewrites++;
+            } else {
+                RewriterUtils.log(logger, query, "Rewrite already exist, skipping");
+            }
+        }
+        qTree.setRoot(newRoot);
+        queryModel.setType(Query.Type.ADVANCED); //set type=adv
+        RewriterUtils.log(logger, query, "Added rewrite successfully");
+
+        return query;
+    }
+
+    /**
+     * <p>Retrieve the longest, from left to right non overlapping full
+     * phrase substrings in query based on FSA dictionary</p>
+     *
+     * e.g. query: ((modern AND new AND york AND city AND travel) OR travel) AND
+     *             ((sunny AND travel AND agency) OR nyc)<br>
+     *      dictionary: <br>
+     *                  mny\tmodern new york<br>
+     *                  mo\tmodern<br>
+     *                  modern\tn/a<br>
+     *                  modern\tnew york\tn/a<br>
+     *                  new york\tn/a<br>
+     *                  new york city\tn/a<br>
+     *                  new york city travel\tn/a<br>
+     *                  new york company\tn/a<br>
+     *                  ny\tnew york<br>
+     *                  nyc\tnew york city\tnew york company<br>
+     *                  nyct\tnew york city travel<br>
+     *                  ta\ttravel agency<br>
+     *                  travel agency\tn/a<br>
+     *      return: nyc
+     * @param phraseMatcher PhraseMatcher object loaded with FSA dict
+     * @param query Query object from the searcher
+     * @return Matching phrases
+     */
+    public static Set<PhraseMatcher.Phrase> getNonOverlappingFullPhraseMatches(PhraseMatcher phraseMatcher,
+                                                                               Query query)
+                                                                               throws RuntimeException {
+        RewriterUtils.log(logger, query, "Retrieving longest non-overlapping full phrase matches");
+        if(phraseMatcher==null)
+            return null;
+
+        Item root = query.getModel().getQueryTree().getRoot();
+        List<PhraseMatcher.Phrase> matches = phraseMatcher.matchPhrases(root);
+        if (matches==null || matches.isEmpty())
+            return null;
+
+        Set<PhraseMatcher.Phrase> resultMatches = new HashSet<>();
+        ListIterator<Phrase> matchesIter = matches.listIterator();
+
+        // Iterate through all matches
+        while(matchesIter.hasNext()) {
+            PhraseMatcher.Phrase phrase = matchesIter.next();
+            RewriterUtils.log(logger, query, "Working on phrase: " + phrase);
+            CompositeItem currOwner = phrase.getOwner();
+
+            // Check if this is full phrase
+            // If phrase is not an AND item, only keep those that are single word
+            // in order to eliminate cases such as (new RANK york) from being treated
+            // as match if only new york but not new or york is in the dictionary
+            if((currOwner!=null &&
+                ((phrase.isComplete() && currOwner instanceof AndItem) ||
+                 (phrase.getLength()==1 && currOwner instanceof OrItem) ||
+                 (phrase.getLength()==1 && currOwner instanceof RankItem && phrase.getStartIndex()==0))) ||
+               (currOwner==null && phrase.getLength()==1)) {
+               resultMatches.add(phrase);
+               RewriterUtils.log(logger, query, "Keeping phrase: " + phrase);
+            }
+        }
+
+        RewriterUtils.log(logger, query, "Successfully Retrieved longest non-overlapping full phrase matches");
+        return resultMatches;
+    }
+
+
+    /**
+     * <p>Retrieve the longest, from left to right non overlapping partial
+     * phrase substrings in query based on FSA dictionary</p>
+     *
+     * e.g. query: ((modern AND new AND york AND city AND travel) OR travel) AND
+     *             ((sunny AND travel AND agency) OR nyc)<br>
+     *      dictionary: <br>
+     *                  mny\tmodern new york<br>
+     *                  mo\tmodern<br>
+     *                  modern\tn/a<br>
+     *                  modern new york\tn/a<br>
+     *                  new york\tn/a<br>
+     *                  new york city\tn/a<br>
+     *                  new york city travel\tn/a<br>
+     *                  new york company\tn/a<br>
+     *                  ny\tnew york<br>
+     *                  nyc\tnew york city\tnew york company<br>
+     *                  nyct\tnew york city travel<br>
+     *                  ta\ttravel agency<br>
+     *                  travel agency\tn/a<br>
+     *      return: <br>
+     *              modern<br>
+     *              new york city travel<br>
+     *              travel agency<br>
+     *              nyc<br>
+     * @param phraseMatcher PhraseMatcher object loaded with FSA dict
+     * @param query Query object from the searcher
+     * @return Matching phrases
+     */
+    public static Set<PhraseMatcher.Phrase> getNonOverlappingPartialPhraseMatches(PhraseMatcher phraseMatcher,
+                                                                                  Query query)
+                                                                                  throws RuntimeException {
+        RewriterUtils.log(logger, query, "Retrieving longest non-overlapping partial phrase matches");
+        if(phraseMatcher==null)
+            return null;
+
+        Item root = query.getModel().getQueryTree().getRoot();
+        List<PhraseMatcher.Phrase> matches = phraseMatcher.matchPhrases(root);
+        if (matches==null || matches.isEmpty())
+            return null;
+
+        Set<PhraseMatcher.Phrase> resultMatches = new HashSet<>();
+        ArrayList<PhraseMatcher.Phrase> phrasesInSubTree = new ArrayList<>();
+        CompositeItem prevOwner = null;
+        ListIterator<PhraseMatcher.Phrase> matchesIter = matches.listIterator();
+
+        // Iterate through all matches
+        while(matchesIter.hasNext()) {
+            PhraseMatcher.Phrase phrase = matchesIter.next();
+            RewriterUtils.log(logger, query, "Working on phrase: " + phrase);
+            CompositeItem currOwner = phrase.getOwner();
+
+            // Check if previous is AND item and this phrase is in a different item
+            // If so, work on the previous set to eliminate overlapping matches
+            if(!phrasesInSubTree.isEmpty() && currOwner!=null &&
+               prevOwner!=null && !currOwner.equals(prevOwner)) {
+                RewriterUtils.log(logger, query, "Previous phrase is in different AND item");
+                List<PhraseMatcher.Phrase> subTreeMatches
+                    = getNonOverlappingMatchesInAndItem(phrasesInSubTree, query);
+                if(subTreeMatches==null) {
+                    RewriterUtils.error(logger, query, "Error retrieving matches from subtree");
+                    throw new RuntimeException("Error retrieving matches from subtree");
+                }
+                resultMatches.addAll(subTreeMatches);
+                phrasesInSubTree.clear();
+            }
+
+            // Check if this is an AND item
+            if(currOwner!=null && currOwner instanceof AndItem) {
+                phrasesInSubTree.add(phrase);
+            // If phrase is not an AND item, only keep those that are single word
+            // in order to eliminate cases such as (new RANK york) from being treated
+            // as match if only new york but not new or york is in the dictionary
+            } else if (phrase.getLength()==1 &&
+                       !(currOwner!=null && currOwner instanceof RankItem && phrase.getStartIndex()!=0)) {
+                resultMatches.add(phrase);
+            }
+
+            prevOwner = currOwner;
+        }
+
+        // Check if last item is AND item
+        // If so, work on the previous set to elimate overlapping matches
+        if(!phrasesInSubTree.isEmpty()) {
+            RewriterUtils.log(logger, query, "Last phrase is in AND item");
+            List<PhraseMatcher.Phrase> subTreeMatches
+                = getNonOverlappingMatchesInAndItem(phrasesInSubTree, query);
+            if(subTreeMatches==null) {
+                RewriterUtils.error(logger, query, "Error retrieving matches from subtree");
+                throw new RuntimeException("Error retrieving matches from subtree");
+            }
+            resultMatches.addAll(subTreeMatches);
+        }
+        RewriterUtils.log(logger, query, "Successfully Retrieved longest non-overlapping partial phrase matches");
+        return resultMatches;
+    }
+
+    /**
+     * <p>Retrieve the longest, from left to right non overlapping substrings in
+     * AndItem based on FSA dictionary</p>
+     *
+     * e.g. subtree: (modern AND new AND york AND city AND travel)<br>
+     *      dictionary:<br>
+     *                  mny\tmodern new york<br>
+     *                  mo\tmodern<br>
+     *                  modern\tn/a<br>
+     *                  modern new york\tn/a<br>
+     *                  new york\tn/a<br>
+     *                  new york city\tn/a<br>
+     *                  new york city travel\tn/a<br>
+     *                  new york company\tn/a<br>
+     *                  ny\tnew york<br>
+     *                  nyc\tnew york city\tnew york company<br>
+     *                  nyct\tnew york city travel<br>
+     *      allMatches:<br>
+     *                  modern<br>
+     *                  modern new york<br>
+     *                  new york<br>
+     *                  new york city<br>
+     *                  new york city travel<br>
+     *      return: <br>
+     *              modern<br>
+     *              new york city travel<br>
+     * @param allMatches All matches within the subtree
+     * @param query Query object from the searcher
+     * @return Matching phrases
+     */
+    public static List<PhraseMatcher.Phrase> getNonOverlappingMatchesInAndItem(
+                                             List<PhraseMatcher.Phrase> allMatches,
+                                             Query query)
+                                             throws RuntimeException {
+        RewriterUtils.log(logger, query, "Retrieving longest non-overlapping matches in subtree");
+
+        if (allMatches==null || allMatches.isEmpty())
+            return null;
+
+        if(allMatches.size()==1) {
+            RewriterUtils.log(logger, query, "Only one match in subtree");
+            return allMatches;
+        }
+
+        // Phrase are sorted based on length, if both have the
+        // same length, the lefter one ranks higher
+        RewriterUtils.log(logger, query, "Sorting the phrases");
+        PhraseLength phraseLength = new PhraseLength();
+        Collections.sort(allMatches, phraseLength);
+
+        // Create a bitset with length equal to the number of
+        // items in the subtree
+        int numWords = allMatches.get(0).getOwner().getItemCount();
+        BitSet matchPos = new BitSet(numWords);
+
+        // Removing matches that are overlapping with previously selected ones
+        RewriterUtils.log(logger, query, "Removing matches that are overlapping " +
+                                         "with previously selected ones");
+        ListIterator<Phrase> allMatchesIter = allMatches.listIterator();
+        while(allMatchesIter.hasNext()) {
+            PhraseMatcher.Phrase currMatch = allMatchesIter.next();
+            PhraseMatcher.Phrase.MatchIterator matchIter = currMatch.itemIterator();
+            if(matchIter.hasNext() && matchIter.next().isFilter()) {
+                RewriterUtils.log(logger, query, "Removing filter item" + currMatch);
+                allMatchesIter.remove();
+                continue;
+            }
+
+            BitSet currMatchPos = new BitSet(numWords);
+            currMatchPos.set(currMatch.getStartIndex(),
+                             currMatch.getLength()+currMatch.getStartIndex());
+            if(currMatchPos.intersects(matchPos)) {
+                RewriterUtils.log(logger, query, "Removing " + currMatch);
+                allMatchesIter.remove();
+            } else {
+                RewriterUtils.log(logger, query, "Keeping " + currMatch);
+                matchPos.or(currMatchPos);
+            }
+        }
+        return allMatches;
+    }
+
+    /**
+     * <p>Add Expansions to the matching phrases</p>
+     *
+     * e.g. Query: nyc travel agency<br>
+     *      matching phrase: nyc\tnew york city\tnew york company
+     *                       travel agency\tn/a<br>
+     *      if expandIndex is not null and removeOriginal is true<br>
+     *      New Query: ((new york city) OR ([expandIndex]:new york city)
+     *                  OR (new york company) OR
+     *                  ([expandIndex]:new york company)) AND
+     *                 ((travel agency) OR ([expandIndex]:travel agency))<br>
+     *      if expandIndex is null and removeOriginal is true<br>
+     *      New Query: ((new york city) OR (new york company)) AND
+     *                 travel agency<br>
+     *      if expandIndex is null and removeOriginal is false<br>
+     *      New Query: (nyc OR (new york city) OR (new york company)) AND
+     *                 travel agency<br>
+     *
+     * @param query Query object from searcher
+     * @param matches Set of longest non-overlapping matches
+     * @param expandIndex Name of expansion index or null if
+     *                    default index
+     * @param maxNumRewrites Max number of rewrites to be added,
+     *                       0 if no limit
+     * @param removeOriginal Whether to remove the original matching phrase
+     * @param addUnitToRewrites Whether to add rewrite as phrase
+     */
+    public static Query addExpansions(Query query, Set<PhraseMatcher.Phrase> matches,
+                                      String expandIndex, int maxNumRewrites,
+                                      boolean removeOriginal, boolean addUnitToRewrites)
+                                      throws RuntimeException {
+
+        if(matches==null) {
+            RewriterUtils.log(logger, query, "No expansions to be added");
+            return query;
+        }
+
+        RewriterUtils.log(logger, query, "Adding expansions to matching phrases");
+        Model queryModel = query.getModel();
+        QueryTree qTree = queryModel.getQueryTree();
+        Iterator<Phrase> matchesIter = matches.iterator();
+        CompositeItem parent = null;
+
+        // Iterate through all matches
+        while(matchesIter.hasNext()) {
+            PhraseMatcher.Phrase match = matchesIter.next();
+            RewriterUtils.log(logger, query, "Working on phrase: " + match);
+
+            // Retrieve expansion phrases
+            String expansionStr = match.getData();
+            if(expansionStr.equalsIgnoreCase("n/a") && expandIndex==null) {
+                continue;
+            }
+            StringTokenizer expansions = new StringTokenizer(expansionStr,"\t");
+
+            // Create this structure for all expansions of this match
+            // (OR (AND expandsion1) indexName:expansion1
+            //  (AND expansion2) indexName:expansion2..)
+            OrItem expansionGrp = new OrItem();
+            int numRewrites = 0;
+            String matchStr = convertMatchToString(match);
+            while(expansions.hasMoreTokens() &&
+                  (maxNumRewrites==0 || numRewrites < maxNumRewrites)) {
+       	        String expansion = expansions.nextToken();
+                RewriterUtils.log(logger, query, "Working on expansion: " + expansion);
+                if(expansion.equalsIgnoreCase("n/a")) {
+                    expansion = matchStr;
+                }
+                // (AND expansion) or "expansion"
+                Item expansionItem = convertStringToQTree(query, expansion);
+                if(addUnitToRewrites && expansionItem instanceof AndItem) {
+                   expansionItem = convertAndToPhrase((AndItem)expansionItem);
+                }
+                expansionGrp.addItem(expansionItem);
+
+                if(expandIndex!=null) {
+                    // indexName:expansion
+                    WordItem expansionIndexItem = new WordItem(expansion, expandIndex);
+                    expansionGrp.addItem(expansionIndexItem);
+                }
+                numRewrites++;
+                RewriterUtils.log(logger, query, "Adding expansion: " + expansion);
+            }
+
+            if(!removeOriginal) {
+                //(AND original)
+                Item matchItem = convertStringToQTree(query, matchStr);
+                if(expansionGrp.getItemIndex(matchItem)==-1) {
+                    expansionGrp.addItem(matchItem);
+                }
+            }
+
+            parent = match.getOwner();
+            int matchIndex = match.getStartIndex();
+            if(parent!=null) {
+                // Remove matching phrase from original query
+                for(int i=0; i<match.getLength(); i++) {
+                    parent.removeItem(matchIndex);
+                }
+                // Adding back expansions
+                parent.addItem(matchIndex, expansionGrp);
+            } else {
+               RewriterUtils.log(logger, query, "Single root item");
+               // If there's no parent, i.e. single root item
+               qTree.setRoot(expansionGrp);
+               break;
+            }
+        }
+
+        // Not root single item
+        if(parent!=null) {
+            // Cleaning up the query after rewrite to remove redundant tags
+            // e.g. (AND (OR (AND a b) c)) => (OR (AND a b) c)
+            String cleanupError = QueryCanonicalizer.canonicalize(qTree);
+            if(cleanupError!=null) {
+                RewriterUtils.error(logger, query, "Error canonicalizing query tree");
+                throw new RuntimeException("Error canonicalizing query tree");
+            }
+        }
+        queryModel.setType(Query.Type.ADVANCED); //set type=adv
+        RewriterUtils.log(logger, query, "Successfully added expansions to matching phrases");
+        return query;
+    }
+
+    /**
+     * Convert Match to String
+     *
+     * @param phrase Match from PhraseMatcher
+     * @return String format of the phrase
+     */
+    public static String convertMatchToString(PhraseMatcher.Phrase phrase) {
+        StringBuilder buffer = new StringBuilder();
+        for (Iterator<Item> i = phrase.itemIterator(); i.hasNext();) {
+            buffer.append(i.next().toString());
+            if (i.hasNext()) {
+                buffer.append(" ");
+            }
+        }
+        return buffer.toString();
+    }
+
+    /**
+     * Convert String to query tree
+     *
+     * @param stringToParse The string to be converted to a
+     *                      query tree
+     * @param query Query object from searcher
+     * @return Item The resulting query tree
+     */
+    static Item convertStringToQTree(Query query, String stringToParse) {
+        RewriterUtils.log(logger, query, "Converting string [" + stringToParse + "] to query tree");
+        if(stringToParse==null) {
+            return new NullItem();
+        }
+        Model model = query.getModel();
+        CustomParser parser = (CustomParser) ParserFactory.newInstance(model.getType(),
+                ParserEnvironment.fromExecutionContext(query.getModel().getExecution().context()));
+        IndexFacts indexFacts = new IndexFacts();
+        Item item = parser.parse(stringToParse, null, model.getParsingLanguage(),
+                                 indexFacts.newSession(model.getSources(), model.getRestrict()),
+                                 model.getDefaultIndex());
+        RewriterUtils.log(logger, query, "Converted string: [" + item.toString() + "]");
+        return item;
+    }
+
+    /**
+     * Convert AndItem to PhraseItem<br>
+     *
+     * e.g. (AND a b) to "a b"
+     * @param andItem query tree to be converted
+     * @return converted PhraseItem
+     */
+    private static PhraseItem convertAndToPhrase(AndItem andItem) {
+        PhraseItem result = new PhraseItem();
+        Iterator<Item> subItems = andItem.getItemIterator();
+        while(subItems.hasNext()) {
+            Item curr = (subItems.next());
+            if(curr instanceof IntItem) {
+                WordItem numItem = new WordItem(((IntItem)curr).stringValue());
+                result.addItem(numItem);
+            } else {
+                result.addItem(curr);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Class for comparing phrase.
+     * A phrase is larger if its length is longer.
+     * If both phrases are of the same length, the lefter one
+     * is considered larger
+     */
+    private static class PhraseLength implements Comparator<PhraseMatcher.Phrase> {
+        public int compare(PhraseMatcher.Phrase phrase1, PhraseMatcher.Phrase phrase2) {
+            if((phrase2.getLength()>phrase1.getLength()) ||
+               (phrase2.getLength()==phrase1.getLength() &&
+                phrase2.getStartIndex()<=phrase1.getStartIndex())) {
+                return 1;
+            } else {
+                return -1;
+            }
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java
new file mode 100644
index 00000000000..26ead8de5e5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java
@@ -0,0 +1,334 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite;
+
+import com.yahoo.fsa.FSA;
+import com.yahoo.log.LogLevel;
+import com.yahoo.search.Query;
+import com.yahoo.search.intent.model.IntentModel;
+import com.yahoo.search.intent.model.InterpretationNode;
+import com.yahoo.text.interpretation.Annotations;
+import com.yahoo.text.interpretation.Modification;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.logging.Logger;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * Contains common utilities used by rewriters
+ *
+ * @author Karen Sze Wing Lee
+ */
+public class RewriterUtils {
+
+    private static final Logger utilsLogger = Logger.getLogger(RewriterUtils.class.getName());
+
+    // Tracelevel for debug log of this rewriter
+    private static final int TRACELEVEL = 3;
+
+    /**
+     * Load FSA from file
+     *
+     * @param file FSA dictionary file object
+     * @param query Query object from the searcher, could be null if not available
+     * @return FSA The FSA object for the input file path
+     */
+    public static FSA loadFSA(File file, Query query) throws IOException {
+        log(utilsLogger, query, "Loading FSA file");
+        String filePath = null;
+
+        try {
+            filePath = file.getAbsolutePath();
+        } catch (SecurityException e1) {
+            error(utilsLogger, query, "No read access for the FSA file");
+            throw new IOException("No read access for the FSA file");
+        }
+
+        FSA fsa = loadFSA(filePath, query);
+
+        return fsa;
+    }
+
+    /**
+     * Load FSA from file
+     *
+     * @param filename FSA dictionary file path
+     * @param query Query object from the searcher, could be null if not available
+     * @return FSA The FSA object for the input file path
+     */
+    public static FSA loadFSA(String filename, Query query) throws IOException {
+        log(utilsLogger, query, "Loading FSA file from: " + filename);
+
+        if(!new File(filename).exists()) {
+            error(utilsLogger, query, "File does not exist : " + filename);
+            throw new IOException("File does not exist : " + filename);
+        }
+
+        FSA fsa;
+        try {
+            fsa = new FSA(filename);
+        } catch (RuntimeException e) {
+            error(utilsLogger, query, "Invalid FSA file");
+            throw new IOException("Invalid FSA file");
+        }
+
+        if (!fsa.isOk()) {
+            error(utilsLogger, query, "Unable to load FSA file from : " + filename);
+            throw new IOException("Not able to load FSA file from : " + filename);
+        }
+        log(utilsLogger, query, "Loaded FSA successfully from file : " + filename);
+        return fsa;
+    }
+
+    /**
+     * Retrieve rewrite from FSA given the original query
+     *
+     * @param query Query object from searcher
+     * @param dictName FSA dictionary name
+     * @param rewriterDicts list of rewriter dictionaries
+     *                      It has the following format:
+     *                      HashMap&lt;dictionary name, FSA&gt;
+     * @param key The original query used to retrieve rewrite
+     *            from the dictionary
+     * @return String The retrieved rewrites, null if query
+     *         doesn't exist
+     */
+    public static String getRewriteFromFSA(Query query,
+                                           HashMap<String, Object> rewriterDicts,
+                                           String dictName,
+                                           String key) throws RuntimeException {
+        if(rewriterDicts==null) {
+            error(utilsLogger, query, "HashMap containing rewriter dicts is null");
+            throw new RuntimeException("HashMap containing rewriter dicts is null");
+        }
+
+        FSA fsa = (FSA)rewriterDicts.get(dictName);
+
+        if(fsa==null) {
+            error(utilsLogger, query, "Error retrieving FSA dictionary: " + dictName);
+            throw new RuntimeException("Error retrieving FSA dictionary: " + dictName);
+        }
+
+        String result = null;
+        result = fsa.lookup(key);
+        log(utilsLogger, query, "Retrieved rewrite: " + result);
+
+        return result;
+    }
+
+    /**
+     * Get config parameter value set in query profile
+     *
+     * @param query Query object from the searcher
+     * @param rewriterName Name of the rewriter
+     * @param paramName parameter to be retrieved
+     * @return parameter value or null if not found
+     */
+    public static String getQPConfig(Query query,
+                                     String rewriterName,
+                                     String paramName) {
+        log(utilsLogger, query, "Retrieving config parameter value of: " +
+            rewriterName + "." + paramName);
+
+        return getUserParam(query, rewriterName + "." + paramName);
+    }
+
+    /**
+     * Get rewriter chain value
+     *
+     * @param query Query object from the searcher
+     * @return parameter value or null if not found
+     */
+    public static String getRewriterChain(Query query) {
+        log(utilsLogger, query, "Retrieving rewriter chain value: " +
+            RewriterConstants.REWRITER_CHAIN);
+
+        return getUserParam(query, RewriterConstants.REWRITER_CHAIN);
+    }
+
+    /**
+     * Get user param value
+     *
+     * @param query Query object from the searcher
+     * @param paramName parameter to be retrieved
+     *
+     * @return parameter value or null if not found
+     */
+    public static String getUserParam(Query query, String paramName) {
+        log(utilsLogger, query, "Retrieving user param value: " + paramName);
+
+        if(paramName==null) {
+            error(utilsLogger, query, "Parameter name is null");
+            return null;
+        }
+
+        String paramValue = null;
+        paramValue = query.properties().getString(paramName);
+        log(utilsLogger, query, "Param value retrieved is: " + paramValue);
+
+        return paramValue;
+    }
+
+    /**
+     * Retrieve metadata passed by previous rewriter
+     * from query properties
+     * Initialize values if this is the first rewriter
+     *
+     * @param query Query object from the searcher
+     * @return hashmap containing the metadata
+     */
+    public static HashMap<String, Object> getRewriteMeta(Query query) {
+       log(utilsLogger, query, "Retrieving metadata passed by previous rewriter");
+
+        @SuppressWarnings("unchecked")
+        HashMap<String, Object> rewriteMeta = (HashMap<String, Object>) query
+                .properties().get(RewriterConstants.REWRITE_META);
+
+       if(rewriteMeta==null) {
+           log(utilsLogger, query, "No metadata available from previous rewriter");
+           rewriteMeta = new HashMap<>();
+           rewriteMeta.put(RewriterConstants.REWRITTEN, false);
+           rewriteMeta.put(RewriterConstants.DICT_KEY, getNormalizedOriginalQuery(query));
+       } else {
+           if((Boolean)rewriteMeta.get(RewriterConstants.REWRITTEN)) {
+               log(utilsLogger, query, "Query has been rewritten by previous rewriters");
+           } else {
+               log(utilsLogger, query, "Query has not been rewritten by previous rewriters");
+           }
+           log(utilsLogger, query, "Dict key passed by previous rewriter: " +
+                                   rewriteMeta.get(RewriterConstants.DICT_KEY));
+       }
+
+       return rewriteMeta;
+    }
+
+    /**
+     * Pass metadata to the next rewriter through query properties
+     *
+     * @param query Query object from the searcher
+     * @param metadata HashMap containing the metadata
+     */
+    public static void setRewriteMeta(Query query, HashMap<String, Object> metadata) {
+        log(utilsLogger, query, "Passing metadata to the next rewriter");
+
+        query.properties().set(RewriterConstants.REWRITE_META, metadata);
+        log(utilsLogger, query, "Successfully passed metadata to the next rewriter");
+    }
+
+
+    /**
+     * Retrieve spell corrected query with highest score from QLAS
+     *
+     * @param query Query object from the searcher
+     * @param qss_rw Whether to consider qss_rw modification
+     * @param qss_sugg Whether ot consider qss_sugg modification
+     * @return Spell corrected query or null if not found
+     */
+    public static String getSpellCorrected(Query query,
+                                           boolean qss_rw,
+                                           boolean qss_sugg)
+                                           throws RuntimeException {
+        log(utilsLogger, query, "Retrieving spell corrected query");
+
+        // Retrieve Intent Model
+        IntentModel intentModel = IntentModel.getFrom(query);
+        if(intentModel==null) {
+            error(utilsLogger, query, "Unable to retrieve intent model");
+            throw new RuntimeException("Not able to retrieve intent model");
+        }
+
+        double max_score = 0;
+        String spellCorrected = null;
+
+        // Iterate through all interpretations to get a spell corrected
+        // query with highest score
+        for (InterpretationNode interpretationNode : intentModel.children()) {
+            Modification modification = interpretationNode.getInterpretation()
+                                                          .getModification();
+            Annotations annotations = modification.getAnnotation();
+            Double score = annotations.getDouble("score");
+
+            // Check if it's higher than the max score
+            if(score!=null && score>max_score) {
+                Boolean isQSSRewrite = annotations.getBoolean("qss_rw");
+                Boolean isQSSSuggest = annotations.getBoolean("qss_sugg");
+
+                // Check if it's qss_rw or qss_sugg
+                if((qss_rw && isQSSRewrite!=null && isQSSRewrite) ||
+                   (qss_sugg && isQSSSuggest!=null && isQSSSuggest)) {
+                    max_score = score;
+                    spellCorrected = modification.getText();
+                }
+            }
+        }
+
+        if(spellCorrected!=null) {
+            log(utilsLogger, query, "Successfully retrieved spell corrected query: " +
+                spellCorrected);
+        } else {
+            log(utilsLogger, query, "No spell corrected query is retrieved");
+        }
+
+        return spellCorrected;
+    }
+
+    /**
+     * Retrieve normalized original query from query object
+     *
+     * @param query Query object from searcher
+     * @return normalized query
+     */
+    public static String getNormalizedOriginalQuery(Query query) {
+        return toLowerCase(query.getModel().getQueryString()).trim();
+    }
+
+    /**
+     * Log message
+     *
+     * @param logger Logger used for this msg
+     * @param msg Log message
+     */
+    public static void log(Logger logger, String msg) {
+        logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg);
+    }
+
+    /**
+     * Log message
+     *
+     * @param logger Logger used for this msg
+     * @param query Query object from searcher
+     * @param msg Log message
+     */
+    public static void log(Logger logger, Query query, String msg) {
+        if(query!=null) {
+            query.trace(logger.getName() + ": " + msg, true, TRACELEVEL);
+        }
+        logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg);
+    }
+
+    /**
+     * Print error message
+     *
+     * @param logger Logger used for this msg
+     * @param msg Error message
+     */
+    public static void error(Logger logger, String msg) {
+        logger.severe(logger.getName() + ": " + msg);
+    }
+
+    /**
+     * Print error message
+     *
+     * @param logger Logger used for this msg
+     * @param query Query object from searcher
+     * @param msg Error message
+     */
+    public static void error(Logger logger, Query query, String msg) {
+        if(query!=null) {
+            query.trace(logger.getName() + ": " + msg, true, TRACELEVEL);
+        }
+        logger.severe(logger.getName() + ": " + msg);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/SearchChainDispatcherSearcher.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/SearchChainDispatcherSearcher.java
new file mode 100644
index 00000000000..589696c4e77
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/SearchChainDispatcherSearcher.java
@@ -0,0 +1,74 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.search.*;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.component.ComponentId;
+
+import java.util.logging.Logger;
+
+/**
+ * Execute rewriter search chain specified by the user.
+ * It's inteneded to be used for executing rewriter search chains
+ * for different markets.
+ *
+ * @author Karen Sze Wing Lee
+ */
+@Provides("SearchChainDispatcher")
+@After("QLAS")
+public class SearchChainDispatcherSearcher extends Searcher {
+
+    protected final Logger logger = Logger.getLogger(SearchChainDispatcherSearcher.class.getName());
+
+    /**
+     * Constructor for this searcher
+     * @param id Component ID (see vespa's search container doc for more detail)
+     */
+    public SearchChainDispatcherSearcher(ComponentId id) {
+        super(id);
+    }
+
+    /**
+     * Constructor for unit test
+     */
+    public SearchChainDispatcherSearcher() {
+    }
+
+    /**
+     * Execute another search chain specified by the user<br>
+     * - Retrieve search chain specified by the user through
+     *   param<br>
+     * - Execute specified search chain if exist
+     */
+    public @Override Result search(Query query, Execution execution) {
+        RewriterUtils.log(logger, query, "Entering SearchChainDispatcherSearcher");
+
+        // Retrieve search chain specified by user through REWRITER_CHAIN
+        String rewriterChain = RewriterUtils.getRewriterChain(query);
+
+        // Skipping to next searcher if no rewriter chain is specified
+        if(rewriterChain==null || rewriterChain.equals("")) {
+            RewriterUtils.log(logger, query, "No rewriter chain is specified, " +
+                              "skipping to the next searcher");
+            return execution.search(query);
+        }
+
+        // Execute rewriter search chain
+        RewriterUtils.log(logger, query, "Redirecting to chain " + rewriterChain);
+        Chain<Searcher> myChain = execution.searchChainRegistry().getChain(rewriterChain);
+        if(myChain==null) {
+            RewriterUtils.log(logger, query, "Invalid search chain specified, " +
+                              "skipping to the next searcher");
+            return execution.search(query);
+        }
+        new Execution(myChain, execution.context()).search(query);
+        RewriterUtils.log(logger, query, "Finish executing search chain " + rewriterChain);
+
+        // Continue down the chain ignoring the result from REWRITER_CHAIN
+        // since the rewriters only modify the query but not the result
+        return execution.search(query);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/package-info.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/package-info.java
new file mode 100644
index 00000000000..c435ed45623
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.rewrite;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java
new file mode 100644
index 00000000000..3d57675c4ab
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/GenericExpansionRewriter.java
@@ -0,0 +1,213 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite.rewriters;
+
+import java.io.*;
+import java.util.*;
+import java.util.logging.Logger;
+
+import com.google.inject.Inject;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.fsa.FSA;
+import com.yahoo.search.query.rewrite.*;
+import com.yahoo.search.*;
+import com.yahoo.component.ComponentId;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.search.query.rewrite.RewritesConfig;
+import com.yahoo.prelude.querytransform.PhraseMatcher;
+
+/**
+ * This rewriter would add rewrites to entities (e.g abbreviation, synonym, etc)<br>
+ * to boost precision
+ * - FSA dict: [normalized original query]\t[rewrite 1]\t[rewrite 2]\t[etc]<br>
+ * - Features:<br>
+ *   RewritesAsUnitEquiv flag: add proximity boosted rewrites<br>
+ *   PartialPhraseMatch flag: whether to match whole phrase or partial phrase<br>
+ *   MaxRewrites flag: the maximum number of rewrites to be added<br>
+ *
+ * @author Karen Sze Wing Lee
+ */
+@Provides("GenericExpansionRewriter")
+public class GenericExpansionRewriter extends QueryRewriteSearcher {
+
+    // Flag for skipping this rewriter if the query has been rewritten
+    private final boolean SKIP_REWRITER_IF_REWRITTEN = false;
+
+    // Name of the rewriter
+    public static final String REWRITER_NAME = "GenericExpansionRewriter";
+
+    // Generic expansion dictionary name
+    public static final String GENERIC_EXPAND_DICT = "GenericExpansion";
+
+    // Default generic expansion dictionary file name
+    public static final String GENERIC_EXPAND_DICT_FILENAME = "GenericExpansionRewriter.fsa";
+
+    // PhraseMatcher created from FSA dict
+    private PhraseMatcher phraseMatcher;
+
+    private Logger logger;
+
+
+    /**
+     * Constructor for GenericExpansionRewriter.
+     * Load configs using default format
+     */
+    @Inject
+    public GenericExpansionRewriter(ComponentId id,
+                        FileAcquirer fileAcquirer,
+                        RewritesConfig config) {
+        super(id, fileAcquirer, config);
+    }
+
+    /**
+     * Constructor for GenericExpansionRewriter unit test.
+     * Load configs using default format
+     */
+    public GenericExpansionRewriter(RewritesConfig config,
+                        HashMap<String, File> fileList) {
+        super(config, fileList);
+    }
+
+    /**
+     * Instance creation time config loading besides FSA.
+     * Create PhraseMatcher from FSA dict
+     */
+    public boolean configure(FileAcquirer fileAcquirer,
+                             RewritesConfig config,
+                             HashMap<String, File> fileList) {
+        logger = Logger.getLogger(GenericExpansionRewriter.class.getName());
+        FSA fsa = (FSA)rewriterDicts.get(GENERIC_EXPAND_DICT);
+        if(fsa==null) {
+            RewriterUtils.error(logger, "Error retrieving FSA dictionary: " +
+                                              GENERIC_EXPAND_DICT);
+            return false;
+        }
+        // Create Phrase Matcher
+        RewriterUtils.log(logger, "Creating PhraseMatcher");
+        try {
+            phraseMatcher = new PhraseMatcher(fsa, false);
+        } catch (IllegalArgumentException e) {
+            RewriterUtils.error(logger, "Error creating phrase matcher");
+            return false;
+        }
+
+        // Match single word as well
+        phraseMatcher.setMatchSingleItems(true);
+
+        // Return all matches instead of only the longest match
+        phraseMatcher.setMatchAll(true);
+
+        return true;
+    }
+
+    /**
+     * Main logic of rewriter<br>
+     * - Retrieve rewrites from FSA dict<br>
+     * - rewrite query using features that are enabled by user
+     */
+    public HashMap<String, Object> rewrite(Query query,
+                                           String dictKey) throws RuntimeException {
+
+        Boolean rewritten = false;
+
+        // Pass the original dict key to the next rewriter
+        HashMap<String, Object> result = new HashMap<>();
+        result.put(RewriterConstants.REWRITTEN, rewritten);
+        result.put(RewriterConstants.DICT_KEY, dictKey);
+
+        RewriterUtils.log(logger, query,
+                         "In GenericExpansionRewriter, query used for dict retrieval=[" + dictKey + "]");
+
+        // Retrieve flags for choosing between whole query match
+        // or partial query match
+        String partialPhraseMatch = getQPConfig(query, RewriterConstants.PARTIAL_PHRASE_MATCH);
+
+        if(partialPhraseMatch==null) {
+            RewriterUtils.error(logger, query, "Required param " + RewriterConstants.PARTIAL_PHRASE_MATCH +
+                                               " is not set, skipping rewriter");
+            throw new RuntimeException("Required param " + RewriterConstants.PARTIAL_PHRASE_MATCH +
+                                       " is not set, skipping rewriter");
+        }
+
+        // Retrieve max number of rewrites allowed
+        int maxNumRewrites = 0;
+        String maxNumRewritesStr = getQPConfig(query, RewriterConstants.MAX_REWRITES);
+        if(maxNumRewritesStr!=null) {
+            maxNumRewrites = Integer.parseInt(maxNumRewritesStr);
+            RewriterUtils.log(logger, query,
+                              "Limiting max number of rewrites to: " + maxNumRewrites);
+        } else {
+            RewriterUtils.log(logger, query, "No limit on number of rewrites");
+        }
+
+        // Retrieve flags for choosing whether to add
+        // the rewrites as phrase, default to false
+        String rewritesAsUnitEquiv = getQPConfig(query, RewriterConstants.REWRITES_AS_UNIT_EQUIV);
+        if(rewritesAsUnitEquiv==null) {
+            rewritesAsUnitEquiv = "false";
+        }
+
+        Set<PhraseMatcher.Phrase> matches;
+
+        // Partial Phrase Matching
+        if(partialPhraseMatch.equalsIgnoreCase("true")) {
+            RewriterUtils.log(logger, query, "Partial phrase matching");
+
+            // Retrieve longest non overlapping matches
+            matches = RewriterFeatures.getNonOverlappingPartialPhraseMatches(phraseMatcher, query);
+
+        // Full Phrase Matching if set to anything else
+        } else {
+            RewriterUtils.log(logger, query, "Full phrase matching");
+
+            // Retrieve longest non overlapping matches
+            matches = RewriterFeatures.getNonOverlappingFullPhraseMatches(phraseMatcher, query);
+        }
+
+        if(matches==null) {
+            return result;
+        }
+
+        // Add expansions to the query
+        query = RewriterFeatures.addExpansions(query, matches, null, maxNumRewrites, false,
+                                               rewritesAsUnitEquiv.equalsIgnoreCase("true"));
+
+        rewritten = true;
+
+        RewriterUtils.log(logger, query, "GenericExpansionRewriter final query: " + query.toDetailString());
+
+        result.put(RewriterConstants.REWRITTEN, rewritten);
+
+        return result;
+    }
+
+    /**
+     * Get the flag which specifies whether this rewriter
+     * should be skipped if the query has been rewritten
+     *
+     * @return true if rewriter should be skipped, false
+     *         otherwise
+     */
+    public boolean getSkipRewriterIfRewritten() {
+        return SKIP_REWRITER_IF_REWRITTEN;
+    }
+
+   /**
+    * Get the name of the rewriter
+    *
+    * @return Name of the rewriter
+    */
+   public String getRewriterName() {
+       return REWRITER_NAME;
+   }
+
+   /**
+    * Get default FSA dictionary names
+    *
+    * @return Pair of FSA dictionary name and filename
+    */
+   public HashMap<String, String> getDefaultFSAs() {
+       HashMap<String, String> defaultDicts = new HashMap<>();
+       defaultDicts.put(GENERIC_EXPAND_DICT, GENERIC_EXPAND_DICT_FILENAME);
+       return defaultDicts;
+   }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/MisspellRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/MisspellRewriter.java
new file mode 100644
index 00000000000..a1b46926cbd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/MisspellRewriter.java
@@ -0,0 +1,151 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite.rewriters;
+
+import java.io.*;
+import java.util.*;
+import java.util.logging.Logger;
+
+import com.google.inject.Inject;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.search.query.rewrite.*;
+import com.yahoo.search.*;
+import com.yahoo.component.ComponentId;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.search.query.rewrite.RewritesConfig;
+
+/**
+ * This rewriter would retrieve spell corrected query from QLAS and
+ * add it to the original query tree as equiv<br>
+ * - Features:<br>
+ *   RewritesAsEquiv flag: add rewrites to original query as equiv
+ *
+ * @author Karen Sze Wing Lee
+ */
+@After("QLAS")
+@Provides("MisspellRewriter")
+public class MisspellRewriter extends QueryRewriteSearcher {
+
+    // Flag for skipping this rewriter if the query has been rewritten
+    private final boolean SKIP_REWRITER_IF_REWRITTEN = false;
+
+    // Name of the rewriter
+    public static final String REWRITER_NAME = "MisspellRewriter";
+
+    private Logger logger = Logger.getLogger(MisspellRewriter.class.getName());
+
+    /**
+     * Constructor for MisspellRewriter
+     */
+    @Inject
+    public MisspellRewriter(ComponentId id) {
+        super(id);
+    }
+
+    /**
+     * Constructor for MisspellRewriter unit test
+     */
+    public MisspellRewriter() {
+        super();
+    }
+
+    /**
+     * Instance creation time config loading besides FSA.
+     * Empty for this rewriter
+     */
+    public boolean configure(FileAcquirer fileAcquirer,
+                             RewritesConfig config,
+                             HashMap<String, File> fileList) {
+        return true;
+    }
+
+    /**
+     * Main logic of rewriter<br>
+     * - Retrieve spell corrected query from QLAS<br>
+     * - Add spell corrected query as equiv
+     */
+    public HashMap<String, Object> rewrite(Query query,
+                                           String dictKey) throws RuntimeException {
+
+        Boolean rewritten = false;
+
+        HashMap<String, Object> result = new HashMap<>();
+        result.put(RewriterConstants.REWRITTEN, rewritten);
+        result.put(RewriterConstants.DICT_KEY, dictKey);
+
+        RewriterUtils.log(logger, query,
+                         "In MisspellRewriter");
+
+        // Retrieve flags for enabling the features
+        String qssRw = getQPConfig(query, RewriterConstants.QSS_RW);
+        String qssSugg = getQPConfig(query, RewriterConstants.QSS_SUGG);
+
+        boolean isQSSRw = false;
+        boolean isQSSSugg = false;
+
+        if(qssRw!=null) {
+            isQSSRw = qssRw.equalsIgnoreCase("true");
+        }
+        if(qssSugg!=null) {
+            isQSSSugg = qssSugg.equalsIgnoreCase("true");
+        }
+
+        // Rewrite is not enabled
+        if(!isQSSRw && !isQSSSugg) {
+            return result;
+        }
+
+        // Retrieve spell corrected query from QLAS
+        String rewrites = RewriterUtils.getSpellCorrected(query, isQSSRw, isQSSSugg);
+
+        // No rewrites
+        if(rewrites==null) {
+            RewriterUtils.log(logger, query, "No rewrite is retrieved");
+            return result;
+        } else {
+            RewriterUtils.log(logger, query, "Retrieved spell corrected query: " +
+                              rewrites);
+        }
+
+        // Adding rewrite to the query tree
+        query = RewriterFeatures.addRewritesAsEquiv(query, dictKey, rewrites, false, 0);
+
+        rewritten = true;
+        RewriterUtils.log(logger, query, "MisspellRewriter final query: " +
+                          query.toDetailString());
+
+        result.put(RewriterConstants.REWRITTEN, rewritten);
+        result.put(RewriterConstants.DICT_KEY, rewrites);
+
+        return result;
+    }
+
+    /**
+     * Get the flag which specifies whether this rewriter
+     * should be skipped if the query has been rewritten
+     *
+     * @return true if rewriter should be skipped, false
+     *         otherwise
+     */
+    public boolean getSkipRewriterIfRewritten() {
+        return SKIP_REWRITER_IF_REWRITTEN;
+    }
+
+   /**
+    * Get the name of the rewriter
+    *
+    * @return Name of the rewriter
+    */
+   public String getRewriterName() {
+       return REWRITER_NAME;
+   }
+
+   /**
+    * Get default FSA dictionary names
+    *
+    * @return Pair of FSA dictionary name and filename
+    */
+   public HashMap<String, String> getDefaultFSAs() {
+       return null;
+   }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/NameRewriter.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/NameRewriter.java
new file mode 100644
index 00000000000..5ecf7893c63
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/NameRewriter.java
@@ -0,0 +1,194 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.rewrite.rewriters;
+
+import java.io.*;
+import java.util.*;
+import java.util.logging.Logger;
+
+import com.google.inject.Inject;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.search.query.rewrite.*;
+import com.yahoo.search.*;
+import com.yahoo.component.ComponentId;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.search.query.rewrite.RewritesConfig;
+
+/**
+ * This rewriter would add rewrites to name entities to boost precision<br>
+ * - FSA dict: [normalized original query]\t[rewrite 1]\t[rewrite 2]\t[etc]<br>
+ * - Features:<br>
+ *   OriginalAsUnit flag: add proximity boosting to original query<br>
+ *   RewritesAsUnitEquiv flag: add proximity boosted rewrites to original query<br>
+ *   RewritesAsEquiv flag: add rewrites to original query<br>
+ *
+ * @author Karen Sze Wing Lee
+ */
+@Provides("NameRewriter")
+public class NameRewriter extends QueryRewriteSearcher {
+
+    // Flag for skipping this rewriter if the query has been rewritten
+    private final boolean SKIP_REWRITER_IF_REWRITTEN = false;
+
+    // Name of the rewriter
+    public static final String REWRITER_NAME = "NameRewriter";
+
+    // Name entity expansion dictionary name
+    public static final String NAME_ENTITY_EXPAND_DICT = "NameEntityExpansion";
+
+    // Default Name entity expansion dictionary file name
+    public static final String NAME_ENTITY_EXPAND_DICT_FILENAME = "NameRewriter.fsa";
+
+    private Logger logger;
+
+    /**
+     * Constructor for NameRewriter<br>
+     * Load configs using default format
+     */
+    @Inject
+    public NameRewriter(ComponentId id,
+                        FileAcquirer fileAcquirer,
+                        RewritesConfig config) {
+        super(id, fileAcquirer, config);
+    }
+
+    /**
+     * Constructor for NameRewriter unit test<br>
+     * Load configs using default format
+     */
+    public NameRewriter(RewritesConfig config,
+                        HashMap<String, File> fileList) {
+        super(config, fileList);
+    }
+
+    /**
+     * Instance creation time config loading besides FSA<br>
+     * Empty for this rewriter
+     */
+    public boolean configure(FileAcquirer fileAcquirer,
+                             RewritesConfig config,
+                             HashMap<String, File> fileList) {
+        logger = Logger.getLogger(NameRewriter.class.getName());
+        return true;
+    }
+
+    /**
+     * Main logic of rewriter<br>
+     * - Retrieve rewrites from FSA dict<br>
+     * - rewrite query using features that are enabled by user
+     */
+    public HashMap<String, Object> rewrite(Query query,
+                                           String dictKey) throws RuntimeException {
+
+        Boolean rewritten = false;
+
+        // Pass the original dict key to the next rewriter
+        HashMap<String, Object> result = new HashMap<>();
+        result.put(RewriterConstants.REWRITTEN, rewritten);
+        result.put(RewriterConstants.DICT_KEY, dictKey);
+
+        RewriterUtils.log(logger, query,
+                         "In NameRewriter, query used for dict retrieval=[" + dictKey + "]");
+
+        // Retrieve rewrite from FSA dict using normalized query
+        String rewrites = super.getRewriteFromFSA(query, NAME_ENTITY_EXPAND_DICT, dictKey);
+        RewriterUtils.log(logger, query, "Retrieved rewrites: " + rewrites);
+
+        // No rewrites
+        if(rewrites==null) {
+            RewriterUtils.log(logger, query, "No rewrite is retrieved");
+            return result;
+        }
+
+        // Retrieve max number of rewrites allowed
+        int maxNumRewrites = 0;
+        String maxNumRewritesStr = getQPConfig(query, RewriterConstants.MAX_REWRITES);
+        if(maxNumRewritesStr!=null) {
+            maxNumRewrites = Integer.parseInt(maxNumRewritesStr);
+            RewriterUtils.log(logger, query,
+                              "Limiting max number of rewrites to: " + maxNumRewrites);
+        } else {
+            RewriterUtils.log(logger, query, "No limit on number of rewrites");
+        }
+
+        // Retrieve flags for enabling the features
+        String originalAsUnit = getQPConfig(query, RewriterConstants.ORIGINAL_AS_UNIT);
+        String originalAsUnitEquiv = getQPConfig(query, RewriterConstants.ORIGINAL_AS_UNIT_EQUIV);
+        String rewritesAsUnitEquiv = getQPConfig(query, RewriterConstants.REWRITES_AS_UNIT_EQUIV);
+        String rewritesAsEquiv = getQPConfig(query, RewriterConstants.REWRITES_AS_EQUIV);
+
+        // Add proximity boosting to original query and keeping
+        // the original query if it's enabled
+        if(originalAsUnitEquiv!=null && originalAsUnitEquiv.equalsIgnoreCase("true")) {
+            RewriterUtils.log(logger, query, "OriginalAsUnitEquiv is enabled");
+            query = RewriterFeatures.addUnitToOriginalQuery(query, dictKey, true);
+            RewriterUtils.log(logger, query,
+                              "Query after OriginalAsUnitEquiv: " + query.toDetailString());
+            rewritten = true;
+
+        // Add proximity boosting to original query
+        // if it's enabled
+        } else if(originalAsUnit!=null && originalAsUnit.equalsIgnoreCase("true")) {
+            RewriterUtils.log(logger, query, "OriginalAsUnit is enabled");
+            query = RewriterFeatures.addUnitToOriginalQuery(query, dictKey, false);
+            RewriterUtils.log(logger, query,
+                              "Query after OriginalAsUnit: " + query.toDetailString());
+            rewritten = true;
+        }
+
+        // Add rewrites as unit equiv if it's enabled
+        if(rewritesAsUnitEquiv!=null && rewritesAsUnitEquiv.equalsIgnoreCase("true")) {
+            RewriterUtils.log(logger, query, "RewritesAsUnitEquiv is enabled");
+            //query = RewriterFeatures.addRewritesAsEquiv(query, dictKey, rewrites, true, maxNumRewrites);
+            query = RewriterFeatures.addRewritesAsEquiv(query, dictKey, rewrites, true, maxNumRewrites);
+            RewriterUtils.log(logger, query,
+                              "Query after RewritesAsUnitEquiv: " + query.toDetailString());
+            rewritten = true;
+
+        // Add rewrites as equiv if it's enabled
+        } else if(rewritesAsEquiv!=null && rewritesAsEquiv.equalsIgnoreCase("true")) {
+            RewriterUtils.log(logger, query, "RewritesAsEquiv is enabled");
+            //query = RewriterFeatures.addRewritesAsEquiv(query, dictKey, rewrites, false, maxNumRewrites);
+            query = RewriterFeatures.addRewritesAsEquiv(query, dictKey, rewrites, false, maxNumRewrites);
+            RewriterUtils.log(logger, query,
+                              "Query after RewritesAsEquiv: " + query.toDetailString());
+            rewritten = true;
+        }
+
+        RewriterUtils.log(logger, query, "NameRewriter final query: " + query.toDetailString());
+
+        result.put(RewriterConstants.REWRITTEN, rewritten);
+
+        return result;
+    }
+
+    /**
+     * Get the flag which specifies whether this rewriter.
+     * should be skipped if the query has been rewritten
+     *
+     * @return true if rewriter should be skipped, false
+     *         otherwise
+     */
+    public boolean getSkipRewriterIfRewritten() {
+        return SKIP_REWRITER_IF_REWRITTEN;
+    }
+
+   /**
+    * Get the name of the rewriter
+    *
+    * @return Name of the rewriter
+    */
+   public String getRewriterName() {
+       return REWRITER_NAME;
+   }
+
+   /**
+    * Get default FSA dictionary names
+    *
+    * @return Pair of FSA dictionary name and filename
+    */
+   public HashMap<String, String> getDefaultFSAs() {
+       HashMap<String, String> defaultDicts = new HashMap<>();
+       defaultDicts.put(NAME_ENTITY_EXPAND_DICT, NAME_ENTITY_EXPAND_DICT_FILENAME);
+       return defaultDicts;
+   }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/package-info.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/package-info.java
new file mode 100644
index 00000000000..bfbb73f661e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/rewriters/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.rewrite.rewriters;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/TextSerialize.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/TextSerialize.java
new file mode 100644
index 00000000000..bac9f2af237
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/TextSerialize.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.item.ItemContext;
+import com.yahoo.search.query.textserialize.item.ItemFormHandler;
+import com.yahoo.search.query.textserialize.parser.ParseException;
+import com.yahoo.search.query.textserialize.parser.Parser;
+import com.yahoo.search.query.textserialize.parser.TokenMgrError;
+import com.yahoo.search.query.textserialize.serializer.QueryTreeSerializer;
+
+import java.io.StringReader;
+
+/**
+ * @author tonytv
+ * Facade
+ * Allows serializing/deserializing  a query to the programmatic format.
+ */
+public class TextSerialize {
+    public static Item parse(String serializedQuery) {
+        try {
+            ItemContext context = new ItemContext();
+            Object result = new Parser(new StringReader(serializedQuery.replace("'", "\"")), new ItemFormHandler(), context).start();
+            context.connectItems();
+
+            if (!(result instanceof Item)) {
+                throw new RuntimeException("The serialized query '" + serializedQuery + "' did not evaluate to an Item" +
+                        "(type = " + result.getClass() + ")");
+            }
+            return (Item) result;
+        } catch (ParseException e) {
+            throw new RuntimeException(e);
+        } catch (TokenMgrError e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static String serialize(Item item) {
+        return new QueryTreeSerializer().serialize(item);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java
new file mode 100644
index 00000000000..c4e54ca748d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java
@@ -0,0 +1,54 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NotItem;
+
+import java.util.List;
+
+import static com.yahoo.search.query.textserialize.item.ListUtil.butFirst;
+import static com.yahoo.search.query.textserialize.item.ListUtil.first;
+
+/**
+ * @author tonytv
+ */
+public class AndNotRestConverter extends CompositeConverter<NotItem> {
+    static final String andNotRest = "AND-NOT-REST";
+
+    public AndNotRestConverter() {
+        super(NotItem.class);
+    }
+
+    @Override
+    protected void addChildren(NotItem item, ItemArguments arguments, ItemContext context) {
+        if (firstIsNull(arguments.children)) {
+            addNegativeItems(item, arguments.children);
+        } else {
+            addItems(item, arguments.children);
+        }
+    }
+
+    private void addNegativeItems(NotItem notItem, List<Object> children) {
+        for (Object child: butFirst(children)) {
+            TypeCheck.ensureInstanceOf(child, Item.class);
+            notItem.addNegativeItem((Item) child);
+        }
+    }
+
+    private void addItems(NotItem notItem, List<Object> children) {
+        for (Object child : children) {
+            TypeCheck.ensureInstanceOf(child, Item.class);
+            notItem.addItem((Item) child);
+        }
+    }
+
+
+    private boolean firstIsNull(List<Object> children) {
+        return !children.isEmpty() && first(children) == null;
+    }
+
+    @Override
+    protected String getFormName(Item item) {
+        return andNotRest;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/CompositeConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/CompositeConverter.java
new file mode 100644
index 00000000000..7f7c5e48d0a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/CompositeConverter.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+import java.util.ListIterator;
+
+/**
+ * @author tonytv
+ */
+public class CompositeConverter<T extends CompositeItem> implements ItemFormConverter {
+    private final Class<T> itemClass;
+
+    public CompositeConverter(Class<T> itemClass) {
+        this.itemClass = itemClass;
+    }
+
+    @Override
+    public Object formToItem(String name, ItemArguments arguments, ItemContext itemContext) {
+        T item = newInstance();
+        addChildren(item, arguments, itemContext);
+        return item;
+    }
+
+    protected void addChildren(T item, ItemArguments arguments, ItemContext itemContext) {
+        for (Object child : arguments.children) {
+            item.addItem(asItem(child));
+        }
+        ItemInitializer.initialize(item, arguments, itemContext);
+    }
+
+    private static Item asItem(Object child) {
+        if (!(child instanceof Item) && child != null) {
+            throw new RuntimeException("Expected query item, but got '" + child.toString() +
+                    "' [" + child.getClass().getName() + "]");
+        }
+        return (Item) child;
+    }
+
+    private T newInstance() {
+        try {
+            return itemClass.newInstance();
+        } catch (InstantiationException | IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper) {
+        CompositeItem compositeItem = (CompositeItem) item;
+
+        DispatchForm form = new DispatchForm(getFormName(item));
+        for (ListIterator<Item> i = compositeItem.getItemIterator(); i.hasNext() ;) {
+            form.addChild(i.next());
+        }
+        ItemInitializer.initializeForm(form, item, itemIdMapper);
+        return form;
+    }
+
+    protected String getFormName(Item item) {
+        return item.getItemType().name();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ExactStringConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ExactStringConverter.java
new file mode 100644
index 00000000000..4b68ecfe5a9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ExactStringConverter.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.ExactstringItem;
+
+/**
+ * @author balder
+ */
+// TODO: balder to fix javadoc
+public class ExactStringConverter extends WordConverter {
+    @Override
+    ExactstringItem newTermItem(String word) {
+        return new ExactstringItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/IntConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/IntConverter.java
new file mode 100644
index 00000000000..43b96d17773
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/IntConverter.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.TermItem;
+
+/**
+ * @author tonytv
+ */
+public class IntConverter extends TermConverter {
+    @Override
+    IntItem newTermItem(String word) {
+        return new IntItem(word);
+    }
+
+    @Override
+    protected String getValue(TermItem item) {
+        return ((IntItem)item).getNumber();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java
new file mode 100644
index 00000000000..50cc9c42773
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemArguments.java
@@ -0,0 +1,26 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static com.yahoo.search.query.textserialize.item.ListUtil.firstInstanceOf;
+
+/**
+ * @author tonytv
+ */
+public class ItemArguments {
+    public final Map<?, ?> properties;
+    public final List<Object> children;
+
+    public ItemArguments(List<Object> arguments) {
+        if (firstInstanceOf(arguments, Map.class)) {
+            properties = (Map<?, ?>) ListUtil.first(arguments);
+            children = ListUtil.rest(arguments);
+        } else {
+            properties = Collections.emptyMap();
+            children = arguments;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java
new file mode 100644
index 00000000000..fd21b4e02e1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemContext.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.TaggableItem;
+
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemContext {
+    private class Connectivity {
+        final String id;
+        final double strength;
+
+        public Connectivity(String id, double strength) {
+            this.id = id;
+            this.strength = strength;
+        }
+    }
+
+    private final Map<String, Item> itemById = new HashMap<>();
+    private final Map<TaggableItem, Connectivity> connectivityByItem = new IdentityHashMap<>();
+
+
+    public void setItemId(String id, Item item) {
+        itemById.put(id, item);
+    }
+
+    public void setConnectivity(TaggableItem item, String id, Double strength) {
+        connectivityByItem.put(item, new Connectivity(id, strength));
+    }
+
+    public void connectItems() {
+        for (Map.Entry<TaggableItem, Connectivity> entry : connectivityByItem.entrySet()) {
+            entry.getKey().setConnectivity(getItem(entry.getValue().id), entry.getValue().strength);
+        }
+    }
+
+    private Item getItem(String id) {
+        Item item = itemById.get(id);
+        if (item == null)
+            throw new IllegalArgumentException("No item with id '" + id + "'.");
+        return item;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemExecutorRegistry.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemExecutorRegistry.java
new file mode 100644
index 00000000000..20ef9f4e5cc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemExecutorRegistry.java
@@ -0,0 +1,71 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.prelude.query.ONearItem;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.RankItem;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemExecutorRegistry {
+
+    private static final Map<String, ItemFormConverter> executorsByName = new HashMap<>();
+    static {
+        register(Item.ItemType.AND, createCompositeConverter(AndItem.class));
+        register(Item.ItemType.OR, createCompositeConverter(OrItem.class));
+        register(Item.ItemType.RANK, createCompositeConverter(RankItem.class));
+        register(Item.ItemType.PHRASE, createCompositeConverter(PhraseItem.class));
+        register(Item.ItemType.EQUIV, createCompositeConverter(EquivItem.class));
+
+        register(AndNotRestConverter.andNotRest, new AndNotRestConverter());
+
+        register(Item.ItemType.NEAR, new NearConverter(NearItem.class));
+        register(Item.ItemType.ONEAR, new NearConverter(ONearItem.class));
+
+        register(Item.ItemType.WORD, new WordConverter());
+        register(Item.ItemType.INT, new IntConverter());
+        register(Item.ItemType.PREFIX, new PrefixConverter());
+        register(Item.ItemType.SUBSTRING, new SubStringConverter());
+        register(Item.ItemType.EXACT, new ExactStringConverter());
+        register(Item.ItemType.SUFFIX, new SuffixConverter());
+    }
+
+    private static <T extends CompositeItem> ItemFormConverter createCompositeConverter(Class<T> itemClass) {
+        return new CompositeConverter<>(itemClass);
+    }
+
+    private static void register(Item.ItemType type, ItemFormConverter executor) {
+        register(type.toString(), executor);
+    }
+
+    private static void register(String type, ItemFormConverter executor) {
+        executorsByName.put(type, executor);
+    }
+
+    public static ItemFormConverter getByName(String name) {
+        ItemFormConverter executor = executorsByName.get(name);
+        ensureNotNull(executor, name);
+        return executor;
+    }
+
+    private static void ensureNotNull(ItemFormConverter executor, String name) {
+        if (executor == null) {
+            throw new RuntimeException("No item type named '" + name + "'.");
+        }
+    }
+
+    public static ItemFormConverter getByType(Item.ItemType itemType) {
+        String name = (itemType == Item.ItemType.NOT) ? AndNotRestConverter.andNotRest : itemType.name();
+        return getByName(name);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormConverter.java
new file mode 100644
index 00000000000..256ad569686
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+/**
+ * @author tonytv
+ */
+public interface ItemFormConverter {
+    Object formToItem(String name, ItemArguments arguments, ItemContext context);
+    DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormHandler.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormHandler.java
new file mode 100644
index 00000000000..81b13a107c8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemFormHandler.java
@@ -0,0 +1,17 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.search.query.textserialize.parser.DispatchFormHandler;
+
+import java.util.List;
+
+/**
+ * @author tonytv
+ */
+public class ItemFormHandler implements DispatchFormHandler{
+    @Override
+    public Object dispatch(String name, List<Object> arguments, Object dispatchContext) {
+        ItemFormConverter executor = ItemExecutorRegistry.getByName(name);
+        return executor.formToItem(name, new ItemArguments(arguments), (ItemContext)dispatchContext);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java
new file mode 100644
index 00000000000..ae54165abef
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ItemInitializer.java
@@ -0,0 +1,137 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.IndexedItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemInitializer {
+    private static final String indexProperty = "index";
+    private static final String idProperty = "id";
+    private static final String significanceProperty = "significance";
+    private static final String uniqueIdProperty = "uniqueId";
+    private static final String weightProperty = "weight";
+
+    public static void initialize(Item item, ItemArguments arguments, ItemContext itemContext) {
+        storeIdInContext(item, arguments.properties, itemContext);
+
+        Object weight = arguments.properties.get(weightProperty);
+        if (weight != null) {
+            TypeCheck.ensureInstanceOf(weight, Number.class);
+            item.setWeight(((Number)weight).intValue());
+        }
+
+        if (item instanceof TaggableItem) {
+            initializeTaggableItem((TaggableItem)item, arguments, itemContext);
+        }
+
+        if (item instanceof IndexedItem) {
+            initializeIndexedItem((IndexedItem)item, arguments, itemContext);
+        }
+    }
+
+    private static void storeIdInContext(Item item, Map<?, ?> properties, ItemContext itemContext) {
+        Object id = properties.get("id");
+        if (id != null) {
+            TypeCheck.ensureInstanceOf(id, String.class);
+            itemContext.setItemId((String) id, item);
+        }
+    }
+
+    private static void initializeTaggableItem(TaggableItem item, ItemArguments arguments, ItemContext itemContext) {
+        Object connectivity = arguments.properties.get("connectivity");
+        if (connectivity != null) {
+            storeConnectivityInContext(item, connectivity, itemContext);
+        }
+
+        Object significance = arguments.properties.get(significanceProperty);
+        if (significance != null) {
+            TypeCheck.ensureInstanceOf(significance, Number.class);
+            item.setSignificance(((Number)significance).doubleValue());
+        }
+
+        Object uniqueId = arguments.properties.get(uniqueIdProperty);
+        if (uniqueId != null) {
+            TypeCheck.ensureInstanceOf(uniqueId, Number.class);
+            item.setUniqueID(((Number)uniqueId).intValue());
+        }
+    }
+
+    private static void initializeIndexedItem(IndexedItem indexedItem, ItemArguments arguments, ItemContext itemContext) {
+        Object index = arguments.properties.get(indexProperty);
+        if (index != null) {
+            TypeCheck.ensureInstanceOf(index, String.class);
+            indexedItem.setIndexName((String) index);
+        }
+    }
+
+    private static void storeConnectivityInContext(TaggableItem item, Object connectivity, ItemContext itemContext) {
+        TypeCheck.ensureInstanceOf(connectivity, List.class);
+        List<?> connectivityList = (List<?>) connectivity;
+        if (connectivityList.size() != 2) {
+            throw new IllegalArgumentException("Expected two elements for connectivity, got " + connectivityList.size());
+        }
+
+        Object id = connectivityList.get(0);
+        Object strength = connectivityList.get(1);
+
+        TypeCheck.ensureInstanceOf(id, String.class);
+        TypeCheck.ensureInstanceOf(strength, Number.class);
+
+        itemContext.setConnectivity(item, (String)id, ((Number)strength).doubleValue());
+    }
+
+    public static void initializeForm(DispatchForm form, Item item, ItemIdMapper itemIdMapper) {
+        if (item.getWeight() != Item.DEFAULT_WEIGHT) {
+            form.setProperty(weightProperty, item.getWeight());
+        }
+
+        if (item instanceof IndexedItem) {
+            initializeIndexedForm(form, (IndexedItem) item);
+        }
+        if (item instanceof TaggableItem) {
+            initializeTaggableForm(form, (TaggableItem) item, itemIdMapper);
+        }
+        initializeFormWithIdIfConnected(form, item, itemIdMapper);
+    }
+
+    private static void initializeFormWithIdIfConnected(DispatchForm form, Item item, ItemIdMapper itemIdMapper) {
+        if (item.hasConnectivityBackLink()) {
+            form.setProperty(idProperty, itemIdMapper.getId(item));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void initializeTaggableForm(DispatchForm form, TaggableItem taggableItem, ItemIdMapper itemIdMapper) {
+        Item connectedItem = taggableItem.getConnectedItem();
+        if (connectedItem != null) {
+            form.setProperty("connectivity",
+                    Arrays.asList(itemIdMapper.getId(connectedItem), taggableItem.getConnectivity()));
+        }
+
+        if (taggableItem.hasExplicitSignificance()) {
+            form.setProperty(significanceProperty, taggableItem.getSignificance());
+        }
+
+        if (taggableItem.hasUniqueID()) {
+            form.setProperty(uniqueIdProperty, taggableItem.getUniqueID());
+        }
+    }
+
+    private static void initializeIndexedForm(DispatchForm form, IndexedItem item) {
+        String index = item.getIndexName();
+        if (!index.isEmpty()) {
+            form.setProperty(indexProperty, index);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ListUtil.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ListUtil.java
new file mode 100644
index 00000000000..9349b01a3bc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/ListUtil.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import java.util.*;
+
+/**
+ * @author tonytv
+ */
+public class ListUtil {
+    public static <T> List<T> rest(List<T> list) {
+        return list.subList(1, list.size());
+    }
+
+    public static <T> T first(Collection<T> collection) {
+        return collection.iterator().next();
+    }
+
+    public static boolean firstInstanceOf(Collection<?> collection, @SuppressWarnings("rawtypes") Class c) {
+        return !collection.isEmpty() && c.isInstance(first(collection));
+    }
+
+    public static <T> List<T> butFirst(List<T> list) {
+        return list.subList(1, list.size());
+    }
+
+    public static <T> Iterable<T> butFirst(final Collection<T> collection) {
+        return () -> {
+            Iterator<T> i = collection.iterator();
+            i.next();
+            return i;
+        };
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/NearConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/NearConverter.java
new file mode 100644
index 00000000000..3be8d3d1c65
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/NearConverter.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+/**
+ * @author tonytv
+ */
+@SuppressWarnings("rawtypes")
+public class NearConverter extends CompositeConverter {
+    final private String distanceProperty = "distance";;
+
+    @SuppressWarnings("unchecked")
+    public NearConverter(Class<? extends NearItem> nearItemClass) {
+        super(nearItemClass);
+    }
+
+    @Override
+    public Object formToItem(String name, ItemArguments arguments, ItemContext itemContext) {
+        NearItem nearItem = (NearItem) super.formToItem(name, arguments, itemContext);
+        setDistance(nearItem, arguments);
+        return nearItem;
+    }
+
+    private void setDistance(NearItem nearItem, ItemArguments arguments) {
+        Object distance = arguments.properties.get(distanceProperty);
+        if (distance != null) {
+            TypeCheck.ensureInteger(distance);
+            nearItem.setDistance(((Number)distance).intValue());
+        }
+    }
+
+    @Override
+    public DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper) {
+        DispatchForm dispatchForm = super.itemToForm(item, itemIdMapper);
+
+        NearItem nearItem = (NearItem)item;
+        dispatchForm.setProperty(distanceProperty, nearItem.getDistance());
+        return dispatchForm;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/PrefixConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/PrefixConverter.java
new file mode 100644
index 00000000000..cb3a6c1943c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/PrefixConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.PrefixItem;
+
+/**
+ * @author tonytv
+ */
+public class PrefixConverter extends WordConverter {
+    @Override
+    PrefixItem newTermItem(String word) {
+        return new PrefixItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SubStringConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SubStringConverter.java
new file mode 100644
index 00000000000..e61a189684f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SubStringConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.SubstringItem;
+
+/**
+ * @author tonytv
+ */
+public class SubStringConverter extends WordConverter {
+    @Override
+    SubstringItem newTermItem(String word) {
+        return new SubstringItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SuffixConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SuffixConverter.java
new file mode 100644
index 00000000000..4390e3464d2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/SuffixConverter.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.SuffixItem;
+
+/**
+ * @author tonytv
+ */
+public class SuffixConverter extends WordConverter {
+    @Override
+    SuffixItem newTermItem(String word) {
+        return new SuffixItem(word);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java
new file mode 100644
index 00000000000..8bc6cba7f67
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TermConverter.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.TermItem;
+import com.yahoo.search.query.textserialize.serializer.DispatchForm;
+import com.yahoo.search.query.textserialize.serializer.ItemIdMapper;
+
+/**
+ * @author tonytv
+ */
+public abstract class TermConverter implements ItemFormConverter {
+    @Override
+    public Object formToItem(String name, ItemArguments arguments, ItemContext context) {
+        ensureOnlyOneChild(arguments);
+        String word = getWord(arguments);
+
+        TermItem item = newTermItem(word);
+        ItemInitializer.initialize(item, arguments, context);
+        return item;
+    }
+
+    abstract TermItem newTermItem(String word);
+
+
+    private void ensureOnlyOneChild(ItemArguments arguments) {
+        if (arguments.children.size() != 1) {
+            throw new IllegalArgumentException("Expected exactly one argument, got '" +
+                    arguments.children.toString() + "'");
+        }
+    }
+
+    private String getWord(ItemArguments arguments) {
+        Object word = arguments.children.get(0);
+
+        if (!(word instanceof String)) {
+            throw new RuntimeException("Expected string, got '" + word + "' [" + word.getClass().getName() + "].");
+        }
+        return (String)word;
+    }
+
+    @Override
+    public DispatchForm itemToForm(Item item, ItemIdMapper itemIdMapper) {
+        TermItem termItem = (TermItem)item;
+
+        DispatchForm form = new DispatchForm(termItem.getItemType().name());
+        ItemInitializer.initializeForm(form, item, itemIdMapper);
+        form.addChild(getValue(termItem));
+        return form;
+    }
+
+    protected abstract String getValue(TermItem item);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java
new file mode 100644
index 00000000000..a6e38d288a4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/TypeCheck.java
@@ -0,0 +1,27 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.protect.Validator;
+
+/**
+ * @author tonytv
+ */
+public class TypeCheck {
+    public static void ensureInstanceOf(Object object, Class<?> c) {
+        Validator.ensureInstanceOf(expectationString(c.getName(), object.getClass().getSimpleName()),
+                object, c);
+    }
+
+    public static void ensureInteger(Object value) {
+        ensureInstanceOf(value, Number.class);
+        Number number = (Number)value;
+
+        int intValue = number.intValue();
+        if (intValue != number.doubleValue())
+            throw new IllegalArgumentException("Invalid integer '" + number + "'");
+    }
+
+    private static String expectationString(String expected, String got) {
+        return "Expected " + expected + ", but got " + got;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/WordConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/WordConverter.java
new file mode 100644
index 00000000000..dce33e392ae
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/WordConverter.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.item;
+
+import com.yahoo.prelude.query.TermItem;
+import com.yahoo.prelude.query.WordItem;
+
+/**
+ * @author tonytv
+ */
+public class WordConverter extends TermConverter {
+    @Override
+    WordItem newTermItem(String word) {
+        return new WordItem(word);
+    }
+
+    @Override
+    protected String getValue(TermItem item) {
+        return ((WordItem)item).getWord();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/package-info.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/package-info.java
new file mode 100644
index 00000000000..1e1d3052731
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.query.textserialize;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore
new file mode 100644
index 00000000000..add88bd6807
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/.gitignore
@@ -0,0 +1,7 @@
+/TokenMgrError.java
+/Token.java
+/SimpleCharStream.java
+/ParserTokenManager.java
+/ParserConstants.java
+/ParseException.java
+/Parser.java
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/DispatchFormHandler.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/DispatchFormHandler.java
new file mode 100644
index 00000000000..33c8e36bd57
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/parser/DispatchFormHandler.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.parser;
+
+import java.util.List;
+
+/**
+ * @author tonytv
+ */
+public interface DispatchFormHandler {
+    Object dispatch(String name, List<Object> arguments, Object dispatchContext);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/DispatchForm.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/DispatchForm.java
new file mode 100644
index 00000000000..091efa0a01b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/DispatchForm.java
@@ -0,0 +1,56 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class DispatchForm {
+    private final String name;
+    public final Map<Object, Object> properties = new LinkedHashMap<>();
+    public final List<Object> children = new ArrayList<>();
+
+    public DispatchForm(String name) {
+        this.name = name;
+    }
+
+    public void addChild(Object child) {
+        children.add(child);
+    }
+
+    /**
+     * Only public for the purpose of testing.
+     */
+    public String serialize(ItemIdMapper itemIdMapper) {
+        StringBuilder builder = new StringBuilder();
+        builder.append('(').append(name);
+
+        serializeProperties(builder, itemIdMapper);
+        serializeChildren(builder, itemIdMapper);
+
+        builder.append(')');
+        return builder.toString();
+    }
+
+    private void serializeProperties(StringBuilder builder, ItemIdMapper itemIdMapper) {
+        if (properties.isEmpty())
+            return;
+
+        builder.append(' ').append(Serializer.serializeMap(properties, itemIdMapper));
+    }
+
+
+    private void serializeChildren(StringBuilder builder, ItemIdMapper itemIdMapper) {
+        for (Object child : children) {
+            builder.append(' ').append(Serializer.serialize(child, itemIdMapper));
+        }
+    }
+
+    public void setProperty(Object key, Object value) {
+        properties.put(key, value);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/ItemIdMapper.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/ItemIdMapper.java
new file mode 100644
index 00000000000..c32a7f52c0a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/ItemIdMapper.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import com.yahoo.prelude.query.Item;
+
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+/**
+ * @author tonytv
+ */
+public class ItemIdMapper {
+    private final Map<Item, String> idByItem = new IdentityHashMap<>();
+    private int idCounter = 0;
+
+    public String getId(Item item) {
+        String id = idByItem.get(item);
+        if (id != null) {
+            return id;
+        } else {
+            idByItem.put(item, generateId(item));
+            return getId(item);
+        }
+    }
+
+    private String generateId(Item item) {
+        return item.getName() + "_" + nextCount();
+    }
+
+    private int nextCount() {
+        return idCounter++;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/QueryTreeSerializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/QueryTreeSerializer.java
new file mode 100644
index 00000000000..e3090930369
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/QueryTreeSerializer.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.item.ItemExecutorRegistry;
+
+
+/**
+ * @author tonytv
+ */
+public class QueryTreeSerializer {
+    public String serialize(Item root) {
+        ItemIdMapper itemIdMapper = new ItemIdMapper();
+        return ItemExecutorRegistry.getByType(root.getItemType()).itemToForm(root, itemIdMapper).serialize(itemIdMapper);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java
new file mode 100644
index 00000000000..e8352254551
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/serializer/Serializer.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.textserialize.serializer;
+
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.query.textserialize.item.ItemExecutorRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+import static com.yahoo.search.query.textserialize.item.ListUtil.butFirst;
+import static com.yahoo.search.query.textserialize.item.ListUtil.first;
+
+/**
+ * @author tonytv
+ */
+class Serializer {
+    static String serialize(Object child, ItemIdMapper itemIdMapper) {
+        if (child instanceof DispatchForm) {
+            return ((DispatchForm) child).serialize(itemIdMapper);
+        } else if (child instanceof Item) {
+            return serializeItem((Item) child, itemIdMapper);
+        } else if (child instanceof String) {
+            return serializeString((String) child);
+        } else if (child instanceof Number) {
+            return child.toString();
+        } else if (child instanceof Map) {
+            return serializeMap((Map<?, ?>)child, itemIdMapper);
+        } else if (child instanceof List) {
+            return serializeList((List<?>)child, itemIdMapper);
+        } else {
+            throw new IllegalArgumentException("Can't serialize type " + child.getClass());
+        }
+    }
+
+    private static String serializeString(String string) {
+        return '"' + string.replace("\\", "\\\\").replace("\"", "\\\"") + '"';
+    }
+
+    static String serializeList(List<?> list, ItemIdMapper itemIdMapper) {
+        StringBuilder builder = new StringBuilder();
+        builder.append('[');
+
+        if (!list.isEmpty()) {
+            builder.append(serialize(first(list), itemIdMapper));
+
+            for (Object element : butFirst(list)) {
+                builder.append(", ").append(serialize(element, itemIdMapper));
+            }
+        }
+
+        builder.append(']');
+        return builder.toString();
+    }
+
+    static String serializeMap(Map<?, ?> map, ItemIdMapper itemIdMapper) {
+        StringBuilder builder = new StringBuilder();
+        builder.append("{");
+
+        if (!map.isEmpty()) {
+            serializeEntry(builder, first(map.entrySet()), itemIdMapper);
+            for (Map.Entry<?, ?> entry : butFirst(map.entrySet())) {
+                builder.append(", ");
+                serializeEntry(builder, entry, itemIdMapper);
+            }
+        }
+
+        builder.append('}');
+        return builder.toString();
+    }
+
+    static void serializeEntry(StringBuilder builder, Map.Entry<?, ?> entry, ItemIdMapper itemIdMapper) {
+        builder.append(serialize(entry.getKey(), itemIdMapper)).append(' ').
+                append(serialize(entry.getValue(), itemIdMapper));
+    }
+
+    static String serializeItem(Item item, ItemIdMapper itemIdMapper) {
+        return ItemExecutorRegistry.getByType(item.getItemType()).itemToForm(item, itemIdMapper).serialize(itemIdMapper);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/AllLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/AllLowercasingSearcher.java
new file mode 100644
index 00000000000..deed9e20aa5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/AllLowercasingSearcher.java
@@ -0,0 +1,31 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import java.util.Collection;
+
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.WordItem;
+
+/**
+ * Transform all terms in the incoming query tree and highlight terms to lower
+ * case. This searcher is a compatibility layer for customers needing to use
+ * FSAs created for pre-5.1 systems.
+ *
+ * <p>
+ * Add this searcher to your search chain before any searcher running
+ * case-dependent automata with only lowercased contents, query transformers
+ * assuming lowercased input etc. Refer to the Vespa documentation on search
+ * chains and search chain ordering.
+ * </p>
+ *
+ * @since 5.1.3.
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class AllLowercasingSearcher extends LowercasingSearcher {
+
+    @Override
+    public boolean shouldLowercase(WordItem word, IndexFacts.Session settings) {
+        return true;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java
new file mode 100644
index 00000000000..902de89c94e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanAttributeParser.java
@@ -0,0 +1,170 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.text.PositionedString;
+import com.yahoo.text.SimpleMapParser;
+
+import java.math.BigInteger;
+
+/**
+ * Parses an attribute string on the format <code>{attribute:value, ...}</code>
+ * where <code>value</code>' is either a single value or a list of values
+ * <code>[value1,value2,...]</code>, and each of the values can have an optional
+ * bitmap specified <code>value:bitmap</code>. <code>bitmap</code> can be either
+ * a 64-bit hex number <code>0x1234</code> or a list of bits <code>[0, 2, 43,
+ * 22, ...]</code>.
+ *
+ * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a>
+ * @since 5.1.15
+ */
+abstract class BooleanAttributeParser extends SimpleMapParser {
+    private boolean isMap = true;
+
+    @Override
+    public void parse(String s) {
+        if (s == null || s.length() == 0) return;
+        super.parse(s);
+        if (string().position() != string().string().length()) {
+            throw new IllegalArgumentException("Expected end of string " + string().at());
+        }
+    }
+
+    // Value ends at ',' or '}' for map, and at ',' or ']' for list.
+    @Override
+    protected int findEndOfValue() {
+        if (isMap) {
+            return findNextButSkipLists(new char[]{',','}'}, string().string(), string().position());
+        }
+        return findNextButSkipLists(new char[]{',',']'}, string().string(), string().position());
+    }
+
+    @Override
+    protected void handleKeyValue(String attribute, String value) {
+        // string() will point to the start of value.
+        if (string().peek('[') && isMap) {
+            // begin parsing MultiValueQueryTerm
+            isMap = false;
+            parseMultiValue(attribute);
+            isMap = true;
+        } else {
+            handleAttribute(attribute, value);
+        }
+    }
+
+    /**
+     * Parses a list of values for a given attribute. When calling this
+     * function, string() must point to the start of the list.
+     */
+    private void parseMultiValue(String attribute) {
+        // string() will point to the start of value.
+        string().consume('[');
+        while (!string().peek(']')) {
+            string().consumeSpaces();
+            consumeValue(attribute);
+            string().consumeOptional(',');
+            string().consumeSpaces();
+        }
+    }
+
+    /**
+     * Handles one attribute, possibly with a subquery bitmap.
+     * @param attribute Attribute name
+     * @param value Either value, or value:bitmap, where bitmap is either a 64-bit hex number or a list of bits.
+     */
+    private void handleAttribute(String attribute, String value) {
+        int pos = value.indexOf(':');
+        if (pos != -1) {
+            parseBitmap(attribute, value.substring(0, pos), value.substring(pos + 1));
+        } else {
+            addAttribute(attribute, value);
+        }
+    }
+
+    // Parses a bitmap string that's either a list of bits or a hex number.
+    private void parseBitmap(String attribute, String value, String bitmap) {
+        if (bitmap.charAt(0) == '[') {
+            parseBitmapList(attribute, value, bitmap);
+        } else {
+            parseBitmapHex(attribute, value, bitmap);
+        }
+    }
+
+    /**
+     * Adds attributes with the specified bitmap to normalizer.
+     * @param attribute Attribute to add
+     * @param value Value of attribute
+     * @param bitmap Bitmap as a hex number, with a '0x' prefix.
+     */
+    private void parseBitmapHex(String attribute, String value, String bitmap) {
+        PositionedString s = new PositionedString(bitmap);
+        s.consume('0');
+        s.consume('x');
+        addAttribute(attribute, value, new BigInteger(s.substring().trim(),16));
+    }
+
+    /**
+     * Adds attributes with the specified bitmap to normalizer.
+     * @param attribute Attribute to add
+     * @param value Value of attribute
+     * @param bitmap Bitmap as a list of bits, e.g. '[0, 3, 45]'
+     */
+    private void parseBitmapList(String attribute, String value, String bitmap) {
+        PositionedString s = new PositionedString(bitmap);
+        s.consume('[');
+        BigInteger mask = BigInteger.ZERO;
+        while (!s.peek(']')) {
+            s.consumeSpaces();
+            int pos = findNextButSkipLists(new char[]{',',']'}, s.string(), s.position());
+            if (pos == -1) {
+                break;
+            }
+            int subqueryIndex = Integer.parseUnsignedInt(s.substring(pos).trim());
+            if (subqueryIndex > 63 || subqueryIndex < 0) {
+                throw new IllegalArgumentException("Subquery index must be in the range 0-63");
+            }
+            mask = mask.or(BigInteger.ONE.shiftLeft(subqueryIndex));
+            s.setPosition(pos);
+            s.consumeOptional(',');
+            s.consumeSpaces();
+        }
+        addAttribute(attribute, value, mask);
+    }
+
+    /**
+     * Add an attribute without a subquery mask
+     * @param attribute name of attribute
+     * @param value value of attribute
+     */
+    protected abstract void addAttribute(String attribute, String value);
+
+    /**
+     * Add an attribute with a subquery mask
+     * @param attribute name of attribute
+     * @param value value of attribute
+     * @param subqueryMask subquery mask for attribute (64-bit)
+     */
+    protected abstract void addAttribute(String attribute, String value, BigInteger subqueryMask);
+
+    /**
+     * Finds next index of a set of chars, but skips past any lists ("[...]").
+     * @param chars Characters to find. Note that '[' should not be in this list.
+     * @param s String to search
+     * @param position position in s to start at.
+     * @return position of first char from "chars" that does not appear within brackets.
+     */
+    private static int findNextButSkipLists(char[] chars, String s, int position) {
+        for (; position<s.length(); position++) {
+            if (s.charAt(position)=='[') {
+                position = findNextButSkipLists(new char[]{']'}, s, position + 1);
+                if (position<0) return -1;
+            } else {
+                for (char c : chars) {
+                    if (s.charAt(position)==c)
+                        return position;
+                }
+            }
+        }
+        return -1;
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java
new file mode 100644
index 00000000000..1fd394acd54
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.prelude.query.PredicateQueryItem;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.grouping.request.parser.TokenMgrError;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+
+import java.math.BigInteger;
+
+import static com.yahoo.prelude.querytransform.NormalizingSearcher.ACCENT_REMOVAL;
+import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING;
+import static com.yahoo.yolean.Exceptions.toMessageString;
+
+/**
+ * Searcher that builds a PredicateItem from the &amp;boolean properties and inserts it into a query.
+ * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a>
+ */
+@After({ STEMMING, ACCENT_REMOVAL })
+@Provides(BooleanSearcher.PREDICATE)
+public class BooleanSearcher extends Searcher {
+    private static final CompoundName FIELD = new CompoundName("boolean.field");
+    private static final CompoundName ATTRIBUTES = new CompoundName("boolean.attributes");
+    private static final CompoundName RANGE_ATTRIBUTES = new CompoundName("boolean.rangeAttributes");
+    public static final String PREDICATE = "predicate";
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        String fieldName = query.properties().getString(FIELD);
+        if (fieldName != null) {
+            return search(query, execution, fieldName);
+        } else {
+            if (query.isTraceable(5)) {
+                query.trace("BooleanSearcher: Nothing added to query", false, 5);
+            }
+        }
+        return execution.search(query);
+    }
+
+    private Result search(Query query, Execution execution, String fieldName) {
+        String attributes = query.properties().getString(ATTRIBUTES);
+        String rangeAttributes = query.properties().getString(RANGE_ATTRIBUTES);
+        if (query.isTraceable(5)) {
+            query.trace("BooleanSearcher: fieldName(" + fieldName + "), attributes(" + attributes +
+                        "), rangeAttributes(" + rangeAttributes + ")", false, 5);
+        }
+
+        if (attributes != null || rangeAttributes != null) {
+            try {
+                addPredicateTerm(query, fieldName, attributes, rangeAttributes);
+                if (query.isTraceable(4)) {
+                    query.trace("BooleanSearcher: Added boolean operator", true, 4);
+                }
+            } catch (TokenMgrError e) {
+                return new Result(query, ErrorMessage.createInvalidQueryParameter(toMessageString(e)));
+            }
+        } else {
+            if (query.isTraceable(5)) {
+                query.trace("BooleanSearcher: Nothing added to query", false, 5);
+            }
+        }
+        return execution.search(query);
+    }
+
+     // Adds a boolean term ANDed to the query, based on the supplied properties.
+    private void addPredicateTerm(Query query, String fieldName, String attributes, String rangeAttributes) {
+        PredicateQueryItem item = new PredicateQueryItem();
+        item.setIndexName(fieldName);
+        new PredicateValueAttributeParser(item).parse(attributes);
+        new PredicateRangeAttributeParser(item).parse(rangeAttributes);
+        QueryTreeUtil.andQueryItemWithRoot(query, item);
+    }
+
+    static public class PredicateValueAttributeParser extends BooleanAttributeParser {
+        private PredicateQueryItem item;
+        public PredicateValueAttributeParser(PredicateQueryItem item) {
+            this.item = item;
+        }
+
+        @Override
+        protected void addAttribute(String attribute, String value) {
+            item.addFeature(attribute, value);
+        }
+
+        @Override
+        protected void addAttribute(String attribute, String value, BigInteger subQueryMask) {
+            item.addFeature(attribute, value, subQueryMask.longValue());
+        }
+    }
+
+    static private class PredicateRangeAttributeParser extends BooleanAttributeParser {
+        private PredicateQueryItem item;
+        public PredicateRangeAttributeParser(PredicateQueryItem item) {
+            this.item = item;
+        }
+
+        @Override
+        protected void addAttribute(String attribute, String value) {
+            item.addRangeFeature(attribute, Long.parseLong(value));
+        }
+
+        @Override
+        protected void addAttribute(String attribute, String value, BigInteger subQueryMask) {
+            item.addRangeFeature(attribute, Long.parseLong(value), subQueryMask.longValue());
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java
new file mode 100644
index 00000000000..c2d462a17e4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import static com.yahoo.prelude.searcher.PosSearcher.POSITION_PARSING;
+
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Before;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.Location;
+import com.yahoo.search.Query;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.PhaseNames;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * If default position has not been set, it will be set here.
+ *
+ * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a>
+ */
+@After({PhaseNames.RAW_QUERY, POSITION_PARSING})
+@Before(PhaseNames.TRANSFORMED_QUERY)
+public class DefaultPositionSearcher extends Searcher {
+
+    @Override
+    public com.yahoo.search.Result search(Query query, Execution execution) {
+        Location location = query.getRanking().getLocation();
+        if (location != null && (location.getAttribute() == null)) {
+            IndexFacts facts = execution.context().getIndexFacts();
+            List<String> search = facts.newSession(query.getModel().getSources(), query.getModel().getRestrict()).documentTypes();
+
+            for (String sd : search) {
+                String defaultPosition = facts.getDefaultPosition(sd);
+                if (defaultPosition != null) {
+                    location.setAttribute(defaultPosition);
+                }
+            }
+            if (location.getAttribute() == null) {
+                location.setAttribute(facts.getDefaultPosition(null));
+            }
+        }
+        return execution.search(query);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java
new file mode 100644
index 00000000000..41af5736da7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java
@@ -0,0 +1,365 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import com.yahoo.component.chain.dependencies.Before;
+import com.yahoo.language.Language;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.Index;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.IndexedItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NotItem;
+import com.yahoo.prelude.query.NullItem;
+import com.yahoo.prelude.query.RankItem;
+import com.yahoo.prelude.query.parser.CustomParser;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * Compatibility layer to implement the old multi part query syntax, along with
+ * the features of QueryCombinator. Do <b>not</b> use both QueryCombinator and
+ * LegacyCombinator in a single search.
+ *
+ * <p>
+ * A searcher which grabs query parameters of the form
+ * "defidx.(identifier)=(index name)" and "query.(identifier)=(user query)",
+ * parses them and adds them as AND items to the query root.
+ *
+ * <p>
+ * If the given default index does not exist in the search definition, the query
+ * part will be parsed with the settings of the default index set to "".
+ *
+ * <p>
+ * If any of the following arguments exist, they will be used:
+ *
+ * <p>
+ * query.(identifier)=query string<br>
+ * query.(identifier).operator={"req", "rank", "not"}, where "req" is default<br>
+ * query.(identifier).defidx=default index<br>
+ * query.(identifier).type={"all", "any", "phrase", "adv", "web"} where "all" is
+ * default
+ *
+ * <p>
+ * If both defidx.(identifier) and any of
+ * query.(identifier).{operator,defidx,type} is present in the query, an
+ * InvalidQueryParameter error will be added, and the query will be passed
+ * through untransformed.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@Before({"transformedQuery", "com.yahoo.prelude.querytransform.StemmingSearcher"})
+public class LegacyCombinator extends Searcher {
+
+    private static final String TYPESUFFIX = ".type";
+    private static final String OPERATORSUFFIX = ".operator";
+    private static final String DEFIDXSUFFIX = ".defidx";
+    private static final String DEFIDXPREFIX = "defidx.";
+    private static final String QUERYPREFIX = "query.";
+
+    private enum Combinator {
+        REQUIRED("req"), PREFERRED("rank"), EXCLUDED("not");
+
+        String parameterValue;
+
+        private Combinator(String parameterValue) {
+            this.parameterValue = parameterValue;
+        }
+
+        static Combinator getCombinator(String name) {
+            for (Combinator c : Combinator.values()) {
+                if (c.parameterValue.equals(name)) {
+                    return c;
+                }
+            }
+            return REQUIRED;
+        }
+    }
+
+    private static class QueryPart {
+        final String query;
+        final String defaultIndex;
+        final Combinator operator;
+        final String identifier;
+        final Query.Type syntax;
+
+        QueryPart(String identifier, String defaultIndex, String oldIndex,
+                String operator, String query, String syntax) {
+            validateArguments(identifier, defaultIndex, oldIndex,
+                    operator,syntax);
+            this.query = query;
+            if (defaultIndex != null) {
+                this.defaultIndex = defaultIndex;
+            } else {
+                this.defaultIndex = oldIndex;
+            }
+            this.operator = Combinator.getCombinator(operator);
+            this.identifier = identifier;
+            this.syntax = Query.Type.getType(syntax);
+        }
+
+        private static void validateArguments(String identifier, String defaultIndex,
+                String oldIndex, String operator, String syntax) {
+            if (defaultIndex == null) {
+                return;
+            }
+            if (oldIndex != null) {
+                throw new IllegalArgumentException(createErrorMessage(identifier, DEFIDXSUFFIX));
+            }
+            if (operator != null) {
+                throw new IllegalArgumentException(createErrorMessage(identifier, OPERATORSUFFIX));
+            }
+            if (syntax != null) {
+                throw new IllegalArgumentException(createErrorMessage(identifier, TYPESUFFIX));
+            }
+        }
+
+        private static String createErrorMessage(String identifier, String legacyArgument) {
+            return "Cannot set both " + DEFIDXPREFIX + identifier + " and "
+                    + QUERYPREFIX + identifier + legacyArgument + ".";
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result
+                    + ((identifier == null) ? 0 : identifier.hashCode());
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            QueryPart other = (QueryPart) obj;
+            if (identifier == null) {
+                if (other.identifier != null)
+                    return false;
+            } else if (!identifier.equals(other.identifier))
+                return false;
+            return true;
+        }
+
+        @Override
+        public String toString() {
+            return "QueryPart(" + identifier + ", " + defaultIndex + ", "
+                    + operator + ", " + syntax + ")";
+        }
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        Set<QueryPart> pieces;
+        Set<String> usedSources;
+        IndexFacts indexFacts = execution.context().getIndexFacts();
+        try {
+            pieces = findQuerySnippets(query.properties());
+        } catch (IllegalArgumentException e) {
+            query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator got invalid parameters: "
+                    + e.getMessage()));
+            return execution.search(query);
+        }
+        if (pieces.size() == 0) {
+            return execution.search(query);
+        }
+        IndexFacts.Session session = indexFacts.newSession(query);
+        Language language = query.getModel().getParsingLanguage();
+        addAndItems(language, query, pieces, session, execution.context());
+        addRankItems(language, query, pieces, session, execution.context());
+        try {
+            addNotItems(language, query, pieces, session, execution.context());
+        } catch (IllegalArgumentException e) {
+            query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator found only excluding terms, no including."));
+            return execution.search(query);
+        }
+        query.trace("Adding extra query parts.", true, 2);
+        return execution.search(query);
+    }
+
+    private void addNotItems(Language language, Query query, Set<QueryPart> pieces,
+                             IndexFacts.Session session, Execution.Context context) {
+        for (QueryPart part : pieces) {
+            if (part.operator != Combinator.EXCLUDED) continue;
+
+            String defaultIndex = defaultIndex(session, part);
+            Item item = parse(language, query, part, defaultIndex, context);
+            if (item == null) continue;
+
+            setDefaultIndex(part, defaultIndex, item);
+            addNotItem(query.getModel().getQueryTree(), item);
+        }
+
+    }
+
+    private void addNotItem(QueryTree queryTree, Item item) {
+        Item root = queryTree.getRoot();
+        // JavaDoc claims I can get null, code gives NullItem... well, well, well...
+        if (root instanceof NullItem || root == null) {
+            // errr... no positive branch at all?
+            throw new IllegalArgumentException("No positive terms for query.");
+        } else if (root.getClass() == NotItem.class) {
+            ((NotItem) root).addNegativeItem(item);
+        } else {
+            NotItem newRoot = new NotItem();
+            newRoot.addPositiveItem(root);
+            newRoot.addNegativeItem(item);
+            queryTree.setRoot(newRoot);
+        }
+    }
+
+    private void addRankItems(Language language, Query query, Set<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) {
+        for (QueryPart part : pieces) {
+            if (part.operator != Combinator.PREFERRED) continue;
+
+            String defaultIndex = defaultIndex(session, part);
+            Item item = parse(language, query, part, defaultIndex, context);
+            if (item == null) continue;
+
+            setDefaultIndex(part, defaultIndex, item);
+            addRankItem(query.getModel().getQueryTree(), item);
+        }
+    }
+
+    private void addRankItem(QueryTree queryTree, Item item) {
+        Item root = queryTree.getRoot();
+        // JavaDoc claims I can get null, code gives NullItem... well, well, well...
+        if (root instanceof NullItem || root == null) {
+            queryTree.setRoot(item);
+        } else if (root.getClass() == RankItem.class) {
+            // if no clear recall terms, just set the rank term as recall
+            ((RankItem) root).addItem(item);
+        } else {
+            RankItem newRoot = new RankItem();
+            newRoot.addItem(root);
+            newRoot.addItem(item);
+            queryTree.setRoot(newRoot);
+        }
+    }
+
+    private void addAndItems(Language language, Query query, Iterable<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) {
+        for (QueryPart part : pieces) {
+            if (part.operator != Combinator.REQUIRED) continue;
+
+            String defaultIndex = defaultIndex(session, part);
+            Item item = parse(language, query, part, defaultIndex, context);
+            if (item == null) continue;
+
+            setDefaultIndex(part, defaultIndex, item);
+            addAndItem(query.getModel().getQueryTree(), item);
+        }
+    }
+
+    private void setDefaultIndex(QueryPart part, String defaultIndex, Item item) {
+        if (defaultIndex == null) {
+            assignDefaultIndex(item, part.defaultIndex);
+        }
+    }
+
+    private Item parse(Language language, Query query, QueryPart part, String defaultIndex, Execution.Context context) {
+        Item item = null;
+        try {
+            CustomParser parser = (CustomParser)ParserFactory.newInstance(
+                                               part.syntax, ParserEnvironment.fromExecutionContext(context));
+            item = parser.parse(part.query, null, language, query.getModel().getSources(),
+                                context.getIndexFacts(), defaultIndex);
+        } catch (RuntimeException e) {
+            String err = Exceptions.toMessageString(e);
+            query.trace("Query parser threw an exception: " + err, true, 1);
+            getLogger().log(LogLevel.WARNING,
+                    "Query parser threw exception in searcher LegacyCombinator for "
+                    + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err);
+        }
+        return item;
+    }
+
+    private String defaultIndex(IndexFacts.Session indexFacts, QueryPart part) {
+        String defaultIndex;
+        if (indexFacts.getIndex(part.defaultIndex) == Index.nullIndex) {
+            defaultIndex = null;
+        } else {
+            defaultIndex = part.defaultIndex;
+        }
+        return defaultIndex;
+    }
+
+    private static void addAndItem(QueryTree queryTree, Item item) {
+        Item root = queryTree.getRoot();
+        // JavaDoc claims I can get null, code gives NullItem... well, well, well...
+        if (root instanceof NullItem || root == null) {
+            queryTree.setRoot(item);
+        } else if (root.getClass() == AndItem.class) {
+            ((AndItem) root).addItem(item);
+        } else {
+            AndItem newRoot = new AndItem();
+            newRoot.addItem(root);
+            newRoot.addItem(item);
+            queryTree.setRoot(newRoot);
+        }
+    }
+
+    private static void assignDefaultIndex(Item item, String defaultIndex) {
+        if (item instanceof IndexedItem) {
+            IndexedItem indexName = (IndexedItem) item;
+
+            if ("".equals(indexName.getIndexName())) {
+                indexName.setIndexName(defaultIndex);
+            }
+        } else if (item instanceof CompositeItem) {
+            Iterator<Item> items = ((CompositeItem) item).getItemIterator();
+            while (items.hasNext()) {
+                Item i = items.next();
+                assignDefaultIndex(i, defaultIndex);
+            }
+        }
+
+    }
+
+    private static Set<QueryPart> findQuerySnippets(Properties properties) {
+        Set<QueryPart> pieces = new HashSet<>();
+        for (Map.Entry<String, Object> k : properties.listProperties()
+                .entrySet()) {
+            String key = k.getKey();
+            if (!key.startsWith(QUERYPREFIX)) {
+                continue;
+            }
+            String name = key.substring(QUERYPREFIX.length());
+            if (hasDots(name)) {
+                continue;
+            }
+            String index = properties.getString(DEFIDXPREFIX + name);
+            String oldIndex = properties.getString(QUERYPREFIX + name
+                    + DEFIDXSUFFIX);
+            String operator = properties.getString(QUERYPREFIX + name
+                    + OPERATORSUFFIX);
+            String type = properties.getString(QUERYPREFIX + name + TYPESUFFIX);
+            pieces.add(new QueryPart(name, index, oldIndex, operator, k
+                    .getValue().toString(), type));
+        }
+        return pieces;
+    }
+
+    private static boolean hasDots(String name) {
+        int index = name.indexOf('.', 0);
+        return index != -1;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/LowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/LowercasingSearcher.java
new file mode 100644
index 00000000000..d3916c4bfe1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/LowercasingSearcher.java
@@ -0,0 +1,140 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.*;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * Traverse a query tree and lowercase terms based on decision made in subclasses.
+ *
+ * @since 5.1.3
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public abstract class LowercasingSearcher extends Searcher {
+
+    private final boolean transformWeightedSets;
+
+    public LowercasingSearcher() {
+        this(new LowercasingConfig(new LowercasingConfig.Builder()));
+    }
+
+    public LowercasingSearcher(LowercasingConfig cfg) {
+        this.transformWeightedSets = cfg.transform_weighted_sets();
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        IndexFacts.Session indexFacts = execution.context().getIndexFacts().newSession(query);
+        traverse(query.getModel().getQueryTree(), indexFacts);
+        traverseHighlight(query.getPresentation().getHighlight(), indexFacts);
+        query.trace("Lowercasing", true, 2);
+        return execution.search(query);
+    }
+
+    private void traverseHighlight(Highlight highlight, IndexFacts.Session indexFacts) {
+        if (highlight == null) return;
+
+        for (AndItem item : highlight.getHighlightItems().values()) {
+            traverse(item, indexFacts);
+        }
+    }
+
+    private void traverse(CompositeItem base, IndexFacts.Session indexFacts) {
+        for (Iterator<Item> i = base.getItemIterator(); i.hasNext();) {
+            Item next = i.next();
+            if (next instanceof WordItem) {
+                lowerCase((WordItem) next, indexFacts);
+            } else if (next instanceof CompositeItem) {
+                traverse((CompositeItem) next, indexFacts);
+            } else if (next instanceof WeightedSetItem) {
+                if (transformWeightedSets) {
+                    lowerCase((WeightedSetItem) next, indexFacts);
+                }
+            } else if (next instanceof WordAlternativesItem) {
+                lowerCase((WordAlternativesItem) next, indexFacts);
+            }
+        }
+    }
+
+    private void lowerCase(WordItem word, IndexFacts.Session indexFacts) {
+        if (shouldLowercase(word, indexFacts)) {
+            word.setWord(toLowerCase(word.getWord()));
+            word.setLowercased(true);
+        }
+    }
+
+    private static final class WeightedSetToken {
+        final String token;
+        final String originalToken;
+        final int weight;
+
+        WeightedSetToken(String token, String originalToken, int weight) {
+            this.token = token;
+            this.originalToken = originalToken;
+            this.weight = weight;
+        }
+    }
+
+    private boolean syntheticLowerCaseCheck(String indexName, IndexFacts.Session indexFacts, boolean isFromQuery) {
+        WordItem w = new WordItem("", indexName, isFromQuery);
+        return shouldLowercase(w, indexFacts);
+    }
+
+    private void lowerCase(WeightedSetItem set, IndexFacts.Session indexFacts) {
+        if (!syntheticLowerCaseCheck(set.getIndexName(), indexFacts, true)) {
+            return;
+        }
+
+        List<WeightedSetToken> terms = new ArrayList<>(set.getNumTokens());
+        for (Iterator<Map.Entry<Object, Integer>> i = set.getTokens(); i.hasNext();) {
+            Map.Entry<Object, Integer> e = i.next();
+            if (e.getKey() instanceof String) {
+                String originalToken = (String) e.getKey();
+                String token = toLowerCase(originalToken);
+                if ( ! originalToken.equals(token)) {
+                    terms.add(new WeightedSetToken(token, originalToken, e.getValue().intValue()));
+                }
+            }
+        }
+        // has to do it in two passes on cause of the "interesting" API in
+        // weighted set, and remove before put on cause of the semantics of
+        // addInternal as well as changed values...
+        for (WeightedSetToken t : terms) {
+            set.removeToken(t.originalToken);
+            set.addToken(t.token, t.weight);
+        }
+    }
+
+    private void lowerCase(WordAlternativesItem alternatives, IndexFacts.Session indexFacts) {
+        if (!syntheticLowerCaseCheck(alternatives.getIndexName(), indexFacts, alternatives.isFromQuery())) {
+            return;
+        }
+        for (WordAlternativesItem.Alternative term : alternatives.getAlternatives()) {
+            String lowerCased = toLowerCase(term.word);
+            alternatives.addTerm(lowerCased, term.exactness * .7d);
+        }
+
+    }
+
+    /**
+     * Override this to control whether a given term should be lowercased.
+     *
+     * @param word a WordItem or subclass thereof which is a candidate for lowercasing
+     * @return whether to convert the term to lower case
+     */
+    public abstract boolean shouldLowercase(WordItem word, IndexFacts.Session indexFacts);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java
new file mode 100644
index 00000000000..c487182c65d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java
@@ -0,0 +1,285 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.CharacterClasses;
+import com.yahoo.language.process.GramSplitter;
+import com.yahoo.prelude.Index;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.hitfield.AnnotateStringFieldPart;
+import com.yahoo.prelude.hitfield.JSONString;
+import com.yahoo.prelude.hitfield.XMLString;
+import com.yahoo.prelude.query.*;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import static com.yahoo.prelude.searcher.JuniperSearcher.JUNIPER_TAG_REPLACING;
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * Handles NGram indexes by splitting query terms to them into grams and combining summary field values
+ * from such fields into the original text.
+ * <p>
+ * This declares it must be placed after Juniper searchers because it assumes Juniper token separators
+ * (which are returned on bolding) are not replaced by highlight tags when this is run (and "after" means
+ * "before" from the point of view of the result).
+ *
+ * @author bratseth
+ */
+@After(JUNIPER_TAG_REPLACING)
+public class NGramSearcher extends Searcher {
+
+    private final GramSplitter gramSplitter;
+
+    private final CharacterClasses characterClasses;
+
+    public NGramSearcher(Linguistics linguistics) {
+        gramSplitter= linguistics.getGramSplitter();
+        characterClasses= linguistics.getCharacterClasses();
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        IndexFacts indexFacts = execution.context().getIndexFacts();
+        if ( ! indexFacts.hasNGramIndices()) return execution.search(query); // shortcut
+
+        IndexFacts.Session session = indexFacts.newSession(query);
+        boolean rewritten = rewriteToNGramMatching(query.getModel().getQueryTree().getRoot(), 0, session, query);
+        if (rewritten)
+            query.trace("Rewritten to n-gram matching",true,2);
+
+        Result result=execution.search(query);
+        recombineNGrams(result.hits().deepIterator(), session);
+        return result;
+    }
+
+    @Override
+    public void fill(Result result, String summaryClass, Execution execution) {
+        execution.fill(result, summaryClass);
+        IndexFacts indexFacts = execution.context().getIndexFacts();
+        if (indexFacts.hasNGramIndices())
+            recombineNGrams(result.hits().deepIterator(), indexFacts.newSession(result.getQuery()));
+    }
+
+    private boolean rewriteToNGramMatching(Item item, int indexInParent, IndexFacts.Session indexFacts, Query query) {
+        boolean rewritten = false;
+        if (item instanceof SegmentItem) { // handle CJK segmented terms which should be grams instead
+            SegmentItem segments = (SegmentItem)item;
+            Index index = indexFacts.getIndex(segments.getIndexName());
+            if (index.isNGram()) {
+                Item grams = splitToGrams(segments, toLowerCase(segments.getRawWord()), index.getGramSize(), query);
+                replaceItemByGrams(item, grams, indexInParent);
+                rewritten = true;
+            }
+        }
+        else if (item instanceof CompositeItem) {
+            CompositeItem composite = (CompositeItem)item;
+            for (int i=0; i<composite.getItemCount(); i++)
+                rewritten = rewriteToNGramMatching(composite.getItem(i), i, indexFacts, query) || rewritten;
+        }
+        else if (item instanceof TermItem) {
+            TermItem term = (TermItem)item;
+            Index index = indexFacts.getIndex(term.getIndexName());
+            if (index.isNGram()) {
+                Item grams = splitToGrams(term,term.stringValue(), index.getGramSize(), query);
+                replaceItemByGrams(item, grams, indexInParent);
+                rewritten = true;
+            }
+        }
+        return rewritten;
+    }
+
+    /**
+     * Splits the given item into n-grams and adds them as a CompositeItem containing WordItems searching the
+     * index of the input term. If the result is a single gram, that single WordItem is returned rather than the AndItem
+     *
+     * @param term the term to split, must be an item which implement the IndexedItem and BlockItem "mixins"
+     * @param text the text of the item, just stringValue() if the item is a TermItem
+     * @param gramSize the gram size to split to
+     * @param query the query in which this rewriting is done
+     * @return the root of the query subtree produced by this, containing the split items
+     */
+    protected Item splitToGrams(Item term, String text, int gramSize, Query query) {
+        CompositeItem and = createGramRoot(query);
+        String index = ((HasIndexItem)term).getIndexName();
+        Substring origin = ((BlockItem)term).getOrigin();
+        for (Iterator<GramSplitter.Gram> i = getGramSplitter().split(text,gramSize); i.hasNext(); ) {
+            GramSplitter.Gram gram = i.next();
+            WordItem gramWord = new WordItem(gram.extractFrom(text), index, false, origin);
+            gramWord.setWeight(term.getWeight());
+            gramWord.setProtected(true);
+            and.addItem(gramWord);
+        }
+        return and.getItemCount()==1 ? and.getItem(0) : and; // return the AndItem, or just the single gram if not multiple
+    }
+
+    /**
+     * Returns the (thread-safe) object to use to split the query text into grams.
+     */
+    protected final GramSplitter getGramSplitter() { return gramSplitter; }
+
+    /**
+     * Creates the root of the query subtree which will contain the grams to match,
+     * called by {@link #splitToGrams}. This hook is provided to make it easy to create a subclass which
+     * matches grams using a different composite item, e.g an OrItem.
+     * <p>
+     * This default implementation return new AndItem();
+     *
+     * @param query the input query, to make it possible to return a different composite item type
+     *        depending on the query content
+     * @return the composite item to add the gram items to in {@link #splitToGrams}
+     */
+    protected CompositeItem createGramRoot(Query query) {
+        return new AndItem();
+    }
+
+    private void replaceItemByGrams(Item item, Item grams, int indexInParent) {
+        if (!(grams instanceof CompositeItem) || !(item.getParent() instanceof PhraseItem)) { // usually, simply replace
+            item.getParent().setItem(indexInParent, grams);
+        }
+        else { // but if the parent is a phrase, we cannot add the AND to it, so add each gram to the phrase
+            PhraseItem phraseParent = (PhraseItem)item.getParent();
+            phraseParent.removeItem(indexInParent);
+            int addedTerms = 0;
+            for (Iterator<Item> i = ((CompositeItem)grams).getItemIterator(); i.hasNext(); ) {
+                phraseParent.addItem(indexInParent+(addedTerms++),i.next());
+            }
+        }
+    }
+
+    private void recombineNGrams(Iterator<Hit> hits, IndexFacts.Session session) {
+        while (hits.hasNext()) {
+            Hit hit = hits.next();
+            if (hit.isMeta()) continue;
+            Object sddocname = hit.getField(Hit.SDDOCNAME_FIELD);
+            if (sddocname == null) return;
+            for (String fieldName : hit.fieldKeys()) {
+                Index index = session.getIndex(fieldName, sddocname.toString());
+                if (index.isNGram() && (index.getHighlightSummary() || index.getDynamicSummary())) {
+                    hit.setField(fieldName, recombineNGramsField(hit.getField(fieldName), index.getGramSize()));
+                }
+            }
+        }
+    }
+
+    private Object recombineNGramsField(Object fieldValue,int gramSize) {
+        String recombined=recombineNGrams(fieldValue.toString(),gramSize);
+        if (fieldValue instanceof JSONString)
+            return new JSONString(recombined);
+        else if (fieldValue instanceof XMLString)
+            return new XMLString(recombined);
+        else
+            return recombined;
+    }
+
+    /**
+     * Converts grams to the original string.
+     *
+     * Example (gram size 3): <code>blulue rededs</code> becomes <code>blue reds</code>
+     */
+    private String recombineNGrams(final String string,final int gramSize) {
+        StringBuilder b=new StringBuilder();
+        int consecutiveWordChars=0;
+        boolean inBolding=false;
+        MatchTokenStrippingCharacterIterator characters=new MatchTokenStrippingCharacterIterator(string);
+        while (characters.hasNext()) {
+            char c=characters.next();
+            boolean atBoldingSeparator = (c=='\u001f');
+
+            if (atBoldingSeparator && characters.peek()=='\u001f') {
+                characters.next();
+            }
+            else if ( ! characterClasses.isLetterOrDigit(c)) {
+                if (atBoldingSeparator)
+                    inBolding=!inBolding;
+                if ( ! (atBoldingSeparator && nextIsLetterOrDigit(characters)))
+                    consecutiveWordChars=0;
+                if (inBolding && atBoldingSeparator && areWordCharactersBackwards(gramSize-1,b)) {
+                    // we are going to skip characters from a gram, so move bolding start earlier
+                    b.insert(b.length()-(gramSize-1),c);
+                }
+                else {
+                    b.append(c);
+                }
+            }
+            else {
+                consecutiveWordChars++;
+                if (consecutiveWordChars<gramSize || (consecutiveWordChars % gramSize)==0)
+                    b.append(c);
+            }
+        }
+        return b.toString();
+    }
+
+    private boolean areWordCharactersBackwards(int count,StringBuilder b) {
+        for (int i=0; i<count; i++) {
+            int checkIndex=b.length()-1-i;
+            if (checkIndex<0) return false;
+            if ( ! characterClasses.isLetterOrDigit(b.charAt(checkIndex))) return false;
+        }
+        return true;
+    }
+
+    private boolean nextIsLetterOrDigit(MatchTokenStrippingCharacterIterator characters) {
+        return characterClasses.isLetterOrDigit(characters.peek());
+    }
+
+    /**
+     * A string wrapper which skips match token forms marked up Juniper style, such that
+     * \uFFF9originalToken\uFFFAtoken\uFFFB is returned as originalToken
+     */
+    private static class MatchTokenStrippingCharacterIterator {
+
+        private final String s;
+        private int current =0;
+
+        public MatchTokenStrippingCharacterIterator(String s) {
+            this.s=s;
+        }
+
+        public boolean hasNext() {
+            skipMarkup();
+            return current <s.length();
+        }
+
+        public char next() {
+            skipMarkup();
+            return s.charAt(current++);
+        }
+
+        /** Returns the next character without moving to it. Returns \uFFFF if there is no next */
+        public char peek() {
+            skipMarkup();
+            if (s.length()< current +1)
+                return '\uFFFF';
+            else
+                return s.charAt(current);
+        }
+
+        private void skipMarkup() {
+            if (current>=s.length()) return;
+            char c=s.charAt(current);
+            if (c== AnnotateStringFieldPart.RAW_ANNOTATE_BEGIN_CHAR) { // skip it
+                current++;
+            }
+            else if (c==AnnotateStringFieldPart.RAW_ANNOTATE_SEPARATOR_CHAR) { // skip to RAW_ANNOTATE_END_CHAR
+                do {
+                    current++;
+                } while (current<s.length() && s.charAt(current)!=AnnotateStringFieldPart.RAW_ANNOTATE_END_CHAR);
+                current++; // also skip the RAW_ANNOTATE_END_CHAR
+                skipMarkup(); // skip any immediately following markup
+            }
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java
new file mode 100644
index 00000000000..3a209a58f4a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java
@@ -0,0 +1,155 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.language.Language;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.Index;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.parser.CustomParser;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.query.Properties;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.IndexedItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NullItem;
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * <p>A searcher which grabs query parameters of the form "defidx.(identifier)=(index name)" and
+ * "query.(identifier)=(user query)", * parses them and adds them as AND items to the query root.</p>
+ *
+ * <p>If the given default index does not exist in the search definition, the query part will be parsed with the
+ * settings of the default index set to the "".</p>
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class QueryCombinator extends Searcher {
+    private static final String QUERYPREFIX = "query.";
+
+    private static class QueryPart {
+        final String query;
+        final String defaultIndex;
+
+        QueryPart(String query, String defaultIndex) {
+            this.query = query;
+            this.defaultIndex = defaultIndex;
+        }
+    }
+
+    public QueryCombinator(ComponentId id) {
+        super(id);
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        Set<QueryPart> pieces = findQuerySnippets(query.properties());
+        if (pieces.size() == 0) {
+            return execution.search(query);
+        }
+        addAndItems(query, pieces, execution.context());
+        query.trace("Adding extra query parts.", true, 2);
+        return execution.search(query);
+    }
+
+    private void addAndItems(Query query, Iterable<QueryPart> pieces, Execution.Context context) {
+        IndexFacts indexFacts = context.getIndexFacts();
+        IndexFacts.Session session = indexFacts.newSession(query);
+        Set<String> usedSources = new HashSet<>(session.documentTypes());
+        Language language = query.getModel().getParsingLanguage();
+        for (QueryPart part : pieces) {
+            String defaultIndex;
+            Item item = null;
+            Index index = session.getIndex(part.defaultIndex);
+            if (index == Index.nullIndex) {
+                defaultIndex = null;
+            } else {
+                defaultIndex = part.defaultIndex;
+            }
+            try {
+                CustomParser parser = (CustomParser)ParserFactory.newInstance(query.getModel().getType(),
+                                                                              ParserEnvironment.fromExecutionContext(context));
+                item = parser.parse(part.query, null, language, usedSources, indexFacts, defaultIndex);
+            } catch (RuntimeException e) {
+                String err = Exceptions.toMessageString(e);
+                query.trace("Query parser threw an exception: " + err, true, 1);
+                getLogger().log(LogLevel.WARNING,
+                        "Query parser threw exception searcher QueryCombinator for "
+                        + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err);
+            }
+            if (item == null) {
+                continue;
+            }
+            if (defaultIndex == null) {
+                assignDefaultIndex(item, part.defaultIndex);
+            }
+            addAndItem(query.getModel().getQueryTree(), item);
+        }
+    }
+
+    private static void addAndItem(QueryTree queryTree, Item item) {
+        Item root = queryTree.getRoot();
+        // JavaDoc claims I can get null, code gives NullItem... well, well, well...
+        if (root instanceof NullItem || root == null) {
+            queryTree.setRoot(item);
+        } else if (root.getClass() == AndItem.class) {
+            ((AndItem) root).addItem(item);
+        } else {
+            AndItem newRoot = new AndItem();
+            newRoot.addItem(root);
+            newRoot.addItem(item);
+            queryTree.setRoot(newRoot);
+        }
+    }
+
+    private static void assignDefaultIndex(Item item, String defaultIndex) {
+        if (item instanceof IndexedItem) {
+            IndexedItem indexName = (IndexedItem) item;
+
+            if ("".equals(indexName.getIndexName())) {
+                indexName.setIndexName(defaultIndex);
+            }
+        } else if (item instanceof CompositeItem) {
+            Iterator<Item> items = ((CompositeItem) item).getItemIterator();
+            while (items.hasNext()) {
+                Item i = items.next();
+                assignDefaultIndex(i, defaultIndex);
+            }
+        }
+    }
+
+    private static Set<QueryPart> findQuerySnippets(Properties properties) {
+        Set<QueryPart> pieces = new HashSet<>();
+        for (Map.Entry<String, Object> k : properties.listProperties().entrySet()) {
+            String key = k.getKey();
+            if (!key.startsWith(QUERYPREFIX)) {
+                continue;
+            }
+            String name = key.substring(QUERYPREFIX.length());
+            if (hasDots(name)) {
+                continue;
+            }
+            String index = properties.getString("defidx." + name);
+            pieces.add(new QueryPart(k.getValue().toString(), index));
+        }
+        return pieces;
+    }
+
+    private static boolean hasDots(String name) {
+        int index = name.indexOf('.', 0);
+        return index != -1;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java
new file mode 100644
index 00000000000..fb5373d59ea
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.QueryTree;
+
+/**
+ * Utility class for manipulating a QueryTree.
+ *
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ */
+public class QueryTreeUtil {
+
+    static public void andQueryItemWithRoot(Query query, Item item) {
+        andQueryItemWithRoot(query.getModel().getQueryTree(), item);
+    }
+
+    static public void andQueryItemWithRoot(QueryTree tree, Item item) {
+        if (tree.isEmpty()) {
+            tree.setRoot(item);
+        } else {
+            Item oldRoot = tree.getRoot();
+            if (oldRoot.getClass() == AndItem.class) {
+                ((AndItem) oldRoot).addItem(item);
+            } else {
+                AndItem newRoot = new AndItem();
+                newRoot.addItem(oldRoot);
+                newRoot.addItem(item);
+                tree.setRoot(newRoot);
+            }
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/RangeQueryOptimizer.java b/container-search/src/main/java/com/yahoo/search/querytransform/RangeQueryOptimizer.java
new file mode 100644
index 00000000000..65832d99461
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/RangeQueryOptimizer.java
@@ -0,0 +1,212 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.prelude.query.Limit;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.FalseItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.QueryCanonicalizer;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.PhaseNames;
+import com.yahoo.yolean.chain.After;
+import com.yahoo.yolean.chain.Before;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Finds and optimizes ranges in queries:
+ * For single value attributes c1 $lt; x AND x &gt; c2  becomes  x IN &lt;c1; c2&gt;.
+ * The query cost saving from this has been shown to be 2 orders of magnitude in real cases.
+ *
+ * @author bratseth
+ */
+@Before(QueryCanonicalizer.queryCanonicalization)
+@After(PhaseNames.TRANSFORMED_QUERY)
+public class RangeQueryOptimizer extends Searcher {
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        if (execution.context().getIndexFacts() == null) return execution.search(query); // this is a test query
+
+        boolean optimized = recursiveOptimize(query.getModel().getQueryTree(), execution.context().getIndexFacts().newSession(query));
+        if (optimized)
+            query.trace("Optimized query ranges", true, 2);
+        return execution.search(query);
+    }
+
+    /** Recursively performs the range optimization on this query tree and returns whether at least one optimization was done */
+    private boolean recursiveOptimize(Item item, IndexFacts.Session indexFacts) {
+        if ( ! (item instanceof CompositeItem)) return false;
+
+        boolean optimized = false;
+        for (Iterator<Item> i = ((CompositeItem) item).getItemIterator(); i.hasNext(); )
+            optimized |= recursiveOptimize(i.next(), indexFacts);
+
+        if (item instanceof AndItem)
+            optimized |= optimizeAnd((AndItem)item, indexFacts);
+        return optimized;
+    }
+
+    private boolean optimizeAnd(AndItem and, IndexFacts.Session indexFacts) {
+        // Find consolidated ranges by collecting a list of compatible ranges
+        List<FieldRange> fieldRanges = null;
+        for (Iterator<Item> i = and.getItemIterator(); i.hasNext(); ) {
+            Item item = i.next();
+            if ( ! (item instanceof IntItem)) continue;
+            IntItem intItem = (IntItem)item;
+            if (intItem.getHitLimit() != 0) continue; // each such op gets a different partial set: Cannot be optimized
+            if (intItem.getFromLimit().equals(intItem.getToLimit())) continue; // don't optimize searches for single numbers
+            if (indexFacts.getIndex(intItem.getIndexName()).isMultivalue()) continue; // May match different values in each range
+
+            if (fieldRanges == null) fieldRanges = new ArrayList<>();
+            Optional<FieldRange> compatibleRange = findCompatibleRange(intItem, fieldRanges);
+            if (compatibleRange.isPresent())
+                compatibleRange.get().addRange(intItem);
+            else
+                fieldRanges.add(new FieldRange(intItem));
+            i.remove();
+        }
+
+        // Add consolidated ranges
+        if (fieldRanges == null) return false;
+
+        boolean optimized = false;
+        for (FieldRange fieldRange : fieldRanges) {
+            and.addItem(fieldRange.toItem());
+            optimized |= fieldRange.isOptimization();
+        }
+        return optimized;
+    }
+
+    private Optional<FieldRange> findCompatibleRange(IntItem item, List<FieldRange> fieldRanges) {
+        for (FieldRange fieldRange : fieldRanges) {
+            if (fieldRange.isCompatibleWith(item))
+                return Optional.of(fieldRange);
+        }
+        return Optional.empty();
+    }
+
+    /** Represents the ranges searched in a single field */
+    private static final class FieldRange {
+
+        private Range range = new Range(new Limit(Double.NEGATIVE_INFINITY, false), new Limit(Double.POSITIVE_INFINITY, false));
+        private int sourceRangeCount = 0;
+
+        // IntItem fields which must be preserved in the produced item.
+        // This is an unfortunate coupling and ideally we should delegate this (creation, compatibility)
+        // to the Item classes
+        private final String indexName;
+        private final Item.ItemCreator creator;
+        private final boolean ranked;
+        private final int weight;
+
+        public FieldRange(IntItem item) {
+            this.indexName = item.getIndexName();
+            this.creator = item.getCreator();
+            this.ranked = item.isRanked();
+            this.weight = item.getWeight();
+            addRange(item);
+        }
+
+        public String getIndexName() { return indexName; }
+
+        public boolean isCompatibleWith(IntItem item) {
+            if ( ! indexName.equals(item.getIndexName())) return false;
+            if (creator != item.getCreator()) return false;
+            if (ranked != item.isRanked()) return false;
+            if (weight != item.getWeight()) return false;
+            return true;
+        }
+
+        /** Adds a range for this field */
+        public void addRange(IntItem item) {
+            range = range.intersection(new Range(item));
+            sourceRangeCount++;
+        }
+
+        public Item toItem() {
+            Item item = range.toItem(indexName);
+            item.setCreator(creator);
+            item.setRanked(ranked);
+            item.setWeight(weight);
+            return item;
+        }
+
+        /** Returns whether this range is actually an optimization over what was in the source query */
+        public boolean isOptimization() { return sourceRangeCount > 1; }
+
+    }
+
+    /** An immutable numerical range */
+    private static class Range {
+
+        private final Limit from;
+        private final Limit to;
+
+        private static final Range empty = new EmptyRange();
+
+        public Range(Limit from, Limit to) {
+            this.from = from;
+            this.to = to;
+        }
+
+        public Range(IntItem range) {
+            from = range.getFromLimit();
+            to = range.getToLimit();
+        }
+
+        /** Returns true if these two ranges overlap */
+        public boolean overlaps(Range other) {
+            if (other.from.isSmallerOrEqualTo(this.to) && other.to.isLargerOrEqualTo(this.from)) return true;
+            if (other.to.isLargerOrEqualTo(this.from) && other.from.isSmallerOrEqualTo(this.to)) return true;
+            return false;
+        }
+
+        /**
+         * Returns the intersection of this and the given range.
+         * If the ranges does not overlap, an empty range is returned.
+         */
+        public Range intersection(Range other) {
+            if ( ! overlaps(other)) return empty;
+            return new Range(from.max(other.from), to.min(other.to));
+        }
+
+        public Item toItem(String fieldName) {
+            return IntItem.from(fieldName, from, to, 0);
+        }
+
+        @Override
+        public String toString() { return "[" + from + ";" + to + "]"; }
+
+    }
+
+    private static class EmptyRange extends Range {
+
+        public EmptyRange() {
+            super(new Limit(0, false), new Limit(0, false)); // the to and from of an empty range is never used.
+        }
+
+        @Override
+        public boolean overlaps(Range other) { return false; }
+
+        @Override
+        public Range intersection(Range other) { return this; }
+
+        @Override
+        public Item toItem(String fieldName) { return new FalseItem(); }
+
+        @Override
+        public String toString() { return "(empty)"; }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java
new file mode 100644
index 00000000000..5886014deed
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java
@@ -0,0 +1,105 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.prelude.Index;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.QueryCanonicalizer;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.grouping.GroupingQueryParser;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.query.Sorting;
+import com.yahoo.search.query.properties.DefaultProperties;
+import com.yahoo.search.query.ranking.MatchPhase;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.yolean.chain.After;
+import com.yahoo.yolean.chain.Before;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * If the query is eligible, specify that the query should degrade if it causes too many hits
+ * to avoid excessively expensive queries.
+ * <p>
+ * Queries are eligible if they do sorting, don't do grouping, and the first sort criteria is a fast-search attribute.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+
+// This writes fields to query.getRanking which are moved to rank.properties during query.prepare()
+// Query.prepare is done at the same time as canonicalization (by GroupingExecutor), so use that constraint.
+// (we're not adding another constraint at this point because all this preparation and encoding business
+// should be fixed when we move to Slime for serialization. - Jon, in the spring of the year of 2014)
+@Before(QueryCanonicalizer.queryCanonicalization)
+
+// We are checking if there is a grouping expression, not if there is a raw grouping instruction property,
+// so we must run after the property is transferred to a grouping expression
+@After(GroupingQueryParser.SELECT_PARAMETER_PARSING)
+
+public class SortingDegrader extends Searcher {
+
+    /** Set this to false in query.properties to turn off degrading. Default: on */
+    // (this is not called ranking.sorting.degrading because it should not be part of the query object model
+    public static final CompoundName DEGRADING = new CompoundName("sorting.degrading");
+
+    public static final CompoundName PAGINATION = new CompoundName("to_be_removed_pagination");
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        if (shouldBeDegraded(query, execution.context().getIndexFacts().newSession(query)))
+            setDegradation(query);
+        return execution.search(query);
+    }
+
+    private boolean shouldBeDegraded(Query query, IndexFacts.Session indexFacts) {
+        if (query.getRanking().getSorting() == null) return false;
+        if (query.getRanking().getSorting().fieldOrders().isEmpty()) return false;
+        if ( ! GroupingRequest.getRequests(query).isEmpty()) return false;
+        if ( ! query.properties().getBoolean(DEGRADING, true)) return false;
+
+        Index index = indexFacts.getIndex(query.getRanking().getSorting().fieldOrders().get(0).getFieldName());
+        if (index == null) return false;
+        if ( ! index.isFastSearch()) return false;
+        if ( ! index.isNumerical()) return false;
+
+        return true;
+    }
+
+    private void setDegradation(Query query) {
+        Sorting.FieldOrder primarySort = query.getRanking().getSorting().fieldOrders().get(0); // ensured above
+        MatchPhase matchPhase = query.getRanking().getMatchPhase();
+
+        matchPhase.setAttribute(primarySort.getFieldName());
+        matchPhase.setAscending(primarySort.getSortOrder() == Sorting.Order.ASCENDING);
+        if (matchPhase.getMaxHits() == null)
+            matchPhase.setMaxHits(decideDefaultMaxHits(query));
+    }
+
+    /**
+     * Look at a "reasonable" number of this by default. We don't want to set this too low because it impacts
+     * the totalHits value returned.
+     * <p>
+     * If maxhits/offset is set high, use that as the default instead because it means somebody will want to be able to
+     * get lots of hits. We could use hits+offset instead of maxhits+maxoffset but that would destroy pagination
+     * with large values because totalHits is wrong.
+     * <p>
+     * If we ever get around to estimate totalhits we can rethink this.
+     */
+    private long decideDefaultMaxHits(Query query) {
+        int maxHits;
+        int maxOffset;
+        if (query.properties().getBoolean(PAGINATION, true)) {
+            maxHits = query.properties().getInteger(DefaultProperties.MAX_HITS);
+            maxOffset = query.properties().getInteger(DefaultProperties.MAX_OFFSET);
+        } else {
+            maxHits = query.getHits();
+            maxOffset = query.getOffset();
+        }
+        return maxHits + maxOffset;
+    }
+
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
new file mode 100644
index 00000000000..2e8e0861656
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
@@ -0,0 +1,39 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import static com.yahoo.prelude.querytransform.NormalizingSearcher.ACCENT_REMOVAL;
+import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING;
+
+import java.util.Collection;
+
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.prelude.Index;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.WordItem;
+
+/**
+ * Transform terms in query tree to lower case based on Vespa index settings.
+ *
+ * @since 5.1.3
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@After({ STEMMING, ACCENT_REMOVAL })
+@Provides(VespaLowercasingSearcher.LOWERCASING)
+public class VespaLowercasingSearcher extends LowercasingSearcher {
+
+    public static final String LOWERCASING = "LowerCasing";
+
+    public VespaLowercasingSearcher(LowercasingConfig cfg) {
+        super(cfg);
+    }
+
+    @Override
+    public boolean shouldLowercase(WordItem word, IndexFacts.Session indexFacts) {
+        if (word.isLowercased()) return false;
+
+        Index index = indexFacts.getIndex(word.getIndexName());
+        return index.isLowercase() || index.isAttribute();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java
new file mode 100644
index 00000000000..6120a7aee30
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java
@@ -0,0 +1,206 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.querytransform;
+
+import com.yahoo.prelude.Index;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.*;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.text.MapParser;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static com.yahoo.container.protect.Error.UNSPECIFIED;
+import com.yahoo.yolean.Exceptions;
+
+/**
+ * Searcher that will create a Vespa WAND item from a list of tokens with weights.
+ * IndexFacts is used to determine which WAND to create.
+ *
+ * @since 5.1.11
+ * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a>
+ * @author bratseth
+ */
+public class WandSearcher extends Searcher {
+
+    /**
+     * Enum used to represent which "wand" this searcher should produce.
+     */
+    private enum WandType {
+        VESPA("vespa"),
+        OR("or"),
+        PARALLEL("parallel"),
+        DOT_PRODUCT("dotProduct");
+
+        private final String type;
+
+        WandType(String type) {
+            this.type = type;
+        }
+
+        public static WandType create(String type) {
+            for (WandType enumType : WandType.values()) {
+                if (enumType.type.equals(type)) {
+                    return enumType;
+                }
+            }
+            return WandType.VESPA;
+        }
+    }
+
+    /**
+     * Class to resolve the inputs used by this searcher.
+     */
+    private static class InputResolver {
+
+        private static final CompoundName WAND_FIELD = new CompoundName("wand.field");
+        private static final CompoundName WAND_TOKENS = new CompoundName("wand.tokens");
+        private static final CompoundName WAND_HEAP_SIZE = new CompoundName("wand.heapSize");
+        private static final CompoundName WAND_TYPE = new CompoundName("wand.type");
+        private static final CompoundName WAND_SCORE_THRESHOLD = new CompoundName("wand.scoreThreshold");
+        private static final CompoundName WAND_THRESHOLD_BOOST_FACTOR = new CompoundName("wand.thresholdBoostFactor");
+        private final String fieldName;
+        private final WandType wandType;
+        private final Map<String, Integer> tokens;
+        private final int heapSize;
+        private final double scoreThreshold;
+        private final double thresholdBoostFactor;
+
+        public InputResolver(Query query, Execution execution) {
+            fieldName = query.properties().getString(WAND_FIELD);
+            if (fieldName != null) {
+                String tokens = query.properties().getString(WAND_TOKENS);
+                if (tokens != null) {
+                    wandType = resolveWandType(execution.context().getIndexFacts().newSession(query), query);
+                    this.tokens = new IntegerMapParser().parse(tokens, new LinkedHashMap<>());
+                    heapSize = resolveHeapSize(query);
+                    scoreThreshold = resolveScoreThreshold(query);
+                    thresholdBoostFactor = resolveThresholdBoostFactor(query);
+                    return;
+                }
+            }
+            wandType = null;
+            tokens = null;
+            heapSize = 0;
+            scoreThreshold = 0;
+            thresholdBoostFactor = 1;
+        }
+
+        private WandType resolveWandType(IndexFacts.Session indexFacts, Query query) {
+            Index index = indexFacts.getIndex(fieldName);
+            if (index.isNull()) {
+                throw new IllegalArgumentException("Field '" + fieldName + "' was not found in " + indexFacts);
+            } else {
+                return WandType.create(query.properties().getString(WAND_TYPE, "vespa"));
+            }
+        }
+
+        private int resolveHeapSize(Query query) {
+            String defaultHeapSize = "100";
+            return Integer.valueOf(query.properties().getString(WAND_HEAP_SIZE, defaultHeapSize));
+        }
+
+        private double resolveScoreThreshold(Query query) {
+            return Double.valueOf(query.properties().getString(WAND_SCORE_THRESHOLD, "0"));
+        }
+
+        private double resolveThresholdBoostFactor(Query query) {
+            return Double.valueOf(query.properties().getString(WAND_THRESHOLD_BOOST_FACTOR, "1"));
+        }
+
+        public boolean hasValidData() {
+            return tokens != null && !tokens.isEmpty();
+        }
+
+        public String getFieldName() {
+            return fieldName;
+        }
+
+        public Map<String, Integer> getTokens() {
+            return tokens;
+        }
+
+        public WandType getWandType() {
+            return wandType;
+        }
+
+        public Integer getHeapSize() {
+            return heapSize;
+        }
+
+        public Double getScoreThreshold() {
+            return scoreThreshold;
+        }
+
+        public Double getThresholdBoostFactor() {
+            return thresholdBoostFactor;
+        }
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        try {
+            InputResolver inputs = new InputResolver(query, execution);
+            if ( ! inputs.hasValidData()) return execution.search(query);
+
+            QueryTreeUtil.andQueryItemWithRoot(query, createWandQueryItem(inputs));
+            query.trace("WandSearcher: Added WAND operator", true, 4);
+            return execution.search(query);
+        }
+        catch (IllegalArgumentException e) {
+            return new Result(query,ErrorMessage.createInvalidQueryParameter(Exceptions.toMessageString(e)));
+        }
+    }
+
+    private Item createWandQueryItem(InputResolver inputs) {
+        if (inputs.getWandType().equals(WandType.VESPA)) {
+            return populate(new WeakAndItem(inputs.getHeapSize()), inputs.getFieldName(), inputs.getTokens());
+        } else if (inputs.getWandType().equals(WandType.OR)) {
+            return populate(new OrItem(), inputs.getFieldName(), inputs.getTokens());
+        } else if (inputs.getWandType().equals(WandType.PARALLEL)) {
+            return populate(new WandItem(inputs.getFieldName(), inputs.getHeapSize()),
+                    inputs.getScoreThreshold(), inputs.getThresholdBoostFactor(), inputs.getTokens());
+        } else if (inputs.getWandType().equals(WandType.DOT_PRODUCT)) {
+            return populate(new DotProductItem(inputs.getFieldName()), inputs.getTokens());
+        }
+        throw new IllegalArgumentException("Unknown type '" + inputs.getWandType() + "'");
+    }
+
+    private CompositeItem populate(CompositeItem parent, String fieldName, Map<String,Integer> tokens) {
+        for (Map.Entry<String,Integer> entry : tokens.entrySet()) {
+            WordItem wordItem = new WordItem(entry.getKey(), fieldName);
+            wordItem.setWeight(entry.getValue());
+            wordItem.setStemmed(true);
+            wordItem.setNormalizable(false);
+            parent.addItem(wordItem);
+        }
+        return parent;
+    }
+
+    private WeightedSetItem populate(WeightedSetItem item, Map<String,Integer> tokens) {
+        for (Map.Entry<String,Integer> entry : tokens.entrySet()) {
+            item.addToken(entry.getKey(), entry.getValue());
+        }
+        return item;
+    }
+
+    private WandItem populate(WandItem item, double scoreThreshold, double thresholdBoostFactor, Map<String,Integer> tokens) {
+        populate(item, tokens);
+        item.setScoreThreshold(scoreThreshold);
+        item.setThresholdBoostFactor(thresholdBoostFactor);
+        return item;
+    }
+
+    private static class IntegerMapParser extends MapParser<Integer> {
+        @Override
+        protected Integer parseValue(String s) {
+            return Integer.parseInt(s);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/package-info.java b/container-search/src/main/java/com/yahoo/search/querytransform/package-info.java
new file mode 100644
index 00000000000..34e59301fca
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/package-info.java
@@ -0,0 +1,9 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Vespa search platform query transformation infrastructure. Not a public
+ * API.
+ */
+@ExportPackage
+package com.yahoo.search.querytransform;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/parser/.gitignore b/container-search/src/main/java/com/yahoo/search/querytransform/parser/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/parser/.gitignore
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java
new file mode 100644
index 00000000000..de817d95393
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java
@@ -0,0 +1,450 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.rendering;
+
+import com.yahoo.concurrent.CopyOnWriteHashMap;
+import com.yahoo.io.ByteWriter;
+import com.yahoo.net.URI;
+import com.yahoo.prelude.fastsearch.FastHit;
+import com.yahoo.prelude.fastsearch.GroupingListHit;
+import com.yahoo.prelude.templates.UserTemplate;
+import com.yahoo.processing.rendering.AsynchronousSectionedRenderer;
+import com.yahoo.processing.response.Data;
+import com.yahoo.processing.response.DataList;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.grouping.result.HitRenderer;
+import com.yahoo.search.query.context.QueryContext;
+import com.yahoo.search.result.*;
+import com.yahoo.text.Utf8String;
+import com.yahoo.text.XMLWriter;
+import com.yahoo.yolean.trace.TraceNode;
+import com.yahoo.yolean.trace.TraceVisitor;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.util.Iterator;
+import java.util.Map;
+
+// TODO: Rename to XmlRenderer and make this a deprecated empty subclass.
+
+/**
+ * XML rendering of search results. This is NOT the default (but it once was).
+ *
+ * @author tonytv
+ */
+@SuppressWarnings({ "rawtypes", "deprecation" })
+public final class DefaultRenderer extends AsynchronousSectionedRenderer<Result> {
+
+    public static final String  DEFAULT_MIMETYPE    = "text/xml";
+    public static final String  DEFAULT_ENCODING    = "utf-8";
+
+    private static final Utf8String RESULT = new Utf8String("result");
+    private static final Utf8String GROUP = new Utf8String("group");
+    private static final Utf8String ID = new Utf8String("id");
+    private static final Utf8String FIELD = new Utf8String("field");
+    private static final Utf8String HIT = new Utf8String("hit");
+    private static final Utf8String ERROR = new Utf8String("error");
+    private static final Utf8String TOTAL_HIT_COUNT = new Utf8String("total-hit-count");
+    private static final Utf8String QUERY_TIME = new Utf8String("querytime");
+    private static final Utf8String SUMMARY_FETCH_TIME = new Utf8String("summaryfetchtime");
+    private static final Utf8String SEARCH_TIME = new Utf8String("searchtime");
+    private static final Utf8String NAME = new Utf8String("name");
+    private static final Utf8String CODE = new Utf8String("code");
+    private static final Utf8String COVERAGE_DOCS = new Utf8String("coverage-docs");
+    private static final Utf8String COVERAGE_NODES = new Utf8String("coverage-nodes");
+    private static final Utf8String COVERAGE_FULL = new Utf8String("coverage-full");
+    private static final Utf8String COVERAGE = new Utf8String("coverage");
+    private static final Utf8String RESULTS_FULL = new Utf8String("results-full");
+    private static final Utf8String RESULTS = new Utf8String("results");
+    private static final Utf8String TYPE = new Utf8String("type");
+    private static final Utf8String RELEVANCY = new Utf8String("relevancy");
+    private static final Utf8String SOURCE = new Utf8String("source");
+
+
+    // this is shared between umpteen threads by design
+    private final CopyOnWriteHashMap<String, Utf8String> fieldNameMap = new CopyOnWriteHashMap<>();
+
+    private boolean utf8Output = false;
+
+    private XMLWriter writer;
+
+    @Override
+    public void init() {
+        super.init();
+        utf8Output = false;
+        writer = null;
+    }
+
+    @Override
+    public String getEncoding() {
+
+        if (getResult() == null
+                || getResult().getQuery() == null
+                || getResult().getQuery().getModel().getEncoding() == null) {
+            return DEFAULT_ENCODING;
+        } else {
+            return getResult().getQuery().getModel().getEncoding();
+        }
+    }
+
+    @Override
+    public String getMimeType() {
+        return DEFAULT_MIMETYPE;
+    }
+
+    private XMLWriter wrapWriter(Writer writer) {
+        return XMLWriter.from(writer, 10, -1);
+    }
+
+    private void header(XMLWriter writer, Result result) throws IOException {
+        // TODO: move setting this to Result
+        utf8Output = "utf-8".equalsIgnoreCase(getRequestedEncoding(result.getQuery()));
+        writer.xmlHeader(getRequestedEncoding(result.getQuery()));
+        writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT, String.valueOf(result.getTotalHitCount()));
+        if (result.getQuery().getPresentation().getReportCoverage()) {
+            renderCoverageAttributes(result.getCoverage(false), writer);
+        }
+        renderTime(writer, result);
+        writer.closeStartTag();
+    }
+
+    private void renderTime(XMLWriter writer, Result result) {
+        if (!result.getQuery().getPresentation().getTiming()) {
+            return;
+        }
+
+        final String threeDecimals = "%.3f";
+        final double milli = .001d;
+        final long now = System.currentTimeMillis();
+        final long searchTime = now - result.getElapsedTime().first();
+        final double searchSeconds = ((double) searchTime) * milli;
+
+        if (result.getElapsedTime().firstFill() != 0L) {
+            final long queryTime = result.getElapsedTime().weightedSearchTime();
+            final long summaryFetchTime = result.getElapsedTime().weightedFillTime();
+            final double querySeconds = ((double) queryTime) * milli;
+            final double summarySeconds = ((double) summaryFetchTime) * milli;
+            writer.attribute(QUERY_TIME, String.format(threeDecimals, querySeconds));
+            writer.attribute(SUMMARY_FETCH_TIME, String.format(threeDecimals, summarySeconds));
+        }
+        writer.attribute(SEARCH_TIME, String.format(threeDecimals, searchSeconds));
+    }
+
+    protected static void renderCoverageAttributes(Coverage coverage, XMLWriter writer) throws IOException {
+        if (coverage == null) return;
+        writer.attribute(COVERAGE_DOCS,coverage.getDocs());
+        writer.attribute(COVERAGE_NODES,coverage.getNodes());
+        writer.attribute(COVERAGE_FULL,coverage.getFull());
+        writer.attribute(COVERAGE,coverage.getResultPercentage());
+        writer.attribute(RESULTS_FULL,coverage.getFullResultSets());
+        writer.attribute(RESULTS,coverage.getResultSets());
+    }
+
+
+    public void error(XMLWriter writer, Result result) throws IOException {
+        ErrorMessage error = result.hits().getError();
+        writer.openTag(ERROR).attribute(CODE,error.getCode()).content(error.getMessage(),false).closeTag();
+    }
+
+
+    @SuppressWarnings("UnusedParameters")
+    protected void emptyResult(XMLWriter writer, Result result) throws IOException {}
+
+    @SuppressWarnings("UnusedParameters")
+    public void queryContext(XMLWriter writer, QueryContext queryContext, Query owner) throws IOException {
+        if (owner.getTraceLevel()!=0) {
+            XMLWriter xmlWriter=XMLWriter.from(writer);
+            xmlWriter.openTag("meta").attribute("type", QueryContext.ID);
+            TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root();
+            traceRoot.accept(new RenderingVisitor(xmlWriter, owner.getStartTime()));
+            xmlWriter.closeTag();
+        }
+    }
+
+
+    private void renderSingularHit(XMLWriter writer, Hit hit) throws IOException {
+        writer.openTag(HIT);
+        renderHitAttributes(writer, hit);
+        writer.closeStartTag();
+        renderHitFields(writer, hit);
+    }
+
+    private void renderHitFields(XMLWriter writer, Hit hit) throws IOException {
+        renderSyntheticRelevanceField(writer, hit);
+        for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) {
+            renderField(writer, hit, it);
+        }
+    }
+
+    private void renderField(XMLWriter writer, Hit hit, Iterator<Map.Entry<String, Object>> it) throws IOException {
+        Map.Entry<String, Object> entry = it.next();
+        boolean isProbablyNotDecoded = false;
+        if (hit instanceof FastHit) {
+            FastHit f = (FastHit) hit;
+            isProbablyNotDecoded = f.fieldIsNotDecoded(entry.getKey());
+        }
+        renderGenericFieldPossiblyNotDecoded(writer, hit, entry, isProbablyNotDecoded);
+    }
+
+    private void renderGenericFieldPossiblyNotDecoded(XMLWriter writer, Hit hit, Map.Entry<String, Object> entry, boolean probablyNotDecoded) throws IOException {
+        String fieldName = entry.getKey();
+
+        // skip depending on hit type
+        if (fieldName.startsWith("$")) return; // Don't render fields that start with $ // TODO: Move to should render
+
+        writeOpenFieldElement(writer, fieldName);
+        renderFieldContentPossiblyNotDecoded(writer, hit, probablyNotDecoded, fieldName);
+        writeCloseFieldElement(writer);
+    }
+
+    private void renderFieldContentPossiblyNotDecoded(XMLWriter writer, Hit hit, boolean probablyNotDecoded, String fieldName) throws IOException {
+        boolean dumpedRaw = false;
+        if (probablyNotDecoded && (hit instanceof FastHit)) {
+            writer.closeStartTag();
+            if ((writer.getWriter() instanceof ByteWriter) && utf8Output) {
+                dumpedRaw = UserTemplate.dumpBytes((ByteWriter) writer.getWriter(), (FastHit) hit, fieldName);
+            }
+            if (dumpedRaw) {
+                writer.content("", false); // let the xml writer note that this tag had content
+            }
+        }
+        if (!dumpedRaw) {
+            String xmlval = hit.getFieldXML(fieldName);
+            if (xmlval == null) {
+                xmlval = "(null)";
+            }
+            writer.escapedContent(xmlval, false);
+        }
+    }
+
+    private void renderSyntheticRelevanceField(XMLWriter writer, Hit hit) throws IOException {
+        final String relevancyFieldName = "relevancy";
+        final Relevance relevance = hit.getRelevance();
+
+        // skip depending on hit type
+        if (relevance != null) {
+            renderSimpleField(writer, relevancyFieldName, relevance);
+        }
+    }
+
+    private void renderSimpleField(XMLWriter writer, String relevancyFieldName, Relevance relevance) throws IOException {
+        writeOpenFieldElement(writer, relevancyFieldName);
+        writer.content(relevance.toString(), false);
+        writeCloseFieldElement(writer);
+    }
+
+    private void writeCloseFieldElement(XMLWriter writer) throws IOException {
+        writer.closeTag();
+    }
+
+    private void writeOpenFieldElement(XMLWriter writer, String relevancyFieldName) throws IOException {
+        Utf8String utf8 = fieldNameMap.get(relevancyFieldName);
+        if (utf8 == null) {
+            utf8 = new Utf8String(relevancyFieldName);
+            fieldNameMap.put(relevancyFieldName, utf8);
+        }
+        writer.openTag(FIELD).attribute(NAME, utf8);
+        writer.closeStartTag();
+    }
+
+    private void renderHitAttributes(XMLWriter writer, Hit hit) throws IOException {
+        writer.attribute(TYPE, hit.getTypeString());
+        if (hit.getRelevance() != null) {
+            writer.attribute(RELEVANCY, hit.getRelevance().toString());
+}
+        writer.attribute(SOURCE, hit.getSource());
+    }
+
+    private void renderHitGroup(XMLWriter writer, HitGroup hit) throws IOException {
+        if (HitRenderer.renderHeader(hit, writer)) {
+            // empty
+        } else if (hit.types().contains("grouphit")) {
+            // TODO Keep this?
+            renderHitGroupOfTypeGroupHit(writer, hit);
+        } else {
+            renderGroup(writer, hit);
+        }
+    }
+
+    private void renderGroup(XMLWriter writer, HitGroup hit) throws IOException {
+        writer.openTag(GROUP);
+        renderHitAttributes(writer, hit);
+        writer.closeStartTag();
+    }
+
+    private void renderHitGroupOfTypeGroupHit(XMLWriter writer, HitGroup hit) throws IOException {
+        writer.openTag(HIT);
+        renderHitAttributes(writer, hit);
+        renderId(writer, hit);
+        writer.closeStartTag();
+    }
+
+    private void renderId(XMLWriter writer, HitGroup hit) throws IOException {
+        URI uri = hit.getId();
+        if (uri != null) {
+            writer.openTag(ID).content(uri.stringValue(),false).closeTag();
+        }
+    }
+
+    private boolean simpleRenderHit(XMLWriter writer, Hit hit) throws IOException {
+        if (hit instanceof DefaultErrorHit) {
+            return simpleRenderDefaultErrorHit(writer, (DefaultErrorHit) hit);
+        } else if (hit instanceof GroupingListHit) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public static boolean simpleRenderDefaultErrorHit(XMLWriter writer, ErrorHit defaultErrorHit) throws IOException {
+        writer.openTag("errordetails");
+        for (Iterator i = defaultErrorHit.errorIterator(); i.hasNext();) {
+            ErrorMessage error = (ErrorMessage) i.next();
+            renderMessageDefaultErrorHit(writer, error);
+        }
+        writer.closeTag();
+        return true;
+    }
+
+    public static void renderMessageDefaultErrorHit(XMLWriter writer, ErrorMessage error) throws IOException {
+        writer.openTag("error");
+        writer.attribute("source", error.getSource());
+        writer.attribute("error", error.getMessage());
+        writer.attribute("code", Integer.toString(error.getCode()));
+        writer.content(error.getDetailedMessage(), false);
+        if (error.getCause()!=null) {
+            writer.openTag("cause");
+            writer.content("\n", true);
+            StringWriter stackTrace=new StringWriter();
+            error.getCause().printStackTrace(new PrintWriter(stackTrace));
+            writer.content(stackTrace.toString(), true);
+            writer.closeTag();
+        }
+        writer.closeTag();
+    }
+
+    public static final class RenderingVisitor extends TraceVisitor {
+
+        private static final String tag = "p";
+        private final XMLWriter writer;
+        private long baseTime;
+
+        public RenderingVisitor(XMLWriter writer,long baseTime) {
+            this.writer=writer;
+            this.baseTime=baseTime;
+        }
+
+        @Override
+        public void entering(TraceNode node) {
+            if (node.isRoot()) return;
+            writer.openTag(tag);
+        }
+
+        @Override
+        public void leaving(TraceNode node) {
+            if (node.isRoot()) return;
+            writer.closeTag();
+        }
+
+        @Override
+        public void visit(TraceNode node) {
+            if (node.isRoot()) return;
+            if (node.payload()==null) return;
+
+            writer.openTag(tag);
+            if (node.timestamp()!=0)
+                writer.content(node.timestamp()-baseTime,false).content(" ms: ", false);
+            writer.content(node.payload().toString(),false);
+            writer.closeTag();
+        }
+
+    }
+
+    private Result getResult() {
+        Result r;
+        try {
+            r = (Result) getResponse();
+        } catch (ClassCastException e) {
+            throw new IllegalArgumentException(
+                    "DefaultRenderer attempted used outside a search context, got a "
+                    + getResponse().getClass().getName());
+        }
+        return r;
+    }
+
+    @Override
+    public void beginResponse(OutputStream stream) throws IOException {
+        Charset cs = Charset.forName(getRequestedEncoding(getResult().getQuery()));
+        CharsetEncoder encoder = cs.newEncoder();
+        writer = wrapWriter(new ByteWriter(stream, encoder));
+
+        header(writer, getResult());
+        if (getResult().hits().getError() != null || getResult().hits().getQuery().errors().size() > 0) {
+            error(writer, getResult());
+        }
+
+        if (getResult().getConcreteHitCount() == 0) {
+            emptyResult(writer, getResult());
+        }
+
+        if (getResult().getContext(false) != null) {
+            queryContext(writer, getResult().getContext(false), getResult().getQuery());
+        }
+
+    }
+
+    /** Returns the encoding of the query, or the encoding given by the template if none is set */
+    public final String getRequestedEncoding(Query query) {
+        String encoding = query.getModel().getEncoding();
+        if (encoding != null) return encoding;
+        return getEncoding();
+    }
+
+    @Override
+    public void beginList(DataList<?> list)
+            throws IOException {
+        if (getRecursionLevel() == 1) {
+            return;
+        }
+        HitGroup hit = (HitGroup) list;
+        boolean renderedSimple = simpleRenderHit(writer, hit);
+
+        if (renderedSimple) {
+            return;
+        }
+        renderHitGroup(writer, hit);
+    }
+
+    @Override
+    public void data(Data data) throws IOException {
+        Hit hit = (Hit) data;
+        boolean renderedSimple = simpleRenderHit(writer, hit);
+
+        if (renderedSimple) {
+            return;
+        }
+        renderSingularHit(writer, hit);
+        writer.closeTag();
+    }
+
+    @Override
+    public void endList(DataList<?> list)
+            throws IOException {
+        if (getRecursionLevel() == 1) {
+            return;
+        }
+        writer.closeTag();
+    }
+
+    @Override
+    public void endResponse() throws IOException {
+        writer.closeTag();
+        writer.close();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
new file mode 100644
index 00000000000..94fe5dd446d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -0,0 +1,790 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.rendering;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.TreeNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+import com.yahoo.data.JsonProducer;
+import com.yahoo.data.access.Inspectable;
+import com.yahoo.data.access.simple.JsonRender;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.datatypes.StringFieldValue;
+import com.yahoo.document.json.JsonWriter;
+import com.yahoo.prelude.fastsearch.FastHit;
+import com.yahoo.processing.Response;
+import com.yahoo.processing.execution.Execution.Trace;
+import com.yahoo.processing.rendering.AsynchronousSectionedRenderer;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.processing.response.Data;
+import com.yahoo.processing.response.DataList;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.grouping.result.AbstractList;
+import com.yahoo.search.grouping.result.BucketGroupId;
+import com.yahoo.search.grouping.result.Group;
+import com.yahoo.search.grouping.result.GroupId;
+import com.yahoo.search.grouping.result.RawBucketId;
+import com.yahoo.search.grouping.result.RawId;
+import com.yahoo.search.grouping.result.RootGroup;
+import com.yahoo.search.grouping.result.ValueGroupId;
+import com.yahoo.search.result.Coverage;
+import com.yahoo.search.result.DefaultErrorHit;
+import com.yahoo.search.result.ErrorHit;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.search.result.NanNumber;
+import com.yahoo.yolean.trace.TraceNode;
+import com.yahoo.yolean.trace.TraceVisitor;
+
+/**
+ * JSON renderer for search results.
+ *
+ * @author Steinar Knutsen
+ */
+// NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc.
+public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
+
+    private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug");
+
+    private enum RenderDecision {
+        YES, NO, DO_NOT_KNOW;
+
+        boolean booleanValue() {
+            switch (this) {
+            case YES:
+                return true;
+            case NO:
+                return false;
+            default:
+                throw new IllegalStateException();
+            }
+        }
+    };
+
+    // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString
+    private static final String BUCKET_LIMITS = "limits";
+    private static final String BUCKET_TO = "to";
+    private static final String BUCKET_FROM = "from";
+    private static final String CHILDREN = "children";
+    private static final String CONTINUATION = "continuation";
+    private static final String COVERAGE = "coverage";
+    private static final String COVERAGE_COVERAGE = "coverage";
+    private static final String COVERAGE_DOCUMENTS = "documents";
+    private static final String COVERAGE_FULL = "full";
+    private static final String COVERAGE_NODES = "nodes";
+    private static final String COVERAGE_RESULTS = "results";
+    private static final String COVERAGE_RESULTS_FULL = "resultsFull";
+    private static final String ERRORS = "errors";
+    private static final String ERROR_CODE = "code";
+    private static final String ERROR_MESSAGE = "message";
+    private static final String ERROR_SOURCE = "source";
+    private static final String ERROR_STACK_TRACE = "stackTrace";
+    private static final String ERROR_SUMMARY = "summary";
+    private static final String FIELDS = "fields";
+    private static final String ID = "id";
+    private static final String LABEL = "label";
+    private static final String RELEVANCE = "relevance";
+    private static final String ROOT = "root";
+    private static final String SOURCE = "source";
+    private static final String TOTAL_COUNT = "totalCount";
+    private static final String TRACE = "trace";
+    private static final String TRACE_CHILDREN = "children";
+    private static final String TRACE_MESSAGE = "message";
+    private static final String TRACE_TIMESTAMP = "timestamp";
+    private static final String TIMING = "timing";
+    private static final String QUERY_TIME = "querytime";
+    private static final String SUMMARY_FETCH_TIME = "summaryfetchtime";
+    private static final String SEARCH_TIME = "searchtime";
+    private static final String TYPES = "types";
+    private static final String GROUPING_VALUE = "value";
+    private static final String VESPA_HIDDEN_FIELD_PREFIX = "$";
+
+    private final JsonFactory generatorFactory;
+
+    private JsonGenerator generator;
+    private Deque<Integer> renderedChildren;
+    private boolean debugRendering;
+    private LongSupplier timeSource;
+
+    private class TraceRenderer extends TraceVisitor {
+        private final long basetime;
+        private boolean hasFieldName = false;
+        int emittedChildNesting = 0;
+        int currentChildNesting = 0;
+        private boolean insideOpenObject = false;
+
+        TraceRenderer(long basetime) {
+            this.basetime = basetime;
+        }
+
+        @Override
+        public void entering(TraceNode node) {
+            ++currentChildNesting;
+        }
+
+        @Override
+        public void leaving(TraceNode node) {
+            conditionalEndObject();
+            if (currentChildNesting == emittedChildNesting) {
+                try {
+                    generator.writeEndArray();
+                    generator.writeEndObject();
+                } catch (IOException e) {
+                    throw new TraceRenderWrapper(e);
+                }
+                --emittedChildNesting;
+            }
+            --currentChildNesting;
+        }
+
+        @Override
+        public void visit(TraceNode node) {
+            try {
+                doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext());
+            } catch (IOException e) {
+                throw new TraceRenderWrapper(e);
+            }
+        }
+
+        private void doVisit(final long timestamp, final Object payload, final boolean hasChildren)
+                throws IOException, JsonGenerationException {
+            boolean dirty = false;
+            if (timestamp != 0L) {
+                header();
+                generator.writeStartObject();
+                generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime);
+                dirty = true;
+            }
+            if (payload != null) {
+                if (!dirty) {
+                    header();
+                    generator.writeStartObject();
+                }
+                generator.writeStringField(TRACE_MESSAGE, payload.toString());
+                dirty = true;
+            }
+            if (dirty) {
+                if (!hasChildren) {
+                    generator.writeEndObject();
+                } else {
+                    setInsideOpenObject(true);
+                }
+            }
+        }
+
+        private void header() {
+            fieldName();
+            for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) {
+                startChildArray();
+            }
+            emittedChildNesting = currentChildNesting;
+        }
+
+        private void startChildArray() {
+            try {
+                conditionalStartObject();
+                generator.writeArrayFieldStart(TRACE_CHILDREN);
+            } catch (IOException e) {
+                throw new TraceRenderWrapper(e);
+            }
+        }
+
+        private void conditionalStartObject() throws IOException, JsonGenerationException {
+            if (!isInsideOpenObject()) {
+                generator.writeStartObject();
+            } else {
+                setInsideOpenObject(false);
+            }
+        }
+
+        private void conditionalEndObject() {
+            if (isInsideOpenObject()) {
+                // This triggers if we were inside a data node with payload and
+                // subnodes, but none of the subnodes contained data
+                try {
+                    generator.writeEndObject();
+                    setInsideOpenObject(false);
+                } catch (IOException e) {
+                    throw new TraceRenderWrapper(e);
+                }
+            }
+        }
+
+        private void fieldName() {
+            if (hasFieldName) {
+                return;
+            }
+
+            try {
+                generator.writeFieldName(TRACE);
+            } catch (IOException e) {
+                throw new TraceRenderWrapper(e);
+            }
+            hasFieldName = true;
+        }
+
+        boolean isInsideOpenObject() {
+            return insideOpenObject;
+        }
+
+        void setInsideOpenObject(boolean insideOpenObject) {
+            this.insideOpenObject = insideOpenObject;
+        }
+    }
+
+    private static final class TraceRenderWrapper extends RuntimeException {
+
+        /**
+         * Should never be serialized, but this is still needed.
+         */
+        private static final long serialVersionUID = 2L;
+
+        TraceRenderWrapper(IOException wrapped) {
+            super(wrapped);
+        }
+
+    }
+
+    public JsonRenderer() {
+        generatorFactory = new JsonFactory();
+        generatorFactory.setCodec(createJsonCodec());
+    }
+
+    /**
+     * Create the codec used for rendering instances of {@link TreeNode}. This
+     * method will be invoked when creating the first renderer instance, but not
+     * for each fresh clone used by individual results.
+     *
+     * @return an object mapper for the internal JsonFactory
+     */
+    protected static ObjectMapper createJsonCodec() {
+        return new ObjectMapper();
+    }
+
+    @Override
+    public void init() {
+        super.init();
+        generator = null;
+        renderedChildren = null;
+        debugRendering = false;
+        timeSource = () -> System.currentTimeMillis();
+    }
+
+    @Override
+    public void beginResponse(OutputStream stream) throws IOException {
+        generator = generatorFactory.createGenerator(stream, JsonEncoding.UTF8);
+        renderedChildren = new ArrayDeque<>();
+        debugRendering = getDebugRendering(getResult().getQuery());
+        generator.writeStartObject();
+        renderTrace(getExecution().trace());
+        renderTiming();
+        generator.writeFieldName(ROOT);
+    }
+
+    private void renderTiming() throws IOException {
+        if (!getResult().getQuery().getPresentation().getTiming()) {
+            return;
+        }
+
+        final double milli = .001d;
+        final long now = timeSource.getAsLong();
+        final long searchTime = now - getResult().getElapsedTime().first();
+        final double searchSeconds = searchTime * milli;
+
+        generator.writeObjectFieldStart(TIMING);
+        if (getResult().getElapsedTime().firstFill() != 0L) {
+            final long queryTime = getResult().getElapsedTime().weightedSearchTime();
+            final long summaryFetchTime = getResult().getElapsedTime().weightedFillTime();
+            final double querySeconds = queryTime * milli;
+            final double summarySeconds = summaryFetchTime * milli;
+            generator.writeNumberField(QUERY_TIME, querySeconds);
+            generator.writeNumberField(SUMMARY_FETCH_TIME, summarySeconds);
+        }
+
+        generator.writeNumberField(SEARCH_TIME, searchSeconds);
+        generator.writeEndObject();
+    }
+
+    private boolean getDebugRendering(Query q) {
+        return q == null ? false : q.properties().getBoolean(DEBUG_RENDERING_KEY, false);
+    }
+
+    private void renderTrace(Trace trace) throws JsonGenerationException, IOException {
+        if (!trace.traceNode().children().iterator().hasNext()) {
+            return;
+        }
+        try {
+            long basetime = trace.traceNode().timestamp();
+            if (basetime == 0L) {
+                basetime = getResult().getElapsedTime().first();
+            }
+            trace.accept(new TraceRenderer(basetime));
+        } catch (TraceRenderWrapper e) {
+            throw new IOException(e);
+        }
+    }
+
+    @Override
+    public void beginList(DataList<?> list) throws IOException {
+        Preconditions.checkArgument(list instanceof HitGroup,
+                "Expected subclass of com.yahoo.search.result.HitGroup, got %s.",
+                list.getClass());
+        moreChildren();
+
+        renderHitGroupHead((HitGroup) list);
+    }
+
+    protected void moreChildren() throws IOException, JsonGenerationException {
+        if (!renderedChildren.isEmpty()) {
+            childrenArray();
+        }
+        renderedChildren.push(0);
+    }
+
+    private void childrenArray() throws IOException, JsonGenerationException {
+        if (renderedChildren.peek() == 0) {
+            generator.writeArrayFieldStart(CHILDREN);
+        }
+        renderedChildren.push(renderedChildren.pop() + 1);
+    }
+
+    private void lessChildren() throws IOException, JsonGenerationException {
+        int lastRenderedChildren = renderedChildren.pop();
+        if (lastRenderedChildren > 0) {
+            generator.writeEndArray();
+        }
+    }
+
+    private void renderHitGroupHead(HitGroup hitGroup) throws JsonGenerationException, IOException {
+        final ErrorHit errorHit = hitGroup.getErrorHit();
+
+        generator.writeStartObject();
+        renderHitContents(hitGroup);
+        if (getRecursionLevel() == 1) {
+            renderCoverage();
+        }
+        if (errorHit != null) {
+            renderErrors(errorHit.errors());
+        }
+
+        // the framework will invoke begin methods as needed from here
+    }
+
+    private void renderErrors(Set<ErrorMessage> errors) throws JsonGenerationException, IOException {
+        if (errors.isEmpty()) {
+            return;
+        }
+        generator.writeArrayFieldStart(ERRORS);
+        for (ErrorMessage e : errors) {
+            String summary = e.getMessage();
+            String source = e.getSource();
+            Throwable cause = e.getCause();
+            String message = e.getDetailedMessage();
+            generator.writeStartObject();
+            generator.writeNumberField(ERROR_CODE, e.getCode());
+            generator.writeStringField(ERROR_SUMMARY, summary);
+            if (source != null) {
+                generator.writeStringField(ERROR_SOURCE, source);
+            }
+            if (message != null) {
+                generator.writeStringField(ERROR_MESSAGE, message);
+            }
+            if (cause != null && cause.getStackTrace().length > 0) {
+                StringWriter s = new StringWriter();
+                PrintWriter p = new PrintWriter(s);
+                cause.printStackTrace(p);
+                p.close();
+                generator.writeStringField(ERROR_STACK_TRACE, s.toString());
+            }
+            generator.writeEndObject();
+        }
+        generator.writeEndArray();
+
+
+    }
+
+    private void renderCoverage() throws JsonGenerationException, IOException {
+        Coverage c = getResult().getCoverage(false);
+        if (c == null) {
+            return;
+        }
+        generator.writeObjectFieldStart(COVERAGE);
+        generator.writeNumberField(COVERAGE_COVERAGE, c.getResultPercentage());
+        generator.writeNumberField(COVERAGE_DOCUMENTS, c.getDocs());
+        generator.writeBooleanField(COVERAGE_FULL, c.getFull());
+        generator.writeNumberField(COVERAGE_NODES, c.getNodes());
+        generator.writeNumberField(COVERAGE_RESULTS, c.getResultSets());
+        generator.writeNumberField(COVERAGE_RESULTS_FULL, c.getFullResultSets());
+        generator.writeEndObject();
+    }
+
+    private void renderHit(Hit hit) throws JsonGenerationException, IOException {
+        if (!shouldRender(hit)) {
+            return;
+        }
+
+        childrenArray();
+        generator.writeStartObject();
+        renderHitContents(hit);
+        generator.writeEndObject();
+    }
+
+    private boolean shouldRender(Hit hit) {
+        if (hit instanceof DefaultErrorHit) {
+            return false;
+        }
+
+        return true;
+    }
+
+    private boolean fieldsStart(boolean hasFieldsField) throws JsonGenerationException, IOException {
+        if (hasFieldsField) {
+            return true;
+        }
+        generator.writeObjectFieldStart(FIELDS);
+        return true;
+    }
+
+    private void fieldsEnd(boolean hasFieldsField) throws JsonGenerationException, IOException {
+        if (!hasFieldsField) {
+            return;
+        }
+        generator.writeEndObject();
+    }
+
+    private void renderHitContents(Hit hit) throws JsonGenerationException, IOException {
+        String id = hit.getDisplayId();
+        Set<String> types = hit.types();
+        String source = hit.getSource();
+
+        if (id != null) {
+            generator.writeStringField(ID, id);
+        }
+        generator.writeNumberField(RELEVANCE, hit.getRelevance().getScore());
+        if (types.size() > 0) {
+            generator.writeArrayFieldStart(TYPES);
+            for (String t : types) {
+                generator.writeString(t);
+            }
+            generator.writeEndArray();
+        }
+        if (source != null) {
+            generator.writeStringField(SOURCE, hit.getSource());
+        }
+        renderSpecialCasesForGrouping(hit);
+
+        renderAllFields(hit);
+    }
+
+    private void renderAllFields(Hit hit) throws JsonGenerationException,
+            IOException {
+        boolean hasFieldsField = false;
+
+        hasFieldsField |= renderTotalHitCount(hit, hasFieldsField);
+        hasFieldsField |= renderStandardFields(hit, hasFieldsField);
+        fieldsEnd(hasFieldsField);
+    }
+
+    private boolean renderStandardFields(Hit hit, boolean initialHasFieldsField)
+            throws JsonGenerationException, IOException {
+        boolean hasFieldsField = initialHasFieldsField;
+        for (String fieldName : hit.fieldKeys()) {
+            if (!shouldRender(fieldName, hit)) continue;
+
+            // We can't look at the size of fieldKeys() and know whether we need
+            // the fields object, as all fields may be hidden.
+            hasFieldsField |= fieldsStart(hasFieldsField);
+            renderField(fieldName, hit);
+        }
+        return hasFieldsField;
+    }
+
+    private boolean shouldRender(String fieldName, Hit hit) {
+        if (debugRendering) {
+            return true;
+        }
+        if (fieldName.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) {
+            return false;
+        }
+
+        RenderDecision r = lazyRenderAwareCheck(fieldName, hit);
+        if (r != RenderDecision.DO_NOT_KNOW) {
+            return r.booleanValue();
+        }
+
+        // this will trigger field decoding, so it is important the lazy decoding magic is done first
+        Object field = hit.getField(fieldName);
+
+        if (field instanceof CharSequence && ((CharSequence) field).length() == 0) {
+            return false;
+        }
+        if (field instanceof StringFieldValue && ((StringFieldValue) field).getString().isEmpty()) {
+            // StringFieldValue cannot hold a null, so checking length directly is OK
+            return false;
+        }
+        if (field instanceof NanNumber) {
+            return false;
+        }
+
+        return true;
+    }
+
+    private RenderDecision lazyRenderAwareCheck(String fieldName, Hit hit) {
+        if (!(hit instanceof FastHit)) return RenderDecision.DO_NOT_KNOW;
+
+        FastHit asFastHit = (FastHit) hit;
+        if (asFastHit.fieldIsNotDecoded(fieldName)) {
+            FastHit.RawField r = asFastHit.fetchFieldAsUtf8(fieldName);
+            if (r != null) {
+                byte[] utf8 = r.getUtf8();
+                if (utf8.length == 0) {
+                    return RenderDecision.NO;
+                } else {
+                    return RenderDecision.YES;
+                }
+            }
+        }
+        return RenderDecision.DO_NOT_KNOW;
+    }
+
+    private void renderSpecialCasesForGrouping(Hit hit)
+            throws JsonGenerationException, IOException {
+        if (hit instanceof AbstractList) {
+            renderGroupingListSyntheticFields((AbstractList) hit);
+        } else if (hit instanceof Group) {
+            renderGroupingGroupSyntheticFields(hit);
+        }
+    }
+
+    private void renderGroupingGroupSyntheticFields(Hit hit)
+            throws JsonGenerationException, IOException {
+        renderGroupMetadata(((Group) hit).getGroupId());
+        if (hit instanceof RootGroup) {
+            renderContinuations(Collections.singletonMap(
+                    Continuation.THIS_PAGE, ((RootGroup) hit).continuation()));
+        }
+    }
+
+    private void renderGroupingListSyntheticFields(AbstractList a)
+            throws JsonGenerationException, IOException {
+        writeGroupingLabel(a);
+        renderContinuations(a.continuations());
+    }
+
+    private void writeGroupingLabel(AbstractList a)
+            throws JsonGenerationException, IOException {
+        generator.writeStringField(LABEL, a.getLabel());
+    }
+
+    private void renderContinuations(Map<String, Continuation> continuations)
+            throws JsonGenerationException, IOException {
+        if (continuations.isEmpty()) {
+            return;
+        }
+        generator.writeObjectFieldStart(CONTINUATION);
+        for (Map.Entry<String, Continuation> e : continuations.entrySet()) {
+            generator.writeStringField(e.getKey(), e.getValue().toString());
+        }
+        generator.writeEndObject();
+    }
+
+    private void renderGroupMetadata(GroupId id) throws JsonGenerationException,
+            IOException {
+        if (!(id instanceof ValueGroupId || id instanceof BucketGroupId)) {
+            return;
+        }
+
+        if (id instanceof ValueGroupId) {
+            final ValueGroupId<?> valueId = (ValueGroupId<?>) id;
+            generator.writeStringField(GROUPING_VALUE, getIdValue(valueId));
+        } else if (id instanceof BucketGroupId) {
+            final BucketGroupId<?> bucketId = (BucketGroupId<?>) id;
+            generator.writeObjectFieldStart(BUCKET_LIMITS);
+            generator.writeStringField(BUCKET_FROM, getBucketFrom(bucketId));
+            generator.writeStringField(BUCKET_TO, getBucketTo(bucketId));
+            generator.writeEndObject();
+        }
+    }
+
+    private static String getIdValue(ValueGroupId<?> id) {
+        return (id instanceof RawId ? Arrays.toString(((RawId) id).getValue())
+                : id.getValue()).toString();
+    }
+
+    private static String getBucketFrom(BucketGroupId<?> id) {
+        return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id)
+                .getFrom()) : id.getFrom()).toString();
+    }
+
+    private static String getBucketTo(BucketGroupId<?> id) {
+        return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id)
+                .getTo()) : id.getTo()).toString();
+    }
+
+    private boolean renderTotalHitCount(Hit hit, boolean hasFieldsField)
+            throws JsonGenerationException, IOException {
+        if (getRecursionLevel() == 1 && hit instanceof HitGroup) {
+            fieldsStart(hasFieldsField);
+            generator.writeNumberField(TOTAL_COUNT, getResult()
+                    .getTotalHitCount());
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private void renderField(String fieldName, Hit hit) throws JsonGenerationException, IOException {
+        generator.writeFieldName(fieldName);
+        if (!tryDirectRendering(fieldName, hit)) {
+            renderFieldContents(hit.getField(fieldName));
+        }
+    }
+
+    private void renderFieldContents(Object field) throws JsonGenerationException, IOException {
+        if (field == null) {
+            generator.writeNull();
+        } else if (field instanceof Number) {
+            renderNumberField((Number) field);
+        } else if (field instanceof TreeNode) {
+            generator.writeTree((TreeNode) field);
+        } else if (field instanceof JsonProducer) {
+            generator.writeRawValue(((JsonProducer) field).toJson());
+        } else if (field instanceof Inspectable) {
+            StringBuilder intermediate = new StringBuilder();
+            JsonRender.render((Inspectable) field, intermediate, true);
+            generator.writeRawValue(intermediate.toString());
+        } else if (field instanceof StringFieldValue) {
+            // This needs special casing as JsonWriter hides empty strings now
+            generator.writeString(((StringFieldValue) field).getString());
+        } else if (field instanceof FieldValue) {
+            // the null below is the field which has already been written
+            ((FieldValue) field).serialize(null, new JsonWriter(generator));
+        } else if (field instanceof JSONArray || field instanceof JSONObject) {
+            // org.json returns null if the object would not result in
+            // syntactically correct JSON
+            String s = field.toString();
+            if (s == null) {
+                generator.writeNull();
+            } else {
+                generator.writeRawValue(s);
+            }
+        } else {
+            generator.writeString(field.toString());
+        }
+    }
+
+    private void renderNumberField(Number field) throws JsonGenerationException, IOException {
+        if (field instanceof Integer) {
+            generator.writeNumber(field.intValue());
+        }  else if (field instanceof Float) {
+            generator.writeNumber(field.floatValue());
+        }  else if (field instanceof Double) {
+            generator.writeNumber(field.doubleValue());
+        } else if (field instanceof Long) {
+            generator.writeNumber(field.longValue());
+        } else if (field instanceof Byte || field instanceof Short) {
+            generator.writeNumber(field.intValue());
+        } else if (field instanceof BigInteger) {
+            generator.writeNumber((BigInteger) field);
+        } else if (field instanceof BigDecimal) {
+            generator.writeNumber((BigDecimal) field);
+        } else {
+            generator.writeNumber(field.doubleValue());
+        }
+    }
+
+    /**
+     * Really a private method, but package access for testability.
+     */
+    boolean tryDirectRendering(String fieldName, Hit hit)
+            throws IOException, JsonGenerationException {
+        boolean renderedAsUtf8 = false;
+        if (hit instanceof FastHit) {
+            FastHit f = (FastHit) hit;
+            if (f.fieldIsNotDecoded(fieldName)) {
+                FastHit.RawField r = f.fetchFieldAsUtf8(fieldName);
+                if (r != null) {
+                    byte[] utf8 = r.getUtf8();
+
+                    generator.writeUTF8String(utf8, 0, utf8.length);
+                    renderedAsUtf8 = true;
+                }
+            }
+        }
+        return renderedAsUtf8;
+    }
+
+    @Override
+    public void data(Data data) throws IOException {
+        Preconditions.checkArgument(data instanceof Hit,
+                "Expected subclass of com.yahoo.search.result.Hit, got %s.",
+                data.getClass());
+        renderHit((Hit) data);
+    }
+
+    @Override
+    public void endList(DataList<?> list) throws IOException {
+        lessChildren();
+        generator.writeEndObject();
+    }
+
+    @Override
+    public void endResponse() throws IOException {
+        generator.close();
+    }
+
+    @Override
+    public String getEncoding() {
+        return "utf-8";
+    }
+
+    @Override
+    public String getMimeType() {
+        return "application/json";
+    }
+
+    private Result getResult() {
+        Response r = getResponse();
+        Preconditions.checkArgument(r instanceof Result,
+                "JsonRenderer can only render instances of com.yahoo.search.Result, got instance of %s.",
+                r.getClass());
+        return (Result) r;
+    }
+
+    /**
+     * Only for testing. Never to be used in any other context.
+     */
+    void setGenerator(JsonGenerator generator) {
+        this.generator = generator;
+    }
+
+    /**
+     * Only for testing. Never to be used in any other context.
+     */
+    void setTimeSource(LongSupplier timeSource) {
+        this.timeSource = timeSource;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/Renderer.java b/container-search/src/main/java/com/yahoo/search/rendering/Renderer.java
new file mode 100644
index 00000000000..92e3bb15d06
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/rendering/Renderer.java
@@ -0,0 +1,96 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.rendering;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import com.yahoo.io.ByteWriter;
+import com.yahoo.processing.Request;
+import com.yahoo.processing.execution.Execution;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+
+/**
+ * Renders a search result to a writer synchronously - the result is completely rendered when the render method returns..
+ * The renderers are cloned just before rendering,
+ * and must therefore obey the following contract:
+ *
+ * <ol>
+ * <li>At construction time, only final members shall be initialized, and these
+ * must refer to immutable data only.</li>
+ * <li>State mutated during rendering shall be initialized in the init method.</li>
+ * </ol>
+ *
+ * @author tonytv
+ */
+abstract public class Renderer extends com.yahoo.processing.rendering.Renderer<Result> {
+
+    /**
+     * Renders synchronously and returns when rendering is complete.
+     *
+     * @return a future which is always completed to true
+     */
+    @Override
+    public final ListenableFuture<Boolean> render(OutputStream stream, Result response, Execution execution, Request request) {
+        Writer writer = null;
+        try {
+            writer = createWriter(stream,response);
+            render(writer, response);
+        }
+        catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        finally {
+            if (writer !=null)
+                try { writer.close(); } catch (IOException e2) {};
+        }
+        SettableFuture<Boolean> completed=SettableFuture.create();
+        completed.set(true);
+        return completed;
+    }
+
+    /**
+     * Renders the result to the writer.
+     */
+    protected abstract void render(Writer writer, Result result) throws IOException;
+
+    private Writer createWriter(OutputStream stream,Result result) {
+        Charset cs = Charset.forName(getCharacterEncoding(result));
+        CharsetEncoder encoder = cs.newEncoder();
+        return new ByteWriter(stream, encoder);
+    }
+
+    public String getCharacterEncoding(Result result) {
+        String encoding = result.getQuery().getModel().getEncoding();
+        return (encoding != null) ? encoding : getEncoding();
+    }
+
+    /**
+     * @return The summary class to fill the hits with if no summary class was
+     * specified in the query presentation.
+     */
+    public String getDefaultSummaryClass() {
+        return null;
+    }
+
+    /** Returns the encoding of the query, or the encoding given by the template if none is set */
+    public final String getRequestedEncoding(Query query) {
+        String encoding = query.getModel().getEncoding();
+        if (encoding != null) return encoding;
+        return getEncoding();
+    }
+
+    /**
+     * Used to create a separate instance for each result to render.
+     */
+    @Override
+    public Renderer clone() {
+        return (Renderer) super.clone();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java
new file mode 100644
index 00000000000..b60c58fd90f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java
@@ -0,0 +1,103 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.rendering;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.prelude.templates.PageTemplateSet;
+import com.yahoo.prelude.templates.SearchRendererAdaptor;
+import com.yahoo.prelude.templates.TiledTemplateSet;
+import com.yahoo.prelude.templates.UserTemplate;
+import com.yahoo.processing.rendering.Renderer;
+import com.yahoo.search.Result;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Holds all configured and built-in renderers.
+ * This registry is always frozen.
+ *
+ * @author bratseth
+ */
+public final class RendererRegistry extends ComponentRegistry<com.yahoo.processing.rendering.Renderer<Result>> {
+
+    public static final ComponentId xmlRendererId = ComponentId.fromString("DefaultRenderer");
+    public static final ComponentId jsonRendererId = ComponentId.fromString("JsonRenderer");
+    public static final ComponentId defaultRendererId = jsonRendererId;
+
+    /** Creates a registry containing the built-in renderers only */
+    public RendererRegistry() {
+        this(Collections.emptyList());
+    }
+
+    /** Creates a registry of the given renderers plus the built-in ones */
+    public RendererRegistry(Collection<Renderer> renderers) {
+        // add json renderer
+        Renderer jsonRenderer = new JsonRenderer();
+        jsonRenderer.initId(RendererRegistry.jsonRendererId);
+        register(jsonRenderer.getId(), jsonRenderer);
+
+        // Add xml renderer
+        Renderer xmlRenderer = new DefaultRenderer();
+        xmlRenderer.initId(xmlRendererId);
+        register(xmlRenderer.getId(), xmlRenderer);
+
+        // add application renderers
+        for (Renderer renderer : renderers)
+            register(renderer.getId(), renderer);
+
+        // add legacy "templates" converted to renderers
+        addTemplateSet(new TiledTemplateSet());
+        addTemplateSet(new PageTemplateSet());
+
+        freeze();
+    }
+
+    @SuppressWarnings({"deprecation", "unchecked"})
+    private void addTemplateSet(UserTemplate<?> templateSet) {
+        Renderer renderer = new SearchRendererAdaptor(templateSet);
+        ComponentId rendererId = new ComponentId(templateSet.getName());
+        renderer.initId(rendererId);
+        register(rendererId, renderer);
+    }
+
+    /**
+     * Returns the default JSON renderer
+     *
+     * @return the default built-in result renderer
+     */
+    public com.yahoo.processing.rendering.Renderer<Result> getDefaultRenderer() {
+        return getComponent(jsonRendererId);
+    }
+
+    /**
+     * Returns the requested renderer.
+     *
+     * @param format the id or format alias of the renderer to return. If null is passed the default renderer
+     *               is returned
+     * @throws IllegalArgumentException if the renderer cannot be resolved
+     */
+    public com.yahoo.processing.rendering.Renderer<Result> getRenderer(ComponentSpecification format) {
+        if (format == null || format.stringValue().equals("default")) return getDefaultRenderer();
+        if (format.stringValue().equals("json")) return getComponent(jsonRendererId);
+        if (format.stringValue().equals("xml")) return getComponent(xmlRendererId);
+
+        com.yahoo.processing.rendering.Renderer<Result> renderer = getComponent(format);
+        if (renderer == null)
+            throw new IllegalArgumentException("No renderer with id or alias '" + format + "'. " +
+                                               "Available renderers are: [" + rendererNames() + "].");
+        return renderer;
+    }
+
+    private String rendererNames() {
+        StringBuilder r = new StringBuilder();
+        for (Renderer<Result> c : allComponents()) {
+            if (r.length() > 0)
+                r.append(", ");
+            r.append(c.getId().stringValue());
+        }
+        return r.toString();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SectionedRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SectionedRenderer.java
new file mode 100644
index 00000000000..98978b76277
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/rendering/SectionedRenderer.java
@@ -0,0 +1,220 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.rendering;
+
+import com.yahoo.search.Result;
+import com.yahoo.search.grouping.result.Group;
+import com.yahoo.search.grouping.result.GroupList;
+import com.yahoo.search.grouping.result.HitList;
+import com.yahoo.search.query.context.QueryContext;
+import com.yahoo.search.result.ErrorHit;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+
+/**
+ * Renders each part of a result to a writer.
+ * The renderers are cloned just before rendering,
+ * and must therefore obey the following contract:
+ * <ol>
+ *   <li>At construction time, only final members shall be initialized,
+ *       and these must refer to immutable data only.
+ *   <li>State mutated during rendering shall be initialized in the init method.
+ * </ol>
+ *
+ * @author tonytv
+ */
+abstract public class SectionedRenderer<WRITER> extends Renderer {
+    /**
+     * Wraps the Writer instance.
+     * The result is given as a parameter to all the callback methods.
+     * Must be overridden if the generic parameter WRITER != java.io.Writer.
+     */
+    @SuppressWarnings("unchecked")
+    public WRITER wrapWriter(Writer writer) {
+        return (WRITER)writer;
+    }
+
+    /**
+     * Called at the start of rendering.
+     */
+    abstract public void beginResult(WRITER writer, Result result) throws IOException;
+
+    /**
+     * Called at the end of rendering.
+     */
+    abstract public void endResult(WRITER writer, Result result) throws IOException;
+
+    /**
+     * Called if there are errors in the result.
+     */
+    abstract public void error(WRITER writer, Collection<ErrorMessage> errorMessages) throws IOException;
+
+    /**
+     * Called if there are no hits in the result.
+     */
+    abstract public void emptyResult(WRITER writer, Result result) throws IOException;
+
+    /**
+     * Called if there is a non-null query context for the query of the result.
+     */
+    abstract public void queryContext(WRITER writer, QueryContext queryContext) throws IOException;
+
+    /**
+     * Called when a HitGroup is encountered. After all its children have been provided
+     * to methods of this class, endHitGroup is called.
+     */
+    abstract public void beginHitGroup(WRITER writer, HitGroup hitGroup) throws IOException;
+
+    /**
+     * Called after all the children of the HitGroup have been provided to methods of this class.
+     * See beginHitGroup.
+     */
+    abstract public void endHitGroup(WRITER writer, HitGroup hitGroup) throws IOException;
+
+    /**
+     * Called when a Hit is encountered.
+     */
+    abstract public void hit(WRITER writer, Hit hit) throws IOException;
+
+    /**
+     * Called when an errorHit is encountered.
+     * Forwards to hit() per default.
+     */
+    public void errorHit(WRITER writer, ErrorHit errorHit) throws IOException {
+        hit(writer, (Hit)errorHit);
+    }
+
+    /* Begin Grouping */
+
+    /**
+     * Same as beginHitGroup, but for Group(grouping api).
+     * Forwards to beginHitGroup() per default.
+     */
+    public void beginGroup(WRITER writer, Group group) throws IOException {
+        beginHitGroup(writer, group);
+    }
+
+    /**
+     * Same as endHitGroup, but for Group(grouping api).
+     * Forwards to endHitGroup() per default.
+     */
+    public void endGroup(WRITER writer, Group group) throws IOException {
+        endHitGroup(writer, group);
+    }
+
+    /**
+     * Same as beginHitGroup, but for GroupList(grouping api).
+     * Forwards to beginHitGroup() per default.
+     */
+    public void beginGroupList(WRITER writer, GroupList groupList) throws IOException {
+        beginHitGroup(writer, groupList);
+    }
+
+    /**
+     * Same as endHitGroup, but for GroupList(grouping api).
+     * Forwards to endHitGroup() per default.
+     */
+    public void endGroupList(WRITER writer, GroupList groupList) throws IOException {
+        endHitGroup(writer, groupList);
+    }
+
+    /**
+     * Same as beginHitGroup, but for HitList(grouping api).
+     * Forwards to beginHitGroup() per default.
+     */
+    public void beginHitList(WRITER writer, HitList hitList) throws IOException {
+        beginHitGroup(writer, hitList);
+    }
+
+    /**
+     * Same as endHitGroup, but for HitList(grouping api).
+     * Forwards to endHitGroup() per default.
+     */
+    public void endHitList(WRITER writer, HitList hitList) throws IOException {
+        endHitGroup(writer, hitList);
+    }
+    /* End Grouping */
+
+    /**
+     * Picks apart the result and feeds it to the other methods.
+     */
+    @Override
+    public final void render(Writer writer, Result result) throws IOException {
+        WRITER wrappedWriter = wrapWriter(writer);
+
+        beginResult(wrappedWriter, result);
+        renderResultContent(wrappedWriter, result);
+        endResult(wrappedWriter, result);
+    }
+
+    private  void renderResultContent(WRITER writer, Result result) throws IOException {
+        if (result.hits().getError() != null || result.hits().getQuery().errors().size() > 0) {
+            error(writer, asUnmodifiableSearchErrorList(result.hits().getQuery().errors(), result.hits().getError()));
+        }
+
+        if (result.getConcreteHitCount() == 0) {
+            emptyResult(writer, result);
+        }
+
+        if (result.getContext(false) != null) {
+            queryContext(writer, result.getContext(false));
+        }
+
+        renderHitGroup(writer, result.hits());
+    }
+
+    private Collection<ErrorMessage> asUnmodifiableSearchErrorList(List<com.yahoo.processing.request.ErrorMessage> queryErrors,ErrorMessage resultError) {
+        if (queryErrors.size() == 0)
+            return Collections.singletonList(resultError);
+        List<ErrorMessage> searchErrors = new ArrayList<>(queryErrors.size() + (resultError != null ? 1 :0) );
+        for (int i=0; i<queryErrors.size(); i++)
+            searchErrors.add(ErrorMessage.from(queryErrors.get(i)));
+        if (resultError != null)
+            searchErrors.add(resultError);
+        return Collections.unmodifiableCollection(searchErrors);
+    }
+
+    private void renderHitGroup(WRITER writer, HitGroup hitGroup) throws IOException {
+        if (hitGroup instanceof GroupList) {
+            beginGroupList(writer, (GroupList) hitGroup);
+            renderHitGroupContent(writer, hitGroup);
+            endGroupList(writer, (GroupList) hitGroup);
+        } else if (hitGroup instanceof HitList) {
+            beginHitList(writer, (HitList) hitGroup);
+            renderHitGroupContent(writer, hitGroup);
+            endHitList(writer, (HitList) hitGroup);
+        } else if (hitGroup instanceof Group) {
+            beginGroup(writer, (Group) hitGroup);
+            renderHitGroupContent(writer, hitGroup);
+            endGroup(writer, (Group) hitGroup);
+        } else {
+            beginHitGroup(writer, hitGroup);
+            renderHitGroupContent(writer, hitGroup);
+            endHitGroup(writer, hitGroup);
+        }
+    }
+
+    private void renderHitGroupContent(WRITER writer, HitGroup hitGroup) throws IOException {
+        for (Hit hit : hitGroup.asList()) {
+            renderHit(writer, hit);
+        }
+    }
+
+    private void renderHit(WRITER writer, Hit hit) throws IOException {
+        if (hit instanceof  HitGroup) {
+            renderHitGroup(writer, (HitGroup) hit);
+        } else if (hit instanceof ErrorHit) {
+            errorHit(writer, (ErrorHit) hit);
+        } else {
+            hit(writer, hit);
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java
new file mode 100644
index 00000000000..d3039925013
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java
@@ -0,0 +1,471 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.rendering;
+
+import com.yahoo.concurrent.CopyOnWriteHashMap;
+import com.yahoo.io.ByteWriter;
+import com.yahoo.log.LogLevel;
+import com.yahoo.net.URI;
+import com.yahoo.prelude.fastsearch.FastHit;
+import com.yahoo.prelude.fastsearch.GroupingListHit;
+import com.yahoo.prelude.templates.Context;
+import com.yahoo.prelude.templates.DefaultTemplateSet;
+import com.yahoo.prelude.templates.MapContext;
+import com.yahoo.prelude.templates.UserTemplate;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.grouping.result.HitRenderer;
+import com.yahoo.search.query.context.QueryContext;
+import com.yahoo.search.result.*;
+import com.yahoo.text.Utf8String;
+import com.yahoo.text.XMLWriter;
+import com.yahoo.yolean.trace.TraceNode;
+import com.yahoo.yolean.trace.TraceVisitor;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * @author tonytv
+ */
+@SuppressWarnings({ "rawtypes", "deprecation" })
+public final class SyncDefaultRenderer extends Renderer {
+
+    private static final Logger log = Logger.getLogger(SyncDefaultRenderer.class.getName());
+
+    public static final String  DEFAULT_MIMETYPE    = "text/xml";
+    public static final String  DEFAULT_ENCODING    = "utf-8";
+
+
+    private static final Utf8String RESULT = new Utf8String("result");
+    private static final Utf8String GROUP = new Utf8String("group");
+    private static final Utf8String ID = new Utf8String("id");
+    private static final Utf8String FIELD = new Utf8String("field");
+    private static final Utf8String HIT = new Utf8String("hit");
+    private static final Utf8String ERROR = new Utf8String("error");
+    private static final Utf8String TOTAL_HIT_COUNT = new Utf8String("total-hit-count");
+    private static final Utf8String QUERY_TIME = new Utf8String("querytime");
+    private static final Utf8String SUMMARY_FETCH_TIME = new Utf8String("summaryfetchtime");
+    private static final Utf8String SEARCH_TIME = new Utf8String("searchtime");
+    private static final Utf8String NAME = new Utf8String("name");
+    private static final Utf8String CODE = new Utf8String("code");
+    private static final Utf8String COVERAGE_DOCS = new Utf8String("coverage-docs");
+    private static final Utf8String COVERAGE_NODES = new Utf8String("coverage-nodes");
+    private static final Utf8String COVERAGE_FULL = new Utf8String("coverage-full");
+    private static final Utf8String COVERAGE = new Utf8String("coverage");
+    private static final Utf8String RESULTS_FULL = new Utf8String("results-full");
+    private static final Utf8String RESULTS = new Utf8String("results");
+    private static final Utf8String TYPE = new Utf8String("type");
+    private static final Utf8String RELEVANCY = new Utf8String("relevancy");
+    private static final Utf8String SOURCE = new Utf8String("source");
+
+
+    //Per instance members, must be created at rendering time, not construction time due to cloning.
+    private Context context;
+
+    private final DefaultTemplateSet defaultTemplate = new DefaultTemplateSet();
+
+    private final CopyOnWriteHashMap<String, Utf8String> fieldNameMap = new CopyOnWriteHashMap<>();
+
+    @Override
+    public void init() {
+        super.init();
+        context = new MapContext();
+    }
+
+    @Override
+    public String getEncoding() {
+        return DEFAULT_ENCODING;
+    }
+
+    @Override
+    public String getMimeType() {
+        return DEFAULT_MIMETYPE;
+    }
+
+    @Override
+    public String getDefaultSummaryClass() {
+        return null;
+    }
+
+    private XMLWriter wrapWriter(Writer writer) {
+        return XMLWriter.from(writer, 10, -1);
+    }
+
+    /**
+     * Renders this result
+     */
+    public void render(Writer writer, Result result) throws IOException {
+        XMLWriter xmlWriter = wrapWriter(writer);
+
+        context.put("context", context);
+        context.put("result", result);
+        context.setBoldOpenTag(defaultTemplate.getBoldOpenTag());
+        context.setBoldCloseTag(defaultTemplate.getBoldCloseTag());
+        context.setSeparatorTag(defaultTemplate.getSeparatorTag());
+
+        try {
+            header(xmlWriter, result);
+        } catch (Exception e) {
+            handleException(e);
+        }
+
+        if (result.hits().getError() != null || result.hits().getQuery().errors().size() > 0) {
+            error(xmlWriter, result);
+        }
+
+        if (result.getConcreteHitCount() == 0) {
+            emptyResult(xmlWriter, result);
+        }
+
+        if (result.getContext(false) != null) {
+            queryContext(xmlWriter, result.getContext(false), result.getQuery());
+        }
+
+        renderHitGroup(xmlWriter, result.hits(), result.hits().getQuery().getOffset() + 1);
+
+        endResult(xmlWriter, result);
+    }
+
+    private void header(XMLWriter writer, Result result) throws IOException {
+        // TODO: move setting this to Result
+        context.setUtf8Output("utf-8".equalsIgnoreCase(getRequestedEncoding(result.getQuery())));
+        writer.xmlHeader(getRequestedEncoding(result.getQuery()));
+        writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT,String.valueOf(result.getTotalHitCount()));
+        if (result.getQuery().getPresentation().getReportCoverage()) {
+            renderCoverageAttributes(result.getCoverage(false), writer);
+        }
+        renderTime(writer, result);
+        writer.closeStartTag();
+    }
+
+    private void renderTime(XMLWriter writer, Result result) {
+        if (!result.getQuery().getPresentation().getTiming()) {
+            return;
+        }
+
+        final String threeDecimals = "%.3f";
+        final double milli = .001d;
+        final long now = System.currentTimeMillis();
+        final long searchTime = now - result.getQuery().getStartTime();
+        final double searchSeconds = ((double) searchTime) * milli;
+
+        if (result.getElapsedTime().firstFill() != 0L) {
+            final long queryTime = result.getElapsedTime().firstFill() - result.getQuery().getStartTime();
+            final long summaryFetchTime = now - result.getElapsedTime().firstFill();
+            final double querySeconds = ((double) queryTime) * milli;
+            final double summarySeconds = ((double) summaryFetchTime) * milli;
+            writer.attribute(QUERY_TIME, String.format(threeDecimals, querySeconds));
+            writer.attribute(SUMMARY_FETCH_TIME, String.format(threeDecimals, summarySeconds));
+        }
+        writer.attribute(SEARCH_TIME, String.format(threeDecimals, searchSeconds));
+    }
+
+    protected static void renderCoverageAttributes(Coverage coverage, XMLWriter writer) throws IOException {
+        if (coverage == null) return;
+        writer.attribute(COVERAGE_DOCS,coverage.getDocs());
+        writer.attribute(COVERAGE_NODES,coverage.getNodes());
+        writer.attribute(COVERAGE_FULL,coverage.getFull());
+        writer.attribute(COVERAGE,coverage.getResultPercentage());
+        writer.attribute(RESULTS_FULL,coverage.getFullResultSets());
+        writer.attribute(RESULTS,coverage.getResultSets());
+    }
+
+    public void endResult(XMLWriter writer, Result result) throws IOException {
+        try {
+            writer.closeTag();
+        } catch (Exception e) {
+            handleException(e);
+        }
+    }
+
+    public void error(XMLWriter writer, Result result) throws IOException {
+        try {
+            ErrorMessage error = result.hits().getError();
+            writer.openTag(ERROR).attribute(CODE,error.getCode()).content(error.getMessage(),false).closeTag();
+        } catch (Exception e) {
+            handleException(e);
+        }
+    }
+
+
+    protected void emptyResult(XMLWriter writer, Result result) throws IOException {}
+
+    public void queryContext(XMLWriter writer, QueryContext queryContext, Query owner) throws IOException {
+        try {
+            if (owner.getTraceLevel()!=0) {
+                XMLWriter xmlWriter=XMLWriter.from(writer);
+                xmlWriter.openTag("meta").attribute("type", QueryContext.ID);
+                TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root();
+                traceRoot.accept(new RenderingVisitor(xmlWriter, owner.getStartTime()));
+                xmlWriter.closeTag();
+            }
+        } catch (Exception e) {
+            handleException(e);
+        }
+    }
+
+    private  void renderHitGroup(XMLWriter writer, HitGroup hitGroup, int hitnumber)
+            throws IOException {
+        for (Hit hit : hitGroup.asList()) {
+            renderHit(writer, hit, hitnumber);
+            if (!hit.isAuxiliary())
+                hitnumber++;
+        }
+    }
+
+
+    /**
+     * Renders this hit as xml. The default implementation will call the simpleRender()
+     * hook. If it returns true, nothing more is done, otherwise the
+     * given template set will be used for rendering.
+     *
+     *
+     * @param writer      the XMLWriter to append this hit to
+     * @throws java.io.IOException if rendering fails
+     */
+    public void renderHit(XMLWriter writer, Hit hit, int hitno) throws IOException {
+        renderRegularHit(writer, hit, hitno);
+    }
+
+    private void renderRegularHit(XMLWriter writer, Hit hit, int hitno) throws IOException {
+        boolean renderedSimple = simpleRenderHit(writer, hit);
+
+        if (renderedSimple) {
+            return;
+        }
+
+        try {
+            if (hit instanceof HitGroup) {
+                renderHitGroup(writer, (HitGroup) hit);
+            } else {
+                renderSingularHit(writer, hit);
+            }
+        } catch (Exception e) {
+            handleException(e);
+        }
+
+        if (hit instanceof HitGroup)
+            renderHitGroup(writer, (HitGroup) hit, hitno);
+
+        try {
+            writer.closeTag();
+        } catch (Exception e) {
+            handleException(e);
+        }
+    }
+
+    private void renderSingularHit(XMLWriter writer, Hit hit) throws IOException {
+        writer.openTag(HIT);
+        renderHitAttributes(writer, hit);
+        writer.closeStartTag();
+        renderHitFields(writer, hit);
+    }
+
+    private void renderHitFields(XMLWriter writer, Hit hit) throws IOException {
+        renderSyntheticRelevanceField(writer, hit);
+        for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) {
+            renderField(writer, hit, it);
+        }
+    }
+
+    private void renderField(XMLWriter writer, Hit hit, Iterator<Map.Entry<String, Object>> it) throws IOException {
+        Map.Entry<String, Object> entry = it.next();
+        boolean isProbablyNotDecoded = false;
+        if (hit instanceof FastHit) {
+            FastHit f = (FastHit) hit;
+            isProbablyNotDecoded = f.fieldIsNotDecoded(entry.getKey());
+        }
+        renderGenericFieldPossiblyNotDecoded(writer, hit, entry, isProbablyNotDecoded);
+    }
+
+    private void renderGenericFieldPossiblyNotDecoded(XMLWriter writer, Hit hit, Map.Entry<String, Object> entry, boolean probablyNotDecoded) throws IOException {
+        String fieldName = entry.getKey();
+
+        if (!shouldRenderField(hit, fieldName)) return;
+        if (fieldName.startsWith("$")) return; // Don't render fields that start with $ // TODO: Move to should render
+
+        writeOpenFieldElement(writer, fieldName);
+        renderFieldContentPossiblyNotDecoded(writer, hit, probablyNotDecoded, fieldName);
+        writeCloseFieldElement(writer);
+    }
+
+    private void renderFieldContentPossiblyNotDecoded(XMLWriter writer, Hit hit, boolean probablyNotDecoded, String fieldName) throws IOException {
+        boolean dumpedRaw = false;
+        if (probablyNotDecoded && (hit instanceof FastHit)) {
+            writer.closeStartTag();
+            if ((writer.getWriter() instanceof ByteWriter) && context.isUtf8Output()) {
+                dumpedRaw = UserTemplate.dumpBytes((ByteWriter) writer.getWriter(), (FastHit) hit, fieldName);
+            }
+            if (dumpedRaw) {
+                writer.content("", false); // let the xml writer note that this tag had content
+            }
+        }
+        if (!dumpedRaw) {
+            String xmlval = hit.getFieldXML(fieldName);
+            if (xmlval == null) {
+                xmlval = "(null)";
+            }
+            writer.escapedContent(xmlval, false);
+        }
+    }
+
+    private void renderSyntheticRelevanceField(XMLWriter writer, Hit hit) throws IOException {
+        final String relevancyFieldName = "relevancy";
+        final Relevance relevance = hit.getRelevance();
+
+        if (shouldRenderField(hit, relevancyFieldName) && relevance != null) {
+            renderSimpleField(writer, relevancyFieldName, relevance);
+        }
+    }
+
+    private void renderSimpleField(XMLWriter writer, String relevancyFieldName, Relevance relevance) throws IOException {
+        writeOpenFieldElement(writer, relevancyFieldName);
+        writer.content(relevance.toString(), false);
+        writeCloseFieldElement(writer);
+    }
+
+    private void writeCloseFieldElement(XMLWriter writer) throws IOException {
+        writer.closeTag();
+    }
+
+    private void writeOpenFieldElement(XMLWriter writer, String relevancyFieldName) throws IOException {
+        Utf8String utf8 = fieldNameMap.get(relevancyFieldName);
+        if (utf8 == null) {
+            utf8 = new Utf8String(relevancyFieldName);
+            fieldNameMap.put(relevancyFieldName, utf8);
+        }
+        writer.openTag(FIELD).attribute(NAME, utf8);
+        writer.closeStartTag();
+    }
+
+    private boolean shouldRenderField(Hit hit, String relevancyFieldName) {
+        // skip depending on hit type
+        return true;
+    }
+
+    private void renderHitAttributes(XMLWriter writer, Hit hit) throws IOException {
+        writer.attribute(TYPE, hit.getTypeString());
+        if (hit.getRelevance() != null) {
+            writer.attribute(RELEVANCY, hit.getRelevance().toString());
+}
+        writer.attribute(SOURCE, hit.getSource());
+    }
+
+    private void renderHitGroup(XMLWriter writer, HitGroup hit) throws IOException {
+        if (HitRenderer.renderHeader((HitGroup) hit, writer)) {
+            // empty
+        } else if (((HitGroup) hit).types().contains("grouphit")) {
+            // TODO Keep this?
+            renderHitGroupOfTypeGroupHit(writer, hit);
+        } else {
+            renderGroup(writer, hit);
+        }
+    }
+
+    private void renderGroup(XMLWriter writer, HitGroup hit) throws IOException {
+        writer.openTag(GROUP);
+        renderHitAttributes(writer, (HitGroup) hit);
+        writer.closeStartTag();
+    }
+
+    private void renderHitGroupOfTypeGroupHit(XMLWriter writer, HitGroup hit) throws IOException {
+        writer.openTag(HIT);
+        renderHitAttributes(writer, (HitGroup) hit);
+        renderId(writer, hit);
+        writer.closeStartTag();
+    }
+
+    private void renderId(XMLWriter writer, HitGroup hit) throws IOException {
+        URI uri = hit.getId();
+        if (uri != null) {
+            writer.openTag(ID).content(uri.stringValue(),false).closeTag();
+        }
+    }
+
+    private boolean simpleRenderHit(XMLWriter writer, Hit hit) throws IOException {
+        if (hit instanceof DefaultErrorHit) {
+            return simpleRenderDefaultErrorHit(writer, (DefaultErrorHit) hit);
+        } else if (hit instanceof GroupingListHit) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public static boolean simpleRenderDefaultErrorHit(XMLWriter writer, ErrorHit defaultErrorHit) throws IOException {
+        writer.openTag("errordetails");
+        for (Iterator i = defaultErrorHit.errorIterator(); i.hasNext();) {
+            ErrorMessage error = (ErrorMessage) i.next();
+            renderMessageDefaultErrorHit(writer, error);
+        }
+        writer.closeTag();
+        return true;
+    }
+
+    public static void renderMessageDefaultErrorHit(XMLWriter writer, ErrorMessage error) throws IOException {
+        writer.openTag("error");
+        writer.attribute("source", error.getSource());
+        writer.attribute("error", error.getMessage());
+        writer.attribute("code", Integer.toString(error.getCode()));
+        writer.content(error.getDetailedMessage(), false);
+        if (error.getCause()!=null) {
+            writer.openTag("cause");
+            writer.content("\n", true);
+            StringWriter stackTrace=new StringWriter();
+            error.getCause().printStackTrace(new PrintWriter(stackTrace));
+            writer.content(stackTrace.toString(), true);
+            writer.closeTag();
+        }
+        writer.closeTag();
+    }
+
+    private void handleException(Exception e) throws IOException {
+        if (e instanceof IOException) {
+            throw (IOException) e;
+        } else {
+            log.log(LogLevel.WARNING, "Exception thrown when rendering the result:", e);
+        }
+    }
+
+    public static final class RenderingVisitor extends TraceVisitor {
+
+        private static final String tag = "p";
+        private final XMLWriter writer;
+        private long baseTime;
+
+        public RenderingVisitor(XMLWriter writer,long baseTime) {
+            this.writer=writer;
+            this.baseTime=baseTime;
+        }
+
+        @Override
+        public void entering(TraceNode node) {
+            if (node.isRoot()) return;
+            writer.openTag(tag);
+        }
+
+        @Override
+        public void leaving(TraceNode node) {
+            if (node.isRoot()) return;
+            writer.closeTag();
+        }
+
+        @Override
+        public void visit(TraceNode node) {
+            if (node.isRoot()) return;
+            if (node.payload()==null) return;
+
+            writer.openTag(tag);
+            if (node.timestamp()!=0)
+                writer.content(node.timestamp()-baseTime,false).content(" ms: ", false);
+            writer.content(node.payload().toString(),false);
+            writer.closeTag();
+        }
+
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/package-info.java b/container-search/src/main/java/com/yahoo/search/rendering/package-info.java
new file mode 100644
index 00000000000..7411055e015
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/rendering/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.rendering;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/result/ChainableComparator.java b/container-search/src/main/java/com/yahoo/search/result/ChainableComparator.java
new file mode 100644
index 00000000000..0750618de67
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/ChainableComparator.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.Comparator;
+
+/**
+ * Superclass of hit comparators which delegates comparisons of hits which are
+ * equal according to this comparator, to a secondary comparator.
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public abstract class ChainableComparator implements Comparator<Hit> {
+
+    private final Comparator<Hit> secondaryComparator;
+
+    /** Creates this comparator, given a secondary comparator, or null if there is no secondary */
+    public ChainableComparator(Comparator<Hit> secondaryComparator) {
+        this.secondaryComparator=secondaryComparator;
+    }
+
+    /** Returns the comparator to use to compare hits which are equal according to this, or null if none */
+    public Comparator<Hit> getSecondaryComparator() { return secondaryComparator; }
+
+    /**
+     * Returns the comparison form the secondary comparison, or 0 if the secondary is null.
+     * When overriding this in the subclass, always <code>return super.compare(first,second)</code>
+     * at the end of the subclass' implementation.
+     */
+    public int compare(Hit first,Hit second) {
+        if (secondaryComparator==null) return 0;
+        return secondaryComparator.compare(first,second);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java
new file mode 100644
index 00000000000..7d1e737bfb8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java
@@ -0,0 +1,24 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+/**
+ * The coverage report for a result set.
+ *
+ * @author  <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author balder
+ */
+public class Coverage extends com.yahoo.container.handler.Coverage {
+
+    public Coverage(long docs, long active) {
+        super(docs, active, 0, 1);
+    }
+
+    public Coverage(long docs, int nodes, boolean full) {
+        this(docs, nodes, full, 1);
+    }
+
+    public Coverage(long docs, int nodes, boolean full, int resultSets) {
+        super(docs, nodes, full, resultSets);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/DeepHitIterator.java b/container-search/src/main/java/com/yahoo/search/result/DeepHitIterator.java
new file mode 100644
index 00000000000..a62a9c66e79
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/DeepHitIterator.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.*;
+
+/**
+ * An iterator for the forest of hits in a result.
+ *
+ * @author havardpe
+ */
+public class DeepHitIterator implements Iterator<Hit> {
+
+    private final boolean ordered;
+    private List<Iterator<Hit>> stack;
+    private boolean canRemove = false;
+    private Iterator<Hit> it = null;
+    private Hit next = null;
+
+
+    /**
+     * Create a deep hit iterator based on the given hit iterator.
+     *
+     * @param it      The hits iterator to traverse.
+     * @param ordered Whether or not the hits should be ordered.
+     */
+    public DeepHitIterator(Iterator<Hit> it, boolean ordered) {
+        this.ordered = ordered;
+        this.it = it;
+    }
+
+    @Override
+    public boolean hasNext() {
+        canRemove = false;
+        return getNext();
+    }
+
+    @Override
+    public Hit next() throws NoSuchElementException {
+        if (next == null && !getNext()) {
+            throw new NoSuchElementException();
+        }
+        Hit ret = next;
+        next = null;
+        canRemove = true;
+        return ret;
+    }
+
+    @Override
+    public void remove() throws UnsupportedOperationException, IllegalStateException {
+        if (!canRemove) {
+            throw new IllegalStateException("Can not remove() an element after calling hasNext().");
+        }
+        it.remove();
+    }
+
+    private boolean getNext() {
+        if (next != null) {
+            return true;
+        }
+
+        if (stack == null) {
+            stack = new ArrayList<>();
+        }
+        while (true) {
+            if (it.hasNext()) {
+                Hit hit = it.next();
+                if (hit instanceof HitGroup) {
+                    stack.add(it);
+                    if (ordered) {
+                        it = ((HitGroup)hit).iterator();
+                    } else {
+                        it = ((HitGroup)hit).unorderedIterator();
+                    }
+                } else {
+                    next = hit;
+                    return true;
+                }
+            } else if (!stack.isEmpty()) {
+                it = stack.remove(stack.size()-1);
+            } else {
+                return false;
+            }
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java
new file mode 100644
index 00000000000..79b8d55bb07
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.collections.ArraySet;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A hit which holds information on error conditions in a result.
+ * En error hit maintains a main error - the main error of the result.
+ *
+ * @author bratseth
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class DefaultErrorHit extends Hit implements ErrorHit, Cloneable {
+
+    /**
+     * A list of unique error messages, where the first is considered the "main"
+     * error. It should always contain at least one error.
+     */
+    private List<ErrorMessage> errors = new ArrayList<>();
+
+    /**
+     * Creates an error hit with a main error
+     *
+     * @param source the name of the source or backend of this hit
+     * @param error an initial main error to add to this hit, cannot be null
+     */
+    public DefaultErrorHit(String source, ErrorMessage error) {
+        super("error:" + source, new Relevance(Double.POSITIVE_INFINITY), source);
+        addError(error);
+    }
+
+    public void setSource(String source) {
+        super.setSource(source);
+        for (Iterator<ErrorMessage> i = errorIterator(); i.hasNext();) {
+            ErrorMessage error = i.next();
+
+            if (error.getSource() == null) {
+                error.setSource(source);
+            }
+        }
+    }
+
+    /**
+     * Returns the main error of this result, never null.
+     *
+     * @deprecated since 5.18, use {@link #errors()}
+     */
+    @Override
+    public ErrorMessage getMainError() {
+        return errors.get(0);
+    }
+
+    /**
+     * Insert the new "main" error at head of list, remove from the list if it
+     * already exists elsewhere.
+     */
+    private void removeAndAddAtHead(ErrorMessage mainError) {
+        errors.remove(mainError); // avoid error duplication
+        errors.add(0, mainError);
+    }
+
+    /**
+     * This is basically a way of making a list simulate a set.
+     */
+    private void removeAndAdd(ErrorMessage error) {
+        errors.remove(error);
+        errors.add(error);
+    }
+
+    /**
+     * Adds an error to this. This may change the main error
+     * and/or the list of detailed errors
+     */
+    public void addError(ErrorMessage error) {
+        if (error.getSource() == null) {
+            error.setSource(getSource());
+        }
+        removeAndAdd(error);
+    }
+
+
+    /** Add all errors from another error hit to this */
+    public void addErrors(ErrorHit errorHit) {
+        for (Iterator<? extends ErrorMessage> i = errorHit.errorIterator(); i.hasNext();) {
+            addError(i.next());
+        }
+    }
+
+    /**
+     * Returns all the detail errors of this error hit, not including the main error.
+     * The iterator is modifiable.
+     */
+    public Iterator<ErrorMessage> errorIterator() {
+        return errors.iterator();
+    }
+
+    /** Returns a read-only set containing all the error of this */
+    public Set<ErrorMessage> errors() {
+        Set<ErrorMessage> s = new ArraySet<>(errors.size());
+        s.addAll(errors);
+        return s;
+    }
+
+    public String toString() {
+        return "Error: " + errors.get(0).toString();
+    }
+
+    /** Returns true - this is a meta hit containing information on other hits */
+    public boolean isMeta() {
+        return true;
+    }
+
+    /**
+     * Returns true if all errors in this have the given code
+     */
+    public boolean hasOnlyErrorCode(int code) {
+        for (ErrorMessage error : errors) {
+            if (error.getCode() != code)
+                return false;
+        }
+        return true;
+    }
+
+    public DefaultErrorHit clone() {
+        DefaultErrorHit clone = (DefaultErrorHit) super.clone();
+
+        clone.errors = new ArrayList<>(this.errors);
+        return clone;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java
new file mode 100644
index 00000000000..a3b79d98e65
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java
@@ -0,0 +1,47 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * A hit which holds information on error conditions in a result.
+ * En error hit maintains a main error - the main error of the result.
+ *
+ * @author bratseth
+ */
+public interface ErrorHit extends Cloneable {
+
+    void setSource(String source);
+
+    /** Returns the main error of this result, never null */
+    @Deprecated // use: errors().iterator().next()
+    ErrorMessage getMainError();
+
+    /**
+     * Adds an error to this. This may change the main error
+     * and/or the list of detailed errors
+     */
+    void addError(ErrorMessage error);
+
+    /** Add all errors from another error hit to this */
+    void addErrors(ErrorHit errorHit);
+
+    /**
+     * Returns all the detail errors of this error hit, including the main error
+     */
+    Iterator<? extends ErrorMessage> errorIterator();
+
+    /** Returns a read-only set containing all the error of this, including the main error */
+    Set<ErrorMessage> errors();
+
+    /** Returns true - this is a meta hit containing information on other hits */
+    boolean isMeta();
+
+    /** Returns true if main error is the given error code or if main error
+        is general error 8 and all suberrors are the given error code */
+    boolean hasOnlyErrorCode(int code);
+
+    Object clone();
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorMessage.java b/container-search/src/main/java/com/yahoo/search/result/ErrorMessage.java
new file mode 100644
index 00000000000..0a0ef731836
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/ErrorMessage.java
@@ -0,0 +1,210 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.container.protect.Error;
+
+import static com.yahoo.container.protect.Error.*;
+
+
+/**
+ * An error message with a code. Use create methods to create messages.
+ * The identity of an error message is determined by its values.
+ *
+ * @author bratseth
+ */
+public class ErrorMessage extends com.yahoo.processing.request.ErrorMessage {
+
+    public static final int NULL_QUERY = Error.NULL_QUERY.code;
+
+    /** The source producing this error, not always set */
+    private String source = null;
+
+    public ErrorMessage(int code, String message) {
+        super(code,message);
+    }
+
+    /**
+     * Creates an application specific error message with an application specific code.
+     * If the error results from an exception a message which includes information from all nested (cause) exceptions
+     * can be generated using com.yahoo.protect.Exceptions.toMessageString(exception).
+     */
+    public ErrorMessage(int code, String message, String detailedMessage) {
+        super(code,message, detailedMessage);
+    }
+
+    /**
+     * Creates an application specific error message with an application specific code and a stack trace.
+     * This should only be used when there is useful information in the cause, i.e when the exception
+     * is not expected. Applications rarely need to handle unexpected exceptions as this is done by the framework.
+     */
+    public ErrorMessage(int code, String message, String detailedMessage, Throwable cause) {
+        super(code, message, detailedMessage, cause);
+    }
+
+    /** Creates an error message indicating that some backend service is unreachable */
+    public static ErrorMessage createNoBackendsInService(String detailedMessage) {
+        return new ErrorMessage(NO_BACKENDS_IN_SERVICE.code, "No backends in service. Try later", detailedMessage);
+    }
+
+    /** Creates an error message indicating that a null query was attempted evaluated */
+    public static ErrorMessage createNullQuery(String detailedMessage) {
+        return new ErrorMessage(NULL_QUERY, "Null query", detailedMessage);
+    }
+
+    /** Creates an error message indicating that the request is too large */
+    public static ErrorMessage createRequestTooLarge(String detailedMessage) {
+        return new ErrorMessage(REQUEST_TOO_LARGE.code, "Request too large", detailedMessage);
+    }
+
+    /** Creates an error message indicating that an illegal query was attempted evaluated. */
+    public static ErrorMessage createIllegalQuery(String detailedMessage) {
+        return new ErrorMessage(ILLEGAL_QUERY.code, "Illegal query", detailedMessage);
+    }
+
+    /** Creates an error message indicating that an invalid request parameter was received. */
+    public static ErrorMessage createInvalidQueryParameter(String detailedMessage) {
+        return new ErrorMessage(INVALID_QUERY_PARAMETER.code, "Invalid query parameter", detailedMessage);
+    }
+
+    /** Creates an error message indicating that an invalid request parameter was received. */
+    public static ErrorMessage createInvalidQueryParameter(String detailedMessage, Throwable cause) {
+        return new ErrorMessage(INVALID_QUERY_PARAMETER.code, "Invalid query parameter", detailedMessage, cause);
+    }
+
+    /** Creates a generic message used when there is no information available on the category of the error. */
+    public static ErrorMessage createUnspecifiedError(String detailedMessage) {
+        return new ErrorMessage(UNSPECIFIED.code, "Unspecified error", detailedMessage);
+    }
+
+    /** Creates a generic message used when there is no information available on the category of the error. */
+    public static ErrorMessage createUnspecifiedError(String detailedMessage, Throwable cause) {
+        return new ErrorMessage(UNSPECIFIED.code, "Unspecified error", detailedMessage, cause);
+    }
+
+    /** Creates a general error from an application components. */
+    public static ErrorMessage createErrorInPluginSearcher(String detailedMessage) {
+        return new ErrorMessage(ERROR_IN_PLUGIN.code, "Error in plugin Searcher", detailedMessage);
+    }
+
+    /** Creates a general error from an application component. */
+    public static ErrorMessage createErrorInPluginSearcher(String detailedMessage, Throwable cause) {
+        return new ErrorMessage(ERROR_IN_PLUGIN.code, "Error in plugin Searcher", detailedMessage, cause);
+    }
+
+    /** Creates an error indicating that an invalid query transformation was attempted. */
+    public static ErrorMessage createInvalidQueryTransformation(String detailedMessage) {
+        return new ErrorMessage(INVALID_QUERY_TRANSFORMATION.code, "Invalid query transformation",detailedMessage);
+    }
+
+    /** Creates an error indicating that the server is misconfigured */
+    public static ErrorMessage createServerIsMisconfigured(String detailedMessage) {
+        return new ErrorMessage(SERVER_IS_MISCONFIGURED.code, "Service is misconfigured", detailedMessage);
+    }
+
+    /** Creates an error indicating that there was a general error communicating with a backend service. */
+    public static ErrorMessage createBackendCommunicationError(String detailedMessage) {
+        return new ErrorMessage(BACKEND_COMMUNICATION_ERROR.code, "Backend communication error", detailedMessage);
+    }
+
+    /** Creates an error indicating that a node could not be pinged. */
+    public static ErrorMessage createNoAnswerWhenPingingNode(String detailedMessage) {
+        return new ErrorMessage(NO_ANSWER_WHEN_PINGING_NODE.code, "No answer when pinging node", detailedMessage);
+    }
+
+    public static final int timeoutCode = Error.TIMEOUT.code;
+    /** Creates an error indicating that a request to a backend timed out. */
+    public static ErrorMessage createTimeout(String detailedMessage) {
+        return new ErrorMessage(timeoutCode, "Timed out",detailedMessage);
+    }
+
+    public static final int emptyDocsumsCode = Error.EMPTY_DOCUMENTS.code;
+    /** Creates an error indicating that a request to a backend returned empty document content data. */
+    public static ErrorMessage createEmptyDocsums(String detailedMessage) {
+        return new ErrorMessage(emptyDocsumsCode, "Empty document summaries",detailedMessage);
+    }
+
+    /**
+     * Creates an error indicating that the requestor is not authorized to perform the requested operation.
+     * If this error is present, a HTTP layer will return 401.
+     */
+    public static ErrorMessage createUnauthorized(String detailedMessage) {
+        return new ErrorMessage(UNAUTHORIZED.code, "Client not authenticated.", detailedMessage);
+    }
+
+    /**
+     * Creates an error indicating that a forbidden operation was requested.
+     * If this error is present, a HTTP layer will return 403.
+     */
+    public static ErrorMessage createForbidden(String detailedMessage) {
+        return new ErrorMessage(FORBIDDEN.code, "Forbidden.", detailedMessage);
+    }
+
+    /**
+     * Creates an error indicating that the requested resource was not found.
+     * If this error is present, a HTTP layer will return 404.
+     */
+    public static ErrorMessage createNotFound(String detailedMessage) {
+        return new ErrorMessage(NOT_FOUND.code, "Resource not found.", detailedMessage);
+    }
+
+    /**
+     * Creates an error analog to HTTP bad request. If this error is present, a
+     * HTTP layer will return 400.
+     */
+    public static ErrorMessage createBadRequest(String detailedMessage) {
+        return new ErrorMessage(BAD_REQUEST.code, "Bad request.", detailedMessage);
+    }
+
+    /**
+     * Creates an error analog to HTTP internal server error. If this error is present, a
+     * HTTP layer will return 500.
+     */
+    public static ErrorMessage createInternalServerError(String detailedMessage) {
+        return new ErrorMessage(INTERNAL_SERVER_ERROR.code, "Internal server error.", detailedMessage);
+    }
+
+    /** Sets the source producing this error */
+    public void setSource(String source) { this.source = source; }
+
+    /** Returns the source producing this error, or null if no source is specified */
+    public String getSource() { return source; }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode() + (source == null ? 0 : 31 * source.hashCode());
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!super.equals(o)) return false;
+
+        ErrorMessage other = (ErrorMessage) o;
+        if (this.source != null) {
+            if (!this.source.equals(other.source)) return false;
+        } else {
+            if (other.source != null) return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return (source==null ? "" : "Source '" + source + "': ") + super.toString();
+    }
+
+    @Override
+    public ErrorMessage clone() {
+        return (ErrorMessage)super.clone();
+    }
+
+    /**
+     * Returns the given error message as this type. If it already is, this is a cast of the given instance.
+     * Otherwise this creates a new instance having the same payload as the given instance.
+     */
+    public static ErrorMessage from(com.yahoo.processing.request.ErrorMessage error) {
+        if (error instanceof ErrorMessage) return (ErrorMessage)error;
+        return new ErrorMessage(error.getCode(),error.getMessage(),error.getDetailedMessage(),error.getCause());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java
new file mode 100644
index 00000000000..5c57d21b455
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java
@@ -0,0 +1,44 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.data.access.Inspector;
+import com.yahoo.data.access.Inspectable;
+import com.yahoo.data.access.Type;
+import com.yahoo.data.JsonProducer;
+import com.yahoo.data.access.simple.JsonRender;
+
+/**
+ * A wrapper for structured data representing feature values.
+ */
+public class FeatureData implements Inspectable, JsonProducer {
+
+    private final Inspector value;
+
+    public FeatureData(Inspector value) {
+        this.value = value;
+    }
+
+    @Override
+    public Inspector inspect() {
+        return value;
+    }
+
+    public String toString() {
+        if (value.type() == Type.EMPTY) {
+            return "";
+        } else {
+            return toJson();
+        }
+    }
+
+    @Override
+    public String toJson() {
+        return writeJson(new StringBuilder()).toString();
+    }
+
+    @Override
+    public StringBuilder writeJson(StringBuilder target) {
+        return JsonRender.render(value, target, true);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java b/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java
new file mode 100644
index 00000000000..77f6db18745
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/FieldComparator.java
@@ -0,0 +1,106 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.search.query.Sorting;
+
+import java.util.Comparator;
+
+/**
+ * Comparator used for ordering hits using the field values and a sorting specification.
+ * <p>
+ * <b>Note:</b> this comparator imposes orderings that are inconsistent with equals.
+ * <p>
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+// Is tested in HitSortSpecOrdererTestCase
+public class FieldComparator extends ChainableComparator {
+
+    /** The definition of sorting order */
+    private Sorting sorting;
+
+    /** Creates a field comparator using a sort order and having no chained comparator */
+    public FieldComparator(Sorting sorting) {
+        this(sorting,null);
+    }
+
+    /** Creates a field comparator using a sort order with a chained comparator */
+    public FieldComparator(Sorting sorting,Comparator<Hit> secondaryComparator) {
+        super(secondaryComparator);
+        this.sorting = sorting;
+    }
+
+    /** Creates a comparator given a sorting, or returns null if the given sorting is null */
+    public static FieldComparator create(Sorting sorting) {
+        if (sorting==null) return null;
+        return new FieldComparator(sorting);
+    }
+
+    /**
+     * Compares hits based on a sorting specification and values
+     * stored in hit fields.0
+     * <p>
+     * When one of the hits has the requested property and the other
+     * has not, the the hit containing the property precedes the one
+     * that does not.
+     * <p>
+     * There is no locale based sorting here, as the backend does
+     * not do that either.
+     *
+     * @return -1, 0, 1 if first should be sorted before, equal to
+     * or after second
+     */
+    @Override
+    public int compare(Hit first, Hit second) {
+        for (Sorting.FieldOrder fieldOrder : sorting.fieldOrders() ) {
+            String fieldName = fieldOrder.getFieldName();
+            Object a = getField(first,fieldName);
+            Object b = getField(second,fieldName);
+
+            // If either of the values are null, don't touch the ordering
+            // This is to avoid problems if the sorting is called before the
+            // result is filled.
+            if ((a == null) || (b == null)) return 0;
+
+            int x = compareValues(a, b, fieldOrder.getSorter());
+            if (x != 0) {
+                if (fieldOrder.getSortOrder() == Sorting.Order.DESCENDING)
+                    x *= -1;
+                return x;
+            }
+        }
+        return super.compare(first,second);
+    }
+
+    public Object getField(Hit hit,String key) {
+        if ("[relevance]".equals(key)) return hit.getRelevance();
+        if ("[rank]".equals(key)) return hit.getRelevance();
+        if ("[source]".equals(key)) return hit.getSource();
+        return hit.getField(key);
+    }
+
+    @SuppressWarnings("rawtypes")
+    private int compareValues(Object first, Object second, Sorting.AttributeSorter s) {
+        if (first.getClass().isInstance(second)
+                && first instanceof Comparable) {
+            // We now know:
+            // second is of a type which is a subclass of first's type
+            // They both implement Comparable
+            return s.compare((Comparable)first, (Comparable)second);
+        } else {
+            return s.compare(first.toString(), second.toString());
+        }
+    }
+
+    public String toString() {
+        StringBuilder b = new StringBuilder();
+        b.append("FieldComparator:");
+        if (sorting == null) {
+            b.append(" null");
+        } else {
+            b.append(sorting.toString());
+        }
+        return b.toString();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java
new file mode 100644
index 00000000000..2cf1dba7efd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java
@@ -0,0 +1,787 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.collections.ArraySet;
+import com.yahoo.component.provider.ListenableFreezableClass;
+import com.yahoo.net.URI;
+import com.yahoo.prelude.hitfield.HitField;
+import com.yahoo.prelude.hitfield.JSONString;
+import com.yahoo.prelude.hitfield.XMLString;
+import com.yahoo.processing.Request;
+import com.yahoo.processing.response.Data;
+import com.yahoo.search.Query;
+import com.yahoo.search.Searcher;
+import com.yahoo.text.XML;
+
+import java.util.*;
+
+/**
+ * <p>A search hit. The identifier of the hit is the uri
+ * (the uri is immutable once set).
+ * If two hits have the same uri they are equal per definition.
+ * Hits are naturally ordered by decreasing relevance.
+ * Note that this definition of equals and natural ordering is inconsistent.</p>
+ *
+ * <p>Hits may be of the <i>meta</i> type, meaning that they contain some information
+ * about the query or result which does not represent a particular piece of matched
+ * content. Meta hits are not counted in the hit count of the result, and should
+ * usually never be filtered out.</p>
+ *
+ * <p>Some hit sources may produce hits which are not <i>filled</i>. A non-filled
+ * hit may miss some or all of its property values. To fill those,
+ * {@link com.yahoo.search.Searcher#fill fill} must be called on the search chain by the searcher
+ * which requires those properties. This mechanism allows initial filtering to be
+ * done of a lightweight version of the hits, which is cheaper if a significant
+ * number of hits are filtered out.</p>
+ *
+ * @author bratseth
+ */
+public class Hit extends ListenableFreezableClass implements Data, Comparable<Hit>, Cloneable {
+
+    private static final String DOCUMENT_ID = "documentid";
+
+    /** A collection of string keyed object properties. */
+    private Map<String,Object> fields = null;
+    private Map<String,Object> unmodifiableFieldMap = null;
+
+    /** Meta data describing how a given searcher should treat this hit. */
+    // TODO: The case for this is to allow multiple levels of federation searcher routing.
+    //       Replace this by a cleaner specific solution to that problem.
+    private Map<Searcher, Object> searcherSpecificMetaData;
+
+    /** The id of this hit */
+    private URI id;
+
+    /** The types of this hit */
+    private Set<String> types = new ArraySet<>(2);
+
+    /** The relevance of this hit */
+    private Relevance relevance;
+
+    /** Says whether this hit is cached or not */
+    private boolean cached = false;
+
+    /**
+     * The summary classes for which this hit is filled. If this set
+     * is 'null', it means that this hit is unfillable, which is
+     * equivalent to a hit where all summary classes have already
+     * been filled, or a hit where further filling will
+     * yield no extra information, if you prefer to look at it that
+     * way.
+     */
+    private Set<String> filled = null;
+    private Set<String> unmodifiableFilled = null;
+
+    /** The name of the source creating this hit */
+    private String source = null;
+
+    /**
+     * Add number, assigned when adding the hit to a result,
+     * used to order equal relevant hit by add order
+     */
+    private int addNumber = -1;
+    private int sourceNumber;
+
+    /** The query which produced this hit. Used for multi phase searching */
+    private Query query;
+
+    /**
+     * Set to true for hits which does not contain content,
+     * but which contains meta information about the query or result
+     */
+    private boolean meta=false;
+
+    /** If this is true, then this hit will not be counted as a concrete hit */
+    private boolean auxiliary=false;
+
+    /**
+     * The hit field used to store rank features. TODO: Remove
+     */
+    public static final String RANKFEATURES_FIELD = "rankfeatures";
+    public static final String SDDOCNAME_FIELD = "sddocname";
+
+    private Map<String,Object> getFieldMap() {
+        if (fields == null) {
+            fields = new LinkedHashMap<>(16);
+        }
+        return fields;
+    }
+
+    private Map<String,Object> getUnmodifiableFieldMap() {
+        if (unmodifiableFieldMap == null) {
+            if (fields == null) {
+                return Collections.emptyMap();
+            } else {
+                unmodifiableFieldMap = Collections.unmodifiableMap(fields);
+            }
+        }
+        return unmodifiableFieldMap;
+    }
+
+    public static String stripCharacter(char strip, String toStripFrom) {
+        StringBuilder builder = null;
+
+        int lastBadChar = 0;
+        for (int i = 0; i < toStripFrom.length(); i++) {
+            if (toStripFrom.charAt(i) == strip) {
+                if (builder == null) {
+                    builder = new StringBuilder(toStripFrom.length());
+                }
+
+                builder.append(toStripFrom, lastBadChar, i);
+                lastBadChar = i + 1;
+            }
+        }
+
+        if (builder == null) {
+            return toStripFrom;
+        } else {
+            if (lastBadChar < toStripFrom.length()) {
+                builder.append(toStripFrom, lastBadChar, toStripFrom.length());
+            }
+
+            return builder.toString();
+        }
+    }
+
+    /** Creates an (invalid) empty hit. Id and relevance must be set before handoff */
+    protected Hit() {}
+
+    /**
+     * Creates a minimal valid hit having relevance 1000
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types refering to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     */
+    public Hit(String id) {
+        this(id, 1);
+    }
+
+    /**
+     * Creates a minimal valid hit having relevance 1
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types referring to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param query the query having this as a hit
+     */
+    public Hit(String id, Query query) {
+        this(id, 1, query);
+    }
+
+    /**
+     * Creates a minimal valid hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types referring to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance a relevance measure, preferably normalized between 0 and 1
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1
+     */
+    public Hit(String id, double relevance) {
+        this(id,new Relevance(relevance));
+    }
+
+    /**
+     * Creates a minimal valid hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types referring to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance a relevance measure, preferably normalized between 0 and 1
+     * @param query the query having this as a hit
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1
+     */
+    public Hit(String id, double relevance, Query query) {
+        this(id,new Relevance(relevance),query);
+    }
+
+    /**
+     * Creates a minimal valid hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types refering to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance the relevance of this hit
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1000
+     */
+    public Hit(String id, Relevance relevance) {
+        this(id, relevance, (String)null);
+    }
+
+    /**
+     * Creates a minimal valid hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types refering to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance the relevance of this hit
+     * @param query the query having this as a hit
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1000
+     */
+    public Hit(String id, Relevance relevance, Query query) {
+        this(id, relevance,null, query);
+    }
+
+    /**
+     * Creates a hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types refering to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance a relevance measure, preferably normalized between 0 and 1
+     * @param source the name of the source of this hit, or null if no source is being specified
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1000
+     */
+    public Hit(String id, double relevance, String source) {
+        this(id, new Relevance(relevance), source, null);
+    }
+
+    /**
+     * Creates a hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types refering to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance a relevance measure, preferably normalized between 0 and 1
+     * @param source the name of the source of this hit, or null if no source is being specified
+     * @param query the query having this as a hit
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1000
+     */
+    public Hit(String id, double relevance, String source, Query query) {
+        this(id, new Relevance(relevance), source);
+    }
+
+    /**
+     * Creates a hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types refering to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance the relevance of this hit
+     * @param source the name of the source of this hit
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1000
+     */
+    public Hit(String id, Relevance relevance, String source) {
+        this(id, relevance, source, null);
+    }
+
+    /**
+     * Creates a hit.
+     *
+     * @param id the URI of a hit. This should be unique for this hit (but not for this
+     *        <i>object instance</i> of course). For hit types refering to resources,
+     *        this will be the full, valid url of the resource, for self-contained hits
+     *        it is simply any unique string identification
+     * @param relevance the relevance of this hit
+     * @param source the name of the source of this hit
+     * @param query the query having this as a hit
+     * @throws IllegalArgumentException if the given relevance is not between 0 and 1000
+     */
+    public Hit(String id, Relevance relevance, String source, Query query) {
+        this.id=new URI(id);
+        this.relevance = relevance;
+        this.source=source;
+        this.query = query;
+    }
+
+    /** Calls setId(new URI(id)) */
+    public void setId(String id) {
+        if (this.id!=null) throw new IllegalStateException("Attempt to change id of " + this + " to " + id);
+        if (id==null) throw new NullPointerException("Attempt to assign id of " + this + " to null");
+        assignId(new URI(id));
+    }
+
+
+    /**
+     * Initializes the id of this hit.
+     *
+     * @throws NullPointerException if the uri is null
+     * @throws IllegalStateException if the uri of this hit is already set
+     */
+    public void setId(URI id) {
+        if (this.id!=null) throw new IllegalStateException("Attempt to change id of " + this + " to " + id);
+        assignId(id);
+    }
+
+    /**
+     * Assigns a new or changed id to this hit.
+     * As this is protected, reassigning isn't legal for Hits by default, however, subclasses may allow it
+     * using this method.
+     */
+    protected final void assignId(URI id) {
+        if (id==null) throw new NullPointerException("Attempt to assign id of " + this + " to null");
+        this.id=id;
+    }
+
+    /** Returns the hit id */
+    public URI getId() { return id; }
+
+    /**
+     * Returns the id to display, or null to not display (render) the id.
+     * This is useful to avoid displaying ids when they are not assigned explicitly
+     * but are just generated values for internal use.
+     * This default implementation returns {@link #getId()}.toString()
+     */
+    public String getDisplayId() {
+        String id = null;
+
+        Object idField = getField(DOCUMENT_ID);
+        if (idField != null) {
+            id = idField.toString();
+        }
+        if (id == null) {
+            id = getId() == null ? null : getId().toString();
+        }
+        return id;
+    }
+
+    /**
+     * Sets the relevance of this hit
+     *
+     * @param relevance the relevance of this hit
+     */
+    public void setRelevance(Relevance relevance) {
+        if (relevance==null) throw new NullPointerException("Cannot assign null as relevance");
+        this.relevance = relevance;
+    }
+
+    /** Does setRelevance(new Relevance(relevance) */
+    public void setRelevance(double relevance) {
+        setRelevance(new Relevance(relevance));
+    }
+
+
+    /** Returns the relevance of this hit */
+    public Relevance getRelevance() { return relevance; }
+
+    /** Sets whether this hit is returned from a cache. Default is false */
+    public void setCached(boolean cached) { this.cached = cached; }
+
+    /** Returns whether this hit was added to this result from a cache or not */
+    public boolean isCached() { return cached; }
+
+    /**
+     * Tag this hit as fillable. This means that additional properties
+     * for this hit may be obtained by fetching document
+     * summaries. This also enables tracking of which summary classes
+     * have been used for filling so far. Invoking this method
+     * multiple times is allowed and will have no addition
+     * effect. Note that a fillable hit may not be made unfillable.
+     **/
+    public void setFillable() {
+        if (filled == null) {
+            filled = Collections.emptySet();
+            unmodifiableFilled = filled;
+        }
+    }
+
+    /**
+     * Register that this hit has been filled with properties using
+     * the given summary class. Note that this method will implicitly
+     * tag this hit as fillable if it is currently not.
+     *
+     * @param summaryClass summary class used for filling
+     **/
+    public void setFilled(String summaryClass) {
+        if (filled == null || filled.size() == 0) {
+            filled = Collections.singleton(summaryClass);
+            unmodifiableFilled = filled;
+        } else if (filled.size() == 1) {
+            filled = new HashSet<>(filled);
+            unmodifiableFilled = Collections.unmodifiableSet(filled);
+
+            filled.add(summaryClass);
+        } else {
+            filled.add(summaryClass);
+        }
+    }
+
+    public boolean isFillable() {
+        return filled != null;
+    }
+
+    /**
+     * Returns the set of summary classes for which this hit is
+     * filled as an unmodifiable set. If this set is 'null', it means that this hit is
+     * unfillable, which is equivalent with a hit where all summary
+     * classes have already been used for filling, or a hit where
+     * further filling will yield no extra information, if you prefer
+     * to look at it that way.
+     *
+     * Note that you might need to overload isFilled if you overload this one.
+     **/
+    public Set<String> getFilled() {
+        return unmodifiableFilled;
+    }
+
+    /**
+     * Returns whether this hit has been filled with the properties
+     * contained in the given summary class. Note that this method
+     * will also return true if this hit is not fillable.
+     */
+    public boolean isFilled(String summaryClass) {
+        return (filled == null) || filled.contains(summaryClass);
+    }
+
+    /** Sets the name of the source creating this hit */
+    public void setSource(String source) { this.source = source; }
+
+    /** Returns the name of the source creating this hit */
+    public String getSource() { return source; }
+
+    /** Returns the fields of this as a read-only map. This is more costly than the preferred iterator(), as
+     * it uses Collections.unmodifiableMap()
+     * @return An readonly map of the fields
+     **/
+    //TODO Should it be deprecated ?
+    public final Map<String,Object> fields() { return getUnmodifiableFieldMap(); }
+
+    /**
+     * Fields
+     * @return An iterator for traversing the fields
+     * @since 5.1.3
+     */
+    public final Iterator<Map.Entry<String,Object>> fieldIterator() { return getFieldMap().entrySet().iterator(); }
+
+    /** Returns a field value */
+    public Object getField(String value) { return fields != null ? fields.get(value) : null; }
+
+    /**
+     * Generate a HitField from a field if the field exists. Does the
+     * same as getField() in earlier versions.
+     *
+     * @since 3.0
+     */
+    public HitField buildHitField(String key) {
+        return buildHitField(key, false);
+    }
+
+    /**
+     * Generate a HitField from a field if the field exists. Does the
+     * same as getField() in earlier versions.
+     *
+     * @since 3.0
+     */
+    public HitField buildHitField(String key, boolean forceNoPreTokenize) {
+        return buildHitField(key, forceNoPreTokenize, false);
+    }
+
+    public HitField buildHitField(String key, boolean forceNoPreTokenize, boolean forceStringHandling) {
+        Object o = getField(key);
+        if (o == null) {
+            return null;
+        }
+
+        if (o instanceof HitField) {
+            return (HitField) o;
+        }
+
+        HitField h;
+        if (forceNoPreTokenize) {
+            if (o instanceof XMLString && !forceStringHandling) {
+                h = new HitField(key, (XMLString) o, false);
+            } else {
+                h = new HitField(key, o.toString(), false);
+            }
+        } else {
+            if (o instanceof XMLString && !forceStringHandling) {
+                h = new HitField(key, (XMLString) o);
+            } else {
+                h = new HitField(key, o.toString());
+            }
+        }
+        h.setOriginal(o);
+        getFieldMap().put(key, h);
+        return h;
+    }
+
+    /**
+     * Sets the value of a field
+     *
+     * @return the previous value, or null if none
+     */
+    public Object setField(String key, Object value) {
+        return getFieldMap().put(key, value);
+    }
+
+    /** Returns the types of this as a modifiable set. Modifications to this set are directly reflected in this hit */
+    public Set<String> types() { return types; }
+
+    /**
+     * Returns all types of this hit as a space-separated string
+     *
+     * @return all the types of this hit on the form "type1 type2 type3"
+     *         (in no particular order). An empty string (never null) if
+     *         no types are added
+     */
+    public String getTypeString() {
+        StringBuilder buffer = new StringBuilder(types.size() * 7);
+
+        for (Iterator<String> i = types.iterator(); i.hasNext();) {
+            buffer.append(i.next());
+            if (i.hasNext())
+                buffer.append(" ");
+        }
+        return buffer.toString();
+    }
+
+    /**
+     * Returns true if the argument is a hit having the same uri as this
+     */
+    public boolean equals(Object object) {
+        if (!(object instanceof Hit)) {
+            return false;
+        }
+        return getId().equals(((Hit) object).getId());
+    }
+
+    /**
+     * Returns the hashCode of this hit, which is the hashcode of its uri.
+     */
+    public int hashCode() {
+        if (getId() == null)
+            throw new IllegalStateException("Id has not been set.");
+
+        return getId().hashCode();
+    }
+
+    /** Compares this hit to another hit */
+    public int compareTo(Hit other) {
+        // higher relevance is better
+        int result = other.getRelevance().compareTo(getRelevance());
+        if (result != 0) {
+            return result;
+        }
+        // lower addnumber is better
+        result = this.getAddNumber() - other.getAddNumber();
+        if (result != 0) {
+            return result;
+        }
+
+        // if all else fails, compare URIs (alphabetically)
+        if (this.getId() == null && other.getId() == null) {
+            return 0;
+        } else if (other.getId() == null) {
+            return -1;
+        } else if (this.getId() == null) {
+            return 1;
+        } else {
+            return this.getId().compareTo(other.getId());
+        }
+    }
+
+    /**
+     * Returns the add number, assigned when adding the hit to a Result.
+     *
+     * Used to order equal relevant hit by add order. -1 if this hit
+     * has never been added to a result.
+     */
+    public int getAddNumber() { return addNumber; }
+
+    /**
+     * Sets the add number, assigned when adding the hit to a Result,
+     * used to order equal relevant hit by add order
+     */
+    public void setAddNumber(int addNumber) { this.addNumber = addNumber; }
+
+    /**
+     * Returns whether this is a concrete hit, containing content of the requested
+     * kind, or a meta hit containing information on the collection of hits,
+     * the query, the service and so on. This default implementation return false.
+     */
+    public boolean isMeta() { return meta; }
+
+    public void setMeta(boolean meta) { this.meta=meta; }
+
+    /**
+     * Auxiliary hits are not counted towards the concrete number of hits to satisfy in the users request.
+     * Any kind of meta hit is auxiliary, but hits containing concrete results can also be auxiliary,
+     * for example ads in a service which does not primarily serve ads, or groups in a hierarchical organization.
+     *
+     * @return true if the auxiliary value is true, or if this is a meta hit
+     */
+    public boolean isAuxiliary() {
+        return isMeta() || auxiliary;
+    }
+
+    public void setAuxiliary(boolean auxiliary) { this.auxiliary=auxiliary; }
+
+    /** Removes all fields from this */
+    public void clearFields() {
+        getFieldMap().clear();
+    }
+
+    /** Removes a field from this */
+    public Object removeField(String field) {
+        return getFieldMap().remove(field);
+    }
+
+    /**
+     * Returns the keys of the fields of this hit as a modifiable view.
+     * This follows the rules of key sets returned from maps: Key removals are reflected
+     * in the map, add and addAll is not supported.
+     */
+    public Set<String> fieldKeys() {
+        return getFieldMap().keySet();
+    }
+
+    /**
+     * Changes the key under which a value is found. This is useful because it allows keys to be changed
+     * without accessing the value (which may be lazily created).
+     */
+    public void changeFieldKey(String oldKey,String newKey) {
+        Map<String,Object> fieldMap = getFieldMap();
+        Object value=fieldMap.remove(oldKey);
+        fieldMap.put(newKey,value);
+    }
+
+    /**
+     * Returns a string describing this hit
+     */
+    public String toString() {
+        return "hit " + getId() + " (relevance " + getRelevance() + ")";
+    }
+
+    public Hit clone() {
+        Hit hit = (Hit) super.clone();
+
+        hit.fields = fields != null ? new LinkedHashMap<>(fields) : null;
+        hit.unmodifiableFieldMap = null;
+        hit.types = new LinkedHashSet<>(types);
+        if (filled != null) {
+            hit.setFilledInternal(new HashSet<>(filled));
+        }
+
+        return hit;
+    }
+
+    public int getSourceNumber() { return sourceNumber; }
+
+    public void setSourceNumber(int number) { this.sourceNumber = number; }
+
+    /** Returns the query which produced this hit, or null if not known */
+    public Query getQuery() { return query; }
+
+    public Request request() { return query; }
+
+    // TODO: rethink hit tagging
+    // hit group -> need option to retag
+    // hit -> should only set query once
+    public final void setQuery(Query query) {
+        if (this.query == null || this instanceof HitGroup) {
+            this.query = query;
+        }
+    }
+
+    // TODO: Deprecate
+    /**
+     * Returns a field of this hit XML escaped and without token
+     * delimiters.
+     *
+     * @return a field of this hit, or null if the property is not set
+     */
+    public String getFieldXML(String key) {
+        Object p = getField(key);
+
+        if (p == null) {
+            return null;
+        } else if (p instanceof HitField) {
+            HitField hf = (HitField) p;
+
+            return hf.quotedContent(false);
+        } else if (p instanceof StructuredData) {
+            return p.toString();
+        } else if (p instanceof XMLString || p instanceof JSONString) {
+            return p.toString();
+        } else {
+            return XML.xmlEscape(p.toString(), false, '\u001f');
+        }
+    }
+
+    // TODO: Move out? If not, delegate here from subclass
+    /**
+     * @return a field without bolding markup
+     */
+    public String getUnboldedField(String key, boolean escape) {
+        Object p = getField(key);
+
+        if (p == null) {
+            return null;
+        } else if (p instanceof HitField) {
+            return ((HitField) p).bareContent(escape, false);
+        } else if (p instanceof StructuredData) {
+            return p.toString();
+        } else if (p instanceof XMLString || p instanceof JSONString) {
+            return p.toString();
+        } else if (escape) {
+            return XML.xmlEscape(p.toString(), false, '\u001f');
+        } else {
+            return stripCharacter('\u001F', p.toString());
+        }
+    }
+
+    /**
+     * set meta data describing how a given searcher should treat this hit.
+     * It is currently recommended that the invoker == searcher.
+     * <b>Internal. Do not use!</b>
+     */
+    public void setSearcherSpecificMetaData(Searcher searcher, Object data) {
+        if (searcherSpecificMetaData == null) {
+            searcherSpecificMetaData = Collections.singletonMap(searcher, data);
+        } else {
+            if (searcherSpecificMetaData.size() == 1) {
+                Object tmp = searcherSpecificMetaData.get(searcher);
+                if (tmp != null) {
+                    searcherSpecificMetaData = Collections.singletonMap(searcher, data);
+                } else {
+                    searcherSpecificMetaData = new TreeMap<>(searcherSpecificMetaData);
+                    searcherSpecificMetaData.put(searcher, data);
+                }
+            } else {
+                searcherSpecificMetaData.put(searcher, data);
+            }
+        }
+    }
+
+    /**
+     * get meta data describing how a given searcher should treat this hit.
+     * It is currently recommended that the invoker == searcher
+     * <b>Internal. Do not use!</b>
+     */
+    public Object getSearcherSpecificMetaData(Searcher searcher) {
+        return searcherSpecificMetaData != null ? searcherSpecificMetaData.get(searcher) : null;
+    }
+
+    /**
+     * For vespa internal use only.
+     * This is only for the ones specially interested. It will replace the backing
+     * for filled.
+     * @param filled the backing set
+     */
+    protected final void setFilledInternal(Set<String> filled) {
+        this.filled = filled;
+        unmodifiableFilled = (filled != null) ? Collections.unmodifiableSet(filled) : null;
+    }
+
+    /**
+     * For vespa internal use only.
+     * Gives access to the modifiable backing set of filled summaries.
+     * This set might be unmodifiable if the size is less than or equal to 1
+     * @return the set of filled summaries.
+     */
+    protected final Set<String> getFilledInternal() {
+        return filled;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java
new file mode 100644
index 00000000000..e58c3dc847e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java
@@ -0,0 +1,898 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.yahoo.collections.ListenableArrayList;
+import com.yahoo.net.URI;
+import com.yahoo.processing.response.ArrayDataList;
+import com.yahoo.processing.response.DataList;
+import com.yahoo.processing.response.DefaultIncomingData;
+import com.yahoo.processing.response.IncomingData;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import static com.yahoo.collections.CollectionUtil.first;
+
+/**
+ * <p>A group of ordered hits. Since hitGroup is itself a kind of Hit,
+ * this can compose hierarchies of grouped hits.</p>
+ *
+ * <p>Group hits has a relevancy just as other hits - they can be ordered
+ * between each other and in comparison to other hits.
+ *
+ * <p>Note that a group is by default a meta hit, but it can also contain its own content
+ * in addition to subgroup content, in which case it should be set to non-meta.</p>
+ *
+ * @author bratseth
+ */
+public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable<Hit> {
+
+    // This does its own book-keeping of its various state variables
+    // (see methods towards the end). For state variables which are recursive
+    // (depending on the state of hits in subgroups), the strategy is to do
+    // book-keeping on only this immediate level, but not do recursive calls to
+    // find the true recursive state when queried. This is sort of a middle ground
+    // between handling the complexity of recursive state book-keeping and the
+    // query cost of not doing any book-keeping.
+    // There is also a method, analyse which recursively updates the recursive
+    // state of the group and all subgroups. This should be called if the hits
+    // may have changed their own state in a way that may impact the recursive
+    // state of this.
+
+    private ListenableArrayList<Hit> hits = new ListenableArrayList<>(16);
+
+    transient private List<Hit> unmodifiableHits = Collections.unmodifiableList(hits);
+
+    /** Whether or not the hits are sorted */
+    private boolean hitsSorted = true;
+
+    /** Whether or not deletion of hits breaks the sorted ordering */
+    private boolean deletionBreaksOrdering = false;
+
+    /** Whether the hits should be sorted (again) */
+    private boolean orderedHits = false;
+
+    /** The current number of concrete (non-meta) hits in the result */
+    private int concreteHitCount = 0;
+
+    /** The class used to determine the ordering of the hits of this */
+    transient private HitOrderer hitOrderer = null;
+
+    /** Accounting the number of subgroups to allow some early returns when the number is 0 */
+    private int subgroupCount=0;
+
+    /**
+     * The number of hits not cached at this level, not counting hits in subgroups or
+     * any nested hitgroups themselves
+     */
+    private int notCachedCount=0;
+
+    /**
+     * A direct reference to the errors of this result, or null if there are no errors.
+     * The error hit will also be listed in the set of this of this result
+     */
+    private ErrorHit errorHit = null;
+
+    private final ListenableFuture<DataList<Hit>> completedFuture;
+
+    private final IncomingData<Hit> incomingHits;
+
+    /** Creates an invalid group of hits. Id must be set before handoff. */
+    public HitGroup() {
+        incomingHits = new IncomingData.NullIncomingData<>(this);
+        setRelevance(new Relevance(1));
+        setMeta(true);
+        completedFuture = new IncomingData.NullIncomingData.ImmediateFuture<>(this);
+    }
+
+    /**
+     * Creates a hit group with max relevancy (1)
+     *
+     * @param id the id of this hit - any string, it is convenient to make this unique in the result containing this
+     */
+    public HitGroup(String id) {
+        this(id,new Relevance(1));
+    }
+
+    /**
+     * Creates a hit group
+     *
+     * @param id the id of this hit - any string, it is convenient to make this unique in the result containing this
+     * @param relevance the relevance of this group of hits, preferably a number between 0 and 1
+     */
+    public HitGroup(String id,double relevance) {
+        this(id,new Relevance(relevance));
+    }
+
+    /**
+     * Creates a group hit
+     *
+     * @param id the id of this hit - any string, it is convenient to make this unique in the result containing this
+     * @param relevance the relevancy of this group of hits
+     */
+    public HitGroup(String id, Relevance relevance) {
+        super(id, relevance);
+        this.incomingHits = new IncomingData.NullIncomingData<>(this);
+        setMeta(true);
+        completedFuture = new IncomingData.NullIncomingData.ImmediateFuture<>(this);
+    }
+
+    /**
+     * Creates a group hit
+     *
+     * @param id the id of this hit - any string, it is convenient to make this unique in the result containing this
+     * @param relevance the relevancy of this group of hits
+     * @param incomingHits the incoming buffer to which new hits can be added asynchronously
+     */
+    protected HitGroup(String id, Relevance relevance, IncomingData<Hit> incomingHits) {
+        super(id, relevance);
+        this.incomingHits = incomingHits;
+        setMeta(true);
+        completedFuture = new ArrayDataList.DrainOnGetFuture<>(this);
+    }
+
+    /**
+     * Creates a HitGroup which contains data which arrives in the future.
+     *
+     * @param id the id of this
+     * @return a HitGroup which is incomplete and which has an {@link #incoming} where new hits can be added later
+     */
+    public static HitGroup createAsync(String id) {
+        DefaultIncomingData<Hit> incomingData = new DefaultIncomingData<>();
+        HitGroup hitGroup = new HitGroup(id, new Relevance(1), incomingData);
+        incomingData.assignOwner(hitGroup);
+        return hitGroup;
+    }
+
+    /** Calls setId(new URI(id)) */
+    @Override
+    public void setId(String id) {
+        setId(new URI(id));
+    }
+
+    /**
+     * Assign an id to this hit.
+     * For HitGroups, this is a legal call also when an id is already set,
+     * i.e hit groups allows their ids to be reassigned.
+     * This is to allow hit groups to be inserted in new structures with an id reflecting their
+     * role/placement in the structure.
+     *
+     * @param id the new or initial iof of this hit
+     */
+    @Override
+    public void setId(URI id) {
+        super.assignId(id);
+    }
+
+    /**
+     * Turn off internal resorting of hits.
+     *
+     * @param ordered set to true to tell this group that the hits set in it is already correctly ordered and should
+     *                never be resorted. Set to false to use the default lazy resorting by hit ordering.
+     */
+    public void setOrdered(boolean ordered) { this.orderedHits = ordered; }
+
+    /**
+     * Returns the number of hits available immediately in this group
+     * (counting a subgroup as one hit).
+     */
+    public int size() {
+        return hits.size();
+    }
+
+    /**
+     * <p>Returns the number of concrete hits contained in this group
+     * and all subgroups. This should equal the
+     * requested hits count if the query has that many matches.</p>
+     */
+    public int getConcreteSize() {
+        if (subgroupCount<1) return concreteHitCount;
+        int recursiveConcreteCount=concreteHitCount;
+        for (Hit hit : hits) {
+            if (hit instanceof HitGroup)
+                recursiveConcreteCount+=((HitGroup)hit).getConcreteSize();
+        }
+        return recursiveConcreteCount;
+    }
+
+    /**
+     * <p>Returns the number of concrete hits contained in <i>this</i> group,
+     * without counting hits in subgroups.
+     */
+    public int getConcreteSizeShallow() { return concreteHitCount; }
+
+    /**
+     * Returns the number of HitGroups present immediately in this list of hits.
+     */
+    public int getSubgroupCount() { return subgroupCount; }
+
+    /**
+     * Adds a hit to this group.
+     * If the given hit is an ErrorHit and this group already have an error hit,
+     * the errors in the given hit are merged into the errors of this.
+     *
+     * @return the resulting hit - this is usually the input hit, but if an error hit was added,
+     *         and there was already an error hit present, that hit, containing the merged information
+     *         is returned
+     */
+    @Override
+    public Hit add(Hit hit) {
+        if (hit.isMeta() && hit instanceof ErrorHit) {
+            boolean add = mergeErrors((ErrorHit) hit);
+            if (!add) return (Hit)errorHit;
+        }
+        handleNewHit(hit);
+        hits.add(hit);
+        return hit;
+    }
+
+    /**
+     * Adds a list of hits to this group, the same
+     */
+    public void addAll(List<Hit> hits) {
+        for (Hit hit : hits)
+            add(hit);
+    }
+
+    /**
+     * Returns the hit at the given (0-base) index in this group of hit
+     * (without searching any subgroups).
+     *
+     * @param  index the index into this list
+     * @throws IndexOutOfBoundsException if there is no hit at the given index
+     */
+    public Hit get(int index) {
+        updateHits();
+        ensureSorted();
+        return hits.get(index);
+    }
+
+    /** Same as {@link #get(String,int)} */
+    public Hit get(String id) {
+        return get(id,-1);
+    }
+
+    public Hit get(String id, int depth) {
+        return get(new URI(id), depth);
+    }
+
+    /**
+     * Returns the hit with the given id, or null if there is no hit with this id
+     * in this group or any subgroup.
+     * This method is o(min(number of nested hits in this result,depth)).
+     *
+     * @param id the id of the hit to return from this or any nested group
+     * @param depth the max depth to recurse into nested groups: -1: Recurse infinitely deep, 0: Only look at hits in
+     *        the list of this group, 1: Look at hits in this group, and the hits of any immediate nested HitGroups,
+     *        etc.
+     * @return The hit, or null if not found.
+     */
+    public Hit get(URI id, int depth) {
+        updateHits();
+        for (Iterator<Hit> i = unorderedIterator(); i.hasNext();) {
+            Hit hit = i.next();
+            URI hitUri = hit.getId();
+
+            if (hitUri != null && hitUri.equals(id)) {
+                return hit;
+            }
+
+            if (hit instanceof HitGroup && depth!=0) {
+                Hit found=((HitGroup)hit).get(id,depth-1);
+                if (found!=null) return found;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Inserts the given hit at the specified index in this group.
+     */
+    public void set(int index, Hit hit) {
+        updateHits();
+        if (hit instanceof ErrorHit) { // Merge instead
+            add(hit);
+            return;
+        }
+
+        handleNewHit(hit);
+        Hit oldHit = hits.set(index, hit);
+
+        if (oldHit!=null)
+            handleRemovedHit(oldHit);
+    }
+
+    /**
+     * Adds a hit to this group in the specified index,
+     * all existing hits on this index and higher will have their index
+     * increased by one.
+     * <b>Note:</b> If the group was sorted, it will still be considered sorted
+     * after this call.
+     */
+    public void add(int index, Hit hit) {
+        if (hit instanceof ErrorHit) { // Merge instead
+            add(hit);
+            return;
+        }
+
+        boolean wasSorted = hitsSorted;
+        handleNewHit(hit);
+        hits.add(index, hit);
+        hitsSorted = wasSorted;
+    }
+
+    /**
+     * Removes a hit from this group or any subgroup
+     *
+     * @param uriString the uri of the hit to remove
+     * @return the hit to remove, or null if the hit was not present
+     */
+    public Hit remove(String uriString) {
+        return remove(new URI(uriString));
+    }
+
+    /**
+     * Removes a hit from this group or any subgroup.
+     *
+     * @param uri The uri of the hit to remove.
+     * @return The hit removed, or null if not found.
+     */
+    public Hit remove(URI uri) {
+        for (Iterator<Hit> it = hits.iterator(); it.hasNext(); ) {
+            Hit hit = it.next();
+            if (uri.equals(hit.getId())) {
+                it.remove();
+                handleRemovedHit(hit);
+                return hit;
+            }
+            if (hit instanceof HitGroup) {
+                Hit removed = ((HitGroup)hit).remove(uri);
+                if (removed != null) {
+                    return removed;
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Removes a hit from this group (not considering the hits of any subgroup)
+     *
+     * @param index the position of the hit to remove
+     * @return the hit removed
+     * @throws IndexOutOfBoundsException if there is no hit at the given position
+     */
+    public Hit remove(int index) {
+        updateHits();
+        Hit hit = hits.remove(index);
+        handleRemovedHit(hit);
+
+        return hit;
+    }
+
+    /** Sets the main error of this result. Prefer addError to add some error information. */
+    public void setError(ErrorMessage error) {
+        if (errorHit == null)
+            add((Hit)createErrorHit(error));
+        else
+            errorHit.addError(error);
+    }
+
+    /** Adds an error to this result */
+    public void addError(ErrorMessage error) {
+        if (errorHit == null)
+            add((Hit)createErrorHit(error));
+        else
+            errorHit.addError(error);
+    }
+
+    /**
+     * Returns the error hit containing all error information,
+     * or null if no error has occurred
+     */
+    public ErrorHit getErrorHit() {
+        getError(); // Make sure the error hit is updated
+        return errorHit;
+    }
+
+    /**
+     * Returns the first error in this result,
+     * or null if no searcher has produced an error AND the query doesn't contain an error
+     */
+    public ErrorMessage getError() {
+        // See updateHits if this method is changed
+        if (errorHit != null) {
+            return errorHit.errors().iterator().next();
+        }
+
+        if (getQuery() != null && getQuery().errors().size() != 0) {
+            updateHits();
+        } // Pull them over
+
+        if (errorHit == null) {
+            return null;
+        }
+
+        return errorHit.errors().iterator().next();
+    }
+
+    /**
+     * Handles the addition of a new error hit, whether or not we already have one
+     *
+     * @return true if this shouls also be added to the list of hits of this reslt
+     */
+    private boolean mergeErrors(ErrorHit newHit) {
+        if (errorHit == null) {
+            errorHit = newHit;
+            return true;
+        } else {
+            errorHit.addErrors(newHit);
+            return false;
+        }
+    }
+
+    /**
+     * Must be called before the list of hits, or anything dependent on the list of hits, is removed.
+     * Merges errors from the query if there is one set for this group
+     */
+    private void updateHits() {
+        if (getQuery()==null) return;
+
+        if (getQuery().errors().size() == 0) return;
+
+        if (errorHit == null) // Creates an error hit where the first error is "main"
+            add((Hit)createErrorHit(toSearchError(getQuery().errors().get(0))));
+
+        // Add the rest of the errors
+        for (int i=1; i<getQuery().errors().size(); i++)
+            errorHit.addError(toSearchError(getQuery().errors().get(i)));
+        getQuery().errors().clear(); // TODO: Really clear them from here?
+    }
+
+    protected ErrorHit createErrorHit(ErrorMessage errorMessage) {
+        return new DefaultErrorHit(getSource(), errorMessage);
+    }
+
+    /** Compatibility */
+    private ErrorMessage toSearchError(com.yahoo.processing.request.ErrorMessage error) {
+        if (error instanceof ErrorMessage) return (ErrorMessage)error;
+        else return new ErrorMessage(error.getCode(),error.getMessage(),error.getDetailedMessage(),error.getCause());
+    }
+
+    /**
+     * Remove the first <code>offset</code> <i>concrete</i> hits in this group,
+     * and hits beyond <code>offset+numHits</code>
+     */
+    public void trim(int offset, int numHits) {
+        updateHits();
+        ensureSorted();
+
+        int highBound = numHits + offset; // Largest offset +1
+
+        int currentIndex = -1;
+
+        for (Iterator<Hit> i = hits.iterator(); i.hasNext();) {
+            Hit hit = i.next();
+
+            if (hit.isAuxiliary()) continue;
+
+            currentIndex++;
+            if (currentIndex < offset || currentIndex >= highBound) {
+                i.remove();
+                handleRemovedHit(hit);
+            }
+        }
+    }
+
+    /**
+     * Returns an iterator of the hits in this group.
+     * <p>
+     * This iterator is modifiable - removals will take effect in this group of hits.
+     */
+    public Iterator<Hit> iterator() {
+        updateHits();
+        ensureSorted();
+        return new HitIterator(this, hits);
+    }
+
+    /**
+     * Returns an iterator that does depth-first traversal of leaf hits of this group. Calling this method has the
+     * side-effect of sorting the internal list of hits.
+     *
+     * @return A modifiable iterator.
+     */
+    public Iterator<Hit> deepIterator() {
+        return new DeepHitIterator(iterator(), true);
+    }
+
+    /**
+     * Returns an iterator that does depth-first traversal of leaf hits of this group, in a potentially unsorted order.
+     * As opposed to {@link #deepIterator()}, this method has no side-effect.
+     *
+     * @return A modifiable iterator.
+     */
+    public Iterator<Hit> unorderedDeepIterator() {
+        return new DeepHitIterator(unorderedIterator(), false);
+    }
+
+    /** Returns a read only list view of the hits in this */
+    public List<Hit> asList() {
+        updateHits();
+        ensureSorted();
+        return unmodifiableHits;
+    }
+
+    /**
+     * Returns a read only list view of the hits in this which is potentially unsorted.
+     * Using this over getHits is potentially faster when a sorted view is not needed.
+     */
+    public List<Hit> asUnorderedHits() {
+        updateHits();
+        return unmodifiableHits;
+    }
+
+    /**
+     * Returns an iterator of the hits in this group in a potentially unsorted order.
+     * <p>
+     * Using this over getPreludeHitIterator is potentially faster when a sorted view is not needed.
+     * <p>
+     * This iterator is modifiable - removals will take effect in this group of hits.
+     */
+    public Iterator<Hit> unorderedIterator() {
+        updateHits();
+        return new HitIterator(this, hits);
+    }
+
+    /**
+     * Force hit sorting now.
+     * This is not normally useful because a group will stay sorted automatically,
+     * but it is in the case where
+     * the hits have changed their internal state in a way that should change ordering
+     */
+    public void sort() {
+        if (hitOrderer == null) {
+            Collections.sort(hits);
+            hitsSorted = true;
+        } else {
+            // This may or may not lead to a sorted result set, but
+            // it's a best effort
+            hitOrderer.order(hits);
+            if (likelyHitsHaveCorrectValueForSortFields()) {
+                hitsSorted = true;
+            }
+        }
+    }
+
+    private boolean likelyHitsHaveCorrectValueForSortFields() {
+        if (hitOrderer == null) {
+            return true;
+        } else {
+            Set<String> filledFields = getFilled();
+            return filledFields == null || !filledFields.isEmpty();
+        }
+    }
+
+    /**
+     * <p>Sets the hit orderer for this group.</p>
+     *
+     * @param hitOrderer the new hit orderer, or null to use default relevancy ordering
+     */
+    public void setOrderer(HitOrderer hitOrderer) {
+        this.hitOrderer = hitOrderer;
+        if (hits.size() > 1) {
+            hitsSorted = false;
+        }
+    }
+
+    /**
+     * Explicitly set whether the hits in this group are correctly sorted at this moment.
+     * If the contained hits are modified directly in a way that
+     * may break ordering, you should call setSorted(false).
+     */
+    public void setSorted(boolean sorted) {
+        this.hitsSorted = sorted;
+    }
+
+
+    /** Returns the orderer used by this group, or null if the default relevancy order is used */
+    public HitOrderer getOrderer() {
+        return hitOrderer;
+    }
+
+    public void setDeletionBreaksOrdering(boolean flag) { deletionBreaksOrdering = flag; }
+
+    public boolean getDeletionBreaksOrdering() { return deletionBreaksOrdering; }
+
+    /** Called before hit lists or positions are used */
+    private void ensureSorted() {
+        if ( ! orderedHits && ! hitsSorted && likelyHitsHaveCorrectValueForSortFields()) {
+            sort();
+        }
+    }
+
+    /**
+     * Returns true if all the hits recursively contained in this
+     * is cached
+     */
+    public @Override boolean isCached() {
+        if (notCachedCount<1) return true;
+        if (subgroupCount<1) return false; // No need to check below
+
+        // Else check recursively
+        for (Hit hit : hits) {
+            if (hit instanceof HitGroup) {
+                if (hit.isCached()) return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns whether all hits in this result have been filled with
+     * the properties contained in the given summary class. Note that
+     * this method will also return true if no hits in this result are
+     * fillable.
+     */
+    public boolean isFilled(String summaryClass) {
+        Set<String> filled = getFilled();
+        return (filled == null || filled.contains(summaryClass));
+    }
+
+
+    /**
+     * Sets sorting information to be the same as for the provided hitGroup.
+     * The contained hits should already be sorted in the order specified by
+     * the hitGroup given as argument.
+     */
+    public void copyOrdering(HitGroup hitGroup) {
+        setOrderer(hitGroup.getOrderer());
+        setDeletionBreaksOrdering(hitGroup.getDeletionBreaksOrdering());
+        setOrdered(hitGroup.orderedHits);
+    }
+
+    // -------------- State bookkeeping
+
+    /** Ensures result invariants. Must be called when a hit is added to this result. */
+    private void handleNewHit(Hit hit) {
+        if (!hit.isAuxiliary())
+            concreteHitCount++;
+
+        if (hit.getAddNumber() < 0) {
+            hit.setAddNumber(size());
+        }
+
+        hitsSorted = false;
+        Set<String> hitFilled = hit.getFilled();
+
+        if (hitFilled != null) {
+            Set<String> filled = getFilledInternal();
+            if (filled == null) {
+                if (hitFilled.isEmpty()) {
+                    filled = null;
+                } else if (hitFilled.size() == 1) {
+                    filled = Collections.singleton(hitFilled.iterator().next());
+                } else {
+                    filled = new HashSet<>(hitFilled);
+                }
+                setFilledInternal(filled);
+            } else {
+                if (filled.size() == 1) {
+                    if ( ! hitFilled.contains(filled.iterator().next())) {
+                        filled = null; // No intersection
+                        setFilledInternal(filled);
+                    }
+                } else {
+                    filled.retainAll(hitFilled);
+                }
+            }
+        }
+
+        if (hit instanceof HitGroup) {
+            subgroupCount++;
+        }
+        if (!hit.isCached()) {
+            notCachedCount++;
+        }
+    }
+
+    // Filled is not kept in sync at removal
+    private void handleRemovedHit(Hit hit) {
+        if (!hit.isAuxiliary()) {
+            concreteHitCount--;
+            if (!hit.isCached())
+                notCachedCount--;
+        }
+        else if (hit instanceof HitGroup) {
+            subgroupCount--;
+        }
+
+        if (deletionBreaksOrdering) {
+            hitsSorted = false;
+        }
+    }
+
+    private void analyzeHit(Hit hit) {
+        if (hit instanceof HitGroup) {
+            ((HitGroup)hit).analyze();
+        }
+        if (!hit.isAuxiliary())
+            concreteHitCount++;
+
+        if (!hit.isCached())
+            notCachedCount++;
+    }
+
+    /**
+     * Update concreteHitCount, cached and filled by iterating trough the hits of this result.
+     * Recursively also update all subgroups.
+     */
+    public void analyze() {
+        concreteHitCount=0;
+        setFilledInternal(null);
+        notCachedCount=0;
+        Set<String> filled = getFilledInternal();
+
+        Iterator<Hit> i = unorderedIterator();
+        while (filled == null && i.hasNext()) {
+            Hit hit = i.next();
+            analyzeHit(hit);
+            Set<String> hitFilled = hit.getFilled();
+            if (hitFilled != null) {
+                filled = (hitFilled.size() == 1)
+                        ? Collections.singleton(hitFilled.iterator().next())
+                        : hitFilled.isEmpty() ? null : new HashSet<>(hitFilled);
+                setFilledInternal(filled);
+            }
+        }
+        String singleKey = null;
+        if (filled != null && filled.size() == 1) {
+            singleKey = filled.iterator().next();
+        }
+
+
+        for (; i.hasNext();) {
+            Hit hit = i.next();
+            analyzeHit(hit);
+
+            if (filled != null) {
+                Set<String> hitFilled = hit.getFilled();
+                if (hitFilled == null) {
+                    // Intentionally empty. Strange semantic, null -> matches everything
+                } else if (hitFilled.isEmpty()) {
+                    filled = null; // No intersection
+                    setFilledInternal(filled);
+                } else {
+                    if (filled.size() == 1) {
+                        if ( ! hitFilled.contains(singleKey)) {
+                            filled = null; // No intersection
+                            setFilledInternal(filled);
+                            singleKey = null;
+                        }
+                    } else {
+                        filled.retainAll(hitFilled);
+                        if (filled.size() == 1) {
+                            singleKey = filled.iterator().next();
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public HitGroup clone() {
+        HitGroup hitGroupClone = (HitGroup) super.clone();
+        hitGroupClone.hits = new ListenableArrayList<>(this.hits.size());
+        hitGroupClone.unmodifiableHits = Collections.unmodifiableList(hitGroupClone.hits);
+        for (Iterator<Hit> i = this.hits.iterator(); i.hasNext();) {
+            Hit hitClone = i.next().clone();
+            hitGroupClone.hits.add(hitClone);
+        }
+        if (this.errorHit!=null) { // Find the cloned error and assign it
+            for (Hit hit : hitGroupClone.asList()) {
+                if (hit instanceof ErrorHit)
+                    hitGroupClone.errorHit=(ErrorHit)hit;
+            }
+        }
+
+        if (this.getFilledInternal()!=null) {
+            hitGroupClone.setFilledInternal(new HashSet<>(this.getFilledInternal()));
+        }
+
+        return hitGroupClone;
+    }
+
+    @Override
+    public void setFillable() {}
+
+    /** Ignored as this should always be derived from the content hits */
+    @Override
+    public void setFilled(String summaryClass) {}
+
+    @Override
+    public boolean isFillable() {
+        return fillableHits().iterator().hasNext();
+    }
+
+    @Override
+    public Set<String> getFilled() {
+        Iterator<Hit> hitIterator = hits.iterator();
+        Set<String> firstSummaryNames = getSummaryNamesNextFilledHit(hitIterator);
+        if (firstSummaryNames == null || firstSummaryNames.isEmpty())
+            return firstSummaryNames;
+
+        Set<String> intersection = firstSummaryNames;
+        while (true) {
+            Set<String> summaryNames = getSummaryNamesNextFilledHit(hitIterator);
+            if (summaryNames == null)
+                break;
+
+            if (intersection.size() == 1)
+                return getFilledSingle(first(intersection), hitIterator);
+
+
+            boolean notInSet = false;
+            if (intersection == firstSummaryNames) {
+                if (intersection.size() == summaryNames.size()) {
+                    for(String s : summaryNames) {
+                        if ( ! intersection.contains(s)) {
+                            intersection = new HashSet<>(firstSummaryNames);
+                            notInSet = true;
+                            break;
+                        }
+                    }
+                }
+            }
+            if (notInSet) {
+                intersection.retainAll(summaryNames);
+            }
+
+        }
+
+        return intersection;
+    }
+
+    private Set<String> getSummaryNamesNextFilledHit(Iterator<Hit> hitIterator) {
+        while (hitIterator.hasNext()) {
+            Set<String> filled = hitIterator.next().getFilled();
+            if (filled != null)
+                return filled;
+        }
+        return null;
+    }
+
+    private Set<String> getFilledSingle(String summaryName, Iterator<Hit> hitIterator) {
+        while (true) {
+            Set<String> summaryNames = getSummaryNamesNextFilledHit(hitIterator);
+            if (summaryNames == null) {
+                return Collections.singleton(summaryName);
+            } else if (!summaryNames.contains(summaryName)) {
+                return Collections.emptySet();
+            }
+        }
+    }
+
+    private Iterable<Hit> fillableHits() {
+        Predicate<Hit> isFillable = hit -> hit.isFillable();
+
+        return Iterables.filter(hits, isFillable);
+    }
+
+    /** Returns the incoming hit buffer to which new hits can be added to this asynchronous, if supported by the instance */
+    @Override
+    public IncomingData<Hit> incoming() { return incomingHits; }
+
+    @Override
+    public ListenableFuture<DataList<Hit>> complete() { return completedFuture; }
+
+    @Override
+    public void addDataListener(Runnable runnable) {
+        hits.addListener(runnable);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroupsLastComparator.java b/container-search/src/main/java/com/yahoo/search/result/HitGroupsLastComparator.java
new file mode 100644
index 00000000000..0fe73a5afb5
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/HitGroupsLastComparator.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.Comparator;
+
+/**
+ * Ensures that HitGroups are placed last in the result.
+ *
+ * @author tonytv
+ */
+public class HitGroupsLastComparator extends ChainableComparator {
+
+    public HitGroupsLastComparator(Comparator<Hit> secondaryComparator) {
+        super(secondaryComparator);
+    }
+
+    @Override
+    public int compare(Hit left, Hit right) {
+        if (isHitGroup(left) ^ isHitGroup(right)) {
+            return isHitGroup(left) ? 1 : -1;
+        } else {
+            return super.compare(left, right);
+        }
+    }
+
+    private boolean isHitGroup(Hit hit) {
+        return hit instanceof HitGroup;
+    }
+
+    @Override
+    public String toString() {
+        return getSecondaryComparator().toString();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/HitIterator.java b/container-search/src/main/java/com/yahoo/search/result/HitIterator.java
new file mode 100644
index 00000000000..adf642a28ec
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/HitIterator.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import com.yahoo.search.Result;
+
+
+/**
+ * An iterator for the list of hits in a result. This iterator supports the remove operation.
+ *
+ * @author  <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class HitIterator implements Iterator<Hit> {
+
+    /** The index into the list of hits */
+    private int index = -1;
+
+    /** The list of hits to iterate over */
+    private List<Hit> hits = null;
+
+    /** The result the hits belong to */
+    private HitGroup hitGroup = null;
+
+    /** Whether the iterator is in a state where remove is OK */
+    private boolean canRemove = false;
+
+    public HitIterator(HitGroup hitGroup, List<Hit> hits) {
+        this.hitGroup = hitGroup;
+        this.hits = hits;
+    }
+
+    public HitIterator(Result result, List<Hit> hits) {
+        this.hitGroup = result.hits();
+        this.hits = hits;
+    }
+
+    public boolean hasNext() {
+        if (hits.size() > (index + 1)) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public Hit next() throws NoSuchElementException {
+        if (hits.size() <= (index + 1)) {
+            throw new NoSuchElementException();
+        } else {
+            canRemove = true;
+            return hits.get(++index);
+        }
+    }
+
+    public void remove() throws IllegalStateException {
+        if (!canRemove) {
+            throw new IllegalStateException();
+        }
+        hitGroup.remove(index);
+        index--;
+        canRemove = false;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/HitOrderer.java b/container-search/src/main/java/com/yahoo/search/result/HitOrderer.java
new file mode 100644
index 00000000000..5982a93d86a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/HitOrderer.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * A class capable of ordering a list of hits
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a>
+ */
+
+public abstract class HitOrderer {
+
+    /** Orders the given list of hits */
+    public abstract void order(List<Hit> hits);
+
+    /**
+     * Returns the Comparator that this HitOrderer uses internally to
+     * sort hits. Returns null if no Comparator is used.
+     * <p>
+     * This default implementation returns null.
+     *
+     * @return the Comparator used to order hits, or null
+     */
+    public Comparator<Hit> getComparator() {
+        return null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/HitSortOrderer.java b/container-search/src/main/java/com/yahoo/search/result/HitSortOrderer.java
new file mode 100644
index 00000000000..c532aba99d8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/HitSortOrderer.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.search.query.Sorting;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * A hit orderer which can be assigned to a HitGroup to keep that group's
+ * hit sorted in accordance with the sorting specification given when this is created.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class HitSortOrderer extends HitOrderer {
+
+    private final Comparator<Hit> fieldComparator;
+
+    /** Create a sort order from a sorting */
+    public HitSortOrderer(Sorting sorting) {
+        fieldComparator =
+                new MetaHitsFirstComparator(
+                        new HitGroupsLastComparator(
+                                new FieldComparator(sorting)));
+    }
+
+    /**
+     * Create a sort order from a comparator.
+     * This will be appended to the standard comparators used by this.
+     */
+    public HitSortOrderer(Comparator<Hit> comparator) {
+        fieldComparator = new MetaHitsFirstComparator(new HitGroupsLastComparator(comparator));
+    }
+
+    /**
+     * Orders the given list of hits according to the sorting given at construction
+     *
+     * Meta hits are sorted before concrete hits, but have no internal
+     * ordering. The sorting is stable.
+     */
+    public void order(List<Hit> hits) {
+        Collections.sort(hits, fieldComparator);
+    }
+
+    public Comparator<Hit> getComparator() {
+        return fieldComparator;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/MetaHitsFirstComparator.java b/container-search/src/main/java/com/yahoo/search/result/MetaHitsFirstComparator.java
new file mode 100644
index 00000000000..900f47da6e4
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/MetaHitsFirstComparator.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.Comparator;
+
+/**
+ * Ensures that meta hits are sorted before normal hits. All meta hits are
+ * considered equal.
+ *
+ * @author tonytv
+ */
+public class MetaHitsFirstComparator extends ChainableComparator {
+
+    public MetaHitsFirstComparator(Comparator<Hit> secondaryComparator) {
+        super(secondaryComparator);
+    }
+
+    @Override
+    public int compare(Hit left, Hit right) {
+        if (left.isMeta() && right.isMeta()) {
+            return 0;
+        } else if (left.isMeta()) {
+            return -1;
+        } else if (right.isMeta()) {
+            return 1;
+        } else {
+            return super.compare(left, right);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return getSecondaryComparator().toString();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/NanNumber.java b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java
new file mode 100644
index 00000000000..385be70cd4c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/NanNumber.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+/**
+ * A class representing unset or undefined numeric values.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@SuppressWarnings("serial")
+public final class NanNumber extends Number {
+    public static final NanNumber NaN = new NanNumber();
+
+    private NanNumber() {
+    }
+
+    @Override
+    public double doubleValue() {
+        return Double.NaN;
+    }
+
+    @Override
+    public float floatValue() {
+        return Float.NaN;
+    }
+
+    @Override
+    public int intValue() {
+        return 0;
+    }
+
+    @Override
+    public long longValue() {
+        return 0L;
+    }
+
+    @Override
+    public String toString() {
+        return "";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/Relevance.java b/container-search/src/main/java/com/yahoo/search/result/Relevance.java
new file mode 100644
index 00000000000..df79b64585e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/Relevance.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.text.DoubleFormatter;
+
+/**
+ * A relevance double value. These values should always be normalized between 0 and 1 (where 1 means perfect),
+ * however, this is not enforced.
+ * <p>
+ * Sources may create subclasses of this to include additional information or functionality.
+ *
+ * @author bratseth
+ */
+public class Relevance implements Comparable<Relevance> {
+
+    private static final long serialVersionUID = 4536685722731661704L;
+
+    /** The relevancy score. */
+    private double score;
+
+    /**
+     * Construct a relevancy object with an initial value.
+     * This initial value should ideally be a normalized value
+     * between 0 and 1, but that is not enforced.
+     *
+     * @param score the inital value (rank score)
+     */
+    public Relevance(double score) {
+        this.score=score;
+    }
+
+    /**
+     * Set score value to this value. This should ideally be a
+     * normalized value between 0 and 1, but that is not enforced.
+     * NaN is also a legal value, for elements where it makes no sense to assign a particular value.
+     */
+    public void setScore(double score) { this.score = score; }
+
+    /**
+     * Returns the relevancy score of this, preferably a normalized value
+     * between 0 and 1 but this is not guaranteed by this framework
+     */
+    public double getScore() { return score; }
+
+    /**
+     * Returns the score value as a string
+     */
+    public @Override String toString() {
+        return DoubleFormatter.stringValue(score);
+    }
+
+    /** Compares relevancy values with */
+    public int compareTo(Relevance other) {
+        double thisScore = getScore();
+        double otherScore = other.getScore();
+        if (Double.isNaN(thisScore)) {
+            if (Double.isNaN(otherScore)) {
+                return 0;
+            } else {
+                return -1;
+            }
+        } else if (Double.isNaN(otherScore)) {
+            return 1;
+        } else {
+            return Double.compare(thisScore, otherScore);
+        }
+    }
+
+    /** Compares relevancy values */
+    public @Override boolean equals(Object object) {
+        if (object==this) return true;
+
+        if (!(object instanceof Relevance)) { return false; }
+
+        Relevance other = (Relevance) object;
+        return getScore() == other.getScore();
+    }
+
+    /** Returns a hash from the relevancy value */
+    public @Override int hashCode() {
+        double hash=getScore()*335451367; // A largish prime
+        if (hash>-1 && hash<1) hash=1/hash;
+        return (int) hash;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/StructuredData.java b/container-search/src/main/java/com/yahoo/search/result/StructuredData.java
new file mode 100644
index 00000000000..c49f8a04b97
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/StructuredData.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import com.yahoo.data.access.Inspector;
+import com.yahoo.data.access.Inspectable;
+import com.yahoo.data.access.simple.JsonRender;
+import com.yahoo.data.JsonProducer;
+import com.yahoo.data.XmlProducer;
+import com.yahoo.prelude.hitfield.XmlRenderer;
+
+/**
+ * A wrapper for structured data representing feature values.
+ */
+public class StructuredData implements Inspectable, JsonProducer, XmlProducer {
+
+    private final Inspector value;
+
+    public StructuredData(Inspector value) {
+        this.value = value;
+    }
+
+    @Override
+    public Inspector inspect() {
+        return value;
+    }
+
+    public String toString() {
+        return toXML();
+    }
+
+    @Override
+    public String toXML() {
+        return writeXML(new StringBuilder()).toString();
+    }
+
+    @Override
+    public StringBuilder writeXML(StringBuilder target) {
+        return XmlRenderer.render(target, value);
+    }
+
+    @Override
+    public String toJson() {
+        return writeJson(new StringBuilder()).toString();
+    }
+
+    @Override
+    public StringBuilder writeJson(StringBuilder target) {
+        return JsonRender.render(value, target, true);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/Templating.java b/container-search/src/main/java/com/yahoo/search/result/Templating.java
new file mode 100644
index 00000000000..61dd38aaf93
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/Templating.java
@@ -0,0 +1,210 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.result;
+
+import java.util.Map;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.prelude.templates.SearchRendererAdaptor;
+import com.yahoo.prelude.templates.TemplateSet;
+import com.yahoo.prelude.templates.UserTemplate;
+import com.yahoo.processing.rendering.Renderer;
+import com.yahoo.search.Result;
+import com.yahoo.search.query.Presentation;
+
+/**
+ * Helper methods and data store for result attributes geared towards result
+ * rendering and presentation.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class Templating {
+
+    private final Result result;
+    private Renderer<Result> renderer;
+
+    public Templating(Result result) {
+        super();
+        this.result = result;
+    }
+
+    /**
+     * Returns The first hit presented in the result as an index into the global
+     * list of all hits generated by the user query.
+     */
+    public int getFirstHitNo() {
+        return result.getQuery().getOffset() + 1;
+    }
+
+    /**
+     * Returns the first hit of the next result page, 0 if there aren't any more
+     * hits available
+     */
+    public long getNextFirstHitNo() {
+        if (result.getQuery().getHits() > result.getConcreteHitCount()) {
+            return 0;
+        }
+
+        return Math.min(getLastHitNo() + 1, result.getTotalHitCount());
+    }
+
+    /**
+     * Returns the first hit of the next result page, 0 if there aren't any more
+     * hits available
+     */
+    public long getNextLastHitNo() {
+        if (result.getQuery().getHits() > result.getConcreteHitCount()) {
+            return 0;
+        }
+
+        return Math.min(getLastHitNo() + result.getConcreteHitCount(), result.getTotalHitCount());
+    }
+
+    /**
+     * Returns the number of the last result of the current hit page.
+     */
+    public int getLastHitNo() {
+        return getFirstHitNo() + result.getConcreteHitCount() - 1;
+    }
+
+    /**
+     * The first hit presented on the previous result page as an index into the
+     * global list of all hits generated by the user query
+     */
+    public int getPrevFirstHitNo() {
+        return Math.max(getFirstHitNo() - result.getQuery().getHits(), 1);
+    }
+
+    /**
+     * The last hit presented on the previous result page as an index into the
+     * global list of all hits generated by the user query
+     */
+    public int getPrevLastHitNo() {
+        return Math.max(getFirstHitNo() - 1, 0);
+    }
+
+    /**
+     * An URL that may be used to obtain the next result page.
+     */
+    public String getNextResultURL() {
+        HttpRequest request = result.getQuery().getHttpRequest();
+        StringBuilder nextURL = new StringBuilder();
+
+        nextURL.append(getPath(request)).append("?");
+        parametersExceptOffset(request, nextURL);
+
+        int offset = getLastHitNo();
+
+        nextURL.append("&").append("offset=").append(Integer.toString(offset));
+        return nextURL.toString();
+    }
+
+    /**
+     * An URL that may be used to obtain the previous result page.
+     */
+    public String getPreviousResultURL() {
+        HttpRequest request = result.getQuery().getHttpRequest();
+        StringBuilder prevURL = new StringBuilder();
+
+        prevURL.append(getPath(request)).append("?");
+        parametersExceptOffset(request, prevURL);
+        int offset = getPrevFirstHitNo() - 1;
+        prevURL.append("&").append("offset=").append(Integer.toString(offset));
+        return prevURL.toString();
+    }
+
+    public String getCurrentResultURL() {
+        HttpRequest request = result.getQuery().getHttpRequest();
+        StringBuilder thisURL = new StringBuilder();
+
+        thisURL.append(getPath(request)).append("?");
+        parameters(request, thisURL);
+        return thisURL.toString();
+    }
+
+    private String getPath(HttpRequest request) {
+        String path = request.getUri().getPath();
+        if (path == null) {
+            path = "";
+        }
+        return path;
+    }
+
+    private void parametersExceptOffset(HttpRequest request, StringBuilder nextURL) {
+        int startLength = nextURL.length();
+        for (Map.Entry<String, String> property : request.propertyMap().entrySet()) {
+            if (property.getKey().equals("offset")) continue;
+
+            if (nextURL.length() > startLength)
+                nextURL.append("&");
+            nextURL.append(property.getKey()).append("=").append(property.getValue());
+        }
+    }
+
+    private void parameters(HttpRequest request, StringBuilder nextURL) {
+        int startLength = nextURL.length();
+        for (Map.Entry<String, String> property : request.propertyMap().entrySet()) {
+            if (nextURL.length() > startLength)
+                nextURL.append("&");
+            nextURL.append(property.getKey()).append("=").append(property.getValue());
+        }
+    }
+
+    /**
+     * Returns the templates which will render the result. This is never null.
+     * If default rendering is used, it is a TemplateSet containing no
+     * templates.
+     */
+    @SuppressWarnings("rawtypes")
+    public UserTemplate getTemplates() {
+        if (renderer == null) {
+            return TemplateSet.getDefault();
+        } else if (renderer instanceof SearchRendererAdaptor) {
+            return ((SearchRendererAdaptor) renderer).getAdaptee();
+        } else {
+            throw new RuntimeException(
+                    "Please use getTemplate() instead of getTemplates() when using the new template api.");
+        }
+    }
+
+    /**
+     * Sets the template set which should render this result set
+     *
+     * @param templates
+     *                the templates which should render this result, or null to
+     *                use the default xml rendering
+     */
+    @SuppressWarnings("deprecation")
+    public void setTemplates(@SuppressWarnings("rawtypes") UserTemplate templates) {
+        if (templates == null) {
+            setTemplates(TemplateSet.getDefault());
+        } else {
+            setRenderer(new SearchRendererAdaptor(templates));
+        }
+    }
+
+    /**
+     * @deprecated since 5.1.21, use {@link Presentation#getRenderer()}
+     */
+    @Deprecated // OK Do not remove on Vespa 6. Remove when we move everything having to do with templates
+    public Renderer<Result> getRenderer() {
+        return renderer;
+    }
+
+    /**
+     * @deprecated since 5.1.21, use {@link Presentation#setRenderer(com.yahoo.component.ComponentSpecification)}
+     */
+    @Deprecated // OK Do not remove on Vespa 6. Remove when we move everything having to do with templates
+    public void setRenderer(Renderer<Result> renderer) {
+        this.renderer = renderer;
+    }
+
+    /**
+     * For internal use only.
+     */
+    public boolean usesDefaultTemplate() {
+        return renderer == null ||
+                (renderer instanceof SearchRendererAdaptor &&
+                ((SearchRendererAdaptor) renderer).getAdaptee().isDefaultTemplateSet());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/result/package-info.java b/container-search/src/main/java/com/yahoo/search/result/package-info.java
new file mode 100644
index 00000000000..aa93d0fdeab
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/result/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * The content of a Result produced in response to a Query.
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.result;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java
new file mode 100644
index 00000000000..e1794a73a93
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java
@@ -0,0 +1,204 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.concurrent.ThreadFactoryFactory;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.*;
+
+/**
+ * Provides asynchronous execution of searchchains.
+ *
+ * <p>
+ * AsyncExecution is implemented as an asynchronous wrapper around Execution
+ * that returns Future.
+ * </p>
+ *
+ * This is used in the following way
+ *
+ * <pre>
+ * Execution execution = new Execution(searchChain, context);
+ * AsyncExecution asyncExecution = new AsyncExecution(execution);
+ * Future&lt;Result&gt; future = asyncExecution.search(query)
+ * try {
+ *     result = future.get(timeout, TimeUnit.milliseconds);
+ * } catch(TimeoutException e) {
+ *     // Handle timeout
+ * }
+ * </pre>
+ *
+ * <p>
+ * Note that the query is not a thread safe object and cannot be shared between
+ * multiple concurrent executions - a clone() must be made, or a new query
+ * created for each AsyncExecution instance.
+ * </p>
+ *
+ * @see com.yahoo.search.searchchain.Execution
+ * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ */
+public class AsyncExecution {
+
+    private static final ThreadFactory threadFactory = ThreadFactoryFactory.getThreadFactory("search");
+
+    private static final Executor executorMain = createExecutor();
+
+    private static Executor createExecutor() {
+        ThreadPoolExecutor executor = new ThreadPoolExecutor(100, Integer.MAX_VALUE, 1L, TimeUnit.SECONDS,
+                                                            new SynchronousQueue<>(false), threadFactory);
+        // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also
+        // get the dreaded thread locals initialized even if they will never run.
+        // That counters what we we want to achieve with the Q that will prefer thread locality.
+        executor.prestartAllCoreThreads();
+        return executor;
+    }
+
+    /** The execution this executes */
+    private final Execution execution;
+
+    /**
+     * Creates an async execution.
+     *
+     * @param chain the chain to execute
+     * @param execution the execution holding the context of this
+     */
+    public AsyncExecution(Chain<? extends Searcher> chain, Execution execution) {
+        this(execution.context(), chain);
+    }
+
+    /**
+     * Creates an async execution.
+     *
+     * @param chain the chain to execute
+     * @param context the the context of this
+     */
+    public AsyncExecution(Chain<? extends Searcher> chain, Execution.Context context) {
+        this(context, chain);
+    }
+
+    /**
+     * <p>
+     * Creates an async execution from an existing execution. This async
+     * execution will execute the chain from the given execution, <i>starting
+     * from the next searcher in that chain.</i> This is handy to execute
+     * multiple queries to the rest of the chain in parallel. If the Execution
+     * is freshly instantiated, the search will obviously start from the first
+     * searcher.
+     * </p>
+     *
+     * <p>
+     * The state of the given execution is read on construction of this and not
+     * used later - the argument execution can be reused for other purposes.
+     * </p>
+     *
+     * @param execution the execution from which the state of this is created
+     *
+     * @see Execution#Execution(Chain, com.yahoo.search.searchchain.Execution.Context)
+     * @see #AsyncExecution(Chain, Execution)
+     */
+    public AsyncExecution(Execution execution) {
+        this.execution = new Execution(execution);
+    }
+
+    private AsyncExecution(Execution.Context context, Chain<? extends Searcher> chain) {
+        this.execution = new Execution(chain, context);
+    }
+
+    /**
+     * Does an async search, note that the query argument cannot simultaneously
+     * be used to execute any other searches, a clone() must be made of the
+     * query for each async execution if the same query is to be used in more
+     * than one.
+     *
+     * @see com.yahoo.search.searchchain.Execution
+     */
+    public FutureResult search(final Query query) {
+        return getFutureResult(() -> execution.search(query), query);
+    }
+
+    public FutureResult searchAndFill(final Query query) {
+        return getFutureResult(() -> {
+            Result result = execution.search(query);
+            execution.fill(result, query.getPresentation().getSummary());
+            return result;
+        }, query);
+    }
+
+    private static Executor getExecutor() {
+        return executorMain;
+    }
+
+    /**
+     * The future of this functions returns the original Result
+     *
+     * @see com.yahoo.search.searchchain.Execution
+     */
+    public FutureResult fill(final Result result, final String summaryClass) {
+        return getFutureResult(() -> {
+            execution.fill(result, summaryClass);
+            return result;
+        }, result.getQuery());
+
+    }
+
+    private static <T> Future<T> getFuture(Callable<T> callable) {
+        final FutureTask<T> future = new FutureTask<>(callable);
+        getExecutor().execute(future);
+        return future;
+    }
+
+    private static Future<Void> runTask(Runnable runnable) {
+        return getFuture(() -> {
+            runnable.run();
+            return null;
+        });
+    }
+
+    private FutureResult getFutureResult(Callable<Result> callable, Query query) {
+        FutureResult future = new FutureResult(callable, execution, query);
+        getExecutor().execute(future);
+        return future;
+    }
+
+    /*
+     * Waits for all futures until the given timeout. If a FutureResult isn't
+     * done when the timeout expires, it will be cancelled, and it will return a
+     * result. All unfinished Futures will be cancelled.
+     *
+     * @return the list of results in the same order as returned from the task
+     * collection
+     */
+    public static List<Result> waitForAll(Collection<FutureResult> tasks, long timeoutMs) {
+
+        // Copy the list in case it is modified while we are waiting
+        final List<FutureResult> workingTasks = new ArrayList<>(tasks);
+        try {
+            runTask(() -> {
+                for (FutureResult task : workingTasks)
+                    task.get();
+            }).get(timeoutMs, TimeUnit.MILLISECONDS);
+        }catch (TimeoutException | InterruptedException | ExecutionException e) {
+            // Handle timeouts below
+        }
+
+        final List<Result> results = new ArrayList<>(tasks.size());
+        for (FutureResult atask : workingTasks) {
+            Result result;
+            if (atask.isDone() && !atask.isCancelled()) {
+                result = atask.get(); // Since isDone() = true, this won't
+                                      // block.
+            } else { // Not done and no errors thrown
+                result = new Result(atask.getQuery(),
+                        atask.createTimeoutError());
+            }
+            results.add(result);
+        }
+        return results;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
new file mode 100644
index 00000000000..a888ad9b59e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
@@ -0,0 +1,672 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.language.Linguistics;
+import com.yahoo.log.LogLevel;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.Ping;
+import com.yahoo.prelude.Pong;
+import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.processing.Processor;
+import com.yahoo.processing.Request;
+import com.yahoo.processing.Response;
+import com.yahoo.protect.Validator;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.cluster.PingableSearcher;
+import com.yahoo.search.rendering.RendererRegistry;
+import com.yahoo.search.statistics.TimeTracker;
+
+import java.util.logging.Logger;
+
+/**
+ * <p>An execution of a search chain. This keeps track of the call state for an execution (in the calling thread)
+ * of the searchers of a search chain.</p>
+ *
+ * <p>To execute a search chain, simply do
+ * <pre>
+ *     Result result = new Execution(mySearchChain, execution.context()).search(query)
+ * </pre>
+ *
+ *
+ * <p>See also {@link AsyncExecution}, which performs an execution in a separate thread than the caller.</p>
+ *
+ * <p>Execution instances should not be reused for multiple separate executions.</p>
+ *
+ * @author bratseth
+ */
+public class Execution extends com.yahoo.processing.execution.Execution {
+
+    public static final String ATTRIBUTEPREFETCH = "attributeprefetch";
+
+    /**
+     * The execution context is the search chain's current view of the indexes,
+     * search chain registrys, etc. Searcher instances may set values here to
+     * change the behavior of the rest of the search chain.
+     * <p>
+     * The Context class simply carries a set of objects which define the
+     * environment for the search. <b>Important:</b> All objects available through context need to
+     * be either truly immutable or support the freeze pattern.
+     * <p>
+     * If you are implementing a searcher where you need to create a new Context
+     * instance to create an Execution, you should use the context from the
+     * execution the searcher was invoked from. You can also copy
+     * (Context.shallowCopy()) the incoming context if it is necessary to do
+     * more. In other words, a minimal example would be:<br>
+     * new Execution(searchChain, execution.context())
+     */
+    public static final class Context {
+
+        /**
+         * Whether the search should perform detailed diagnostics.
+         */
+        private boolean detailedDiagnostics = false;
+
+        /**
+         * Whether the container was considered to be in a breakdown state when
+         * this query started.
+         */
+        private boolean breakdown = false;
+
+        /**
+         * The search chain registry current when this execution was created, or
+         * when the registry was first accessed, or null if it was not set on
+         * creation or has been accessed yet. No setter method is intentional.
+         */
+        private SearchChainRegistry searchChainRegistry = null;
+
+        private IndexFacts indexFacts = null;
+
+        /**
+         * The current set of special tokens.
+         */
+        private SpecialTokenRegistry tokenRegistry = null;
+
+        /**
+         * The current template registry.
+         */
+        private RendererRegistry rendererRegistry = null;
+
+        /**
+         * The current linguistics.
+         */
+        private Linguistics linguistics = null;
+
+        /** Always set if this context belongs to an execution, never set if it does not. */
+        private final Execution owner;
+
+        // Please don't add more constructors to the public interface of Context
+        // unless the constructor is reasonably safe for an inexperienced user
+        // in a production setting. Since queries blow up in a spectacular
+        // fashion if Context is in a bad state, the Context() constructor is
+        // package private.
+
+        /** Create a context used to carry state into another context */
+        Context() { this.owner=null; }
+
+        /** Create a context which belongs to an execution */
+        Context(Execution owner) { this.owner=owner; }
+
+        /**
+         * Creates a context from arguments, all of which may be null, though
+         * this can be risky. If you are doing this outside a test, it is
+         * usually better to do something like execution.context().shallowCopy()
+         * instead, and then set the fields you need to change. It is also safe
+         * to use the context from the incoming execution directly. In other
+         * words, a plug-in writer should practically never construct a Context
+         * instance directly.
+         * <p>
+         * This context is never attached to an execution but is used to carry state into
+         * another context.
+         */
+        public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts,
+                       SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics)
+        {
+            owner=null;
+            // The next time something is added here, compose into wrapper objects. Many arguments...
+
+            // Four methods need to be updated when adding something:
+            // fill(Context), populateFrom(Context), equals(Context) and,
+            // obviously, the most complete constructor.
+            this.searchChainRegistry = searchChainRegistry;
+            this.indexFacts = indexFacts;
+            this.tokenRegistry = tokenRegistry;
+            this.rendererRegistry = rendererRegistry;
+            this.linguistics = linguistics;
+        }
+
+        /** Creates a context stub with no information. This is for unit testing. */
+        public static Context createContextStub() {
+            return new Context(null, null, null, null, null);
+        }
+
+        /**
+         * Create a Context instance where only the index related settings are
+         * initialized. This is for unit testing.
+         */
+        public static Context createContextStub(IndexFacts indexFacts) {
+            return new Context(null, indexFacts, null, null, null);
+        }
+
+        /**
+         * Create a Context instance where only the search chain registry and index facts are
+         * initialized. This is for unit testing.
+         */
+        public static Context createContextStub(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts) {
+            return new Context(searchChainRegistry, indexFacts, null, null, null);
+        }
+
+        /**
+         * Create a Context instance where only the search chain registry, index facts and linguistics are
+         * initialized. This is for unit testing.
+         */
+        public static Context createContextStub(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, Linguistics linguistics) {
+            return new Context(searchChainRegistry, indexFacts, null, null, linguistics);
+        }
+
+        /**
+         * Populate missing values in this from the given context.
+         * Values which are non-null in this will not be overwritten.
+         *
+         * @param sourceContext the context from which to get the parameters
+         */
+        public void populateFrom(Context sourceContext) {
+            // breakdown and detailedDiagnostics has no unset state, so they are always copied
+            detailedDiagnostics = sourceContext.detailedDiagnostics;
+            breakdown = sourceContext.breakdown;
+            if (indexFacts == null) {
+                indexFacts = sourceContext.indexFacts;
+            }
+            if (tokenRegistry == null) {
+                tokenRegistry = sourceContext.tokenRegistry;
+            }
+            if (searchChainRegistry == null) {
+                searchChainRegistry = sourceContext.searchChainRegistry;
+            }
+            if (rendererRegistry == null) {
+                rendererRegistry = sourceContext.rendererRegistry;
+            }
+            if (linguistics == null) {
+                linguistics = sourceContext.linguistics;
+            }
+        }
+
+        /**
+         * The brutal version of populateFrom().
+         *
+         * @param other a Context instance this will copy all state from
+         */
+        void fill(Context other) {
+            searchChainRegistry = other.searchChainRegistry;
+            indexFacts = other.indexFacts;
+            tokenRegistry = other.tokenRegistry;
+            rendererRegistry = other.rendererRegistry;
+            detailedDiagnostics = other.detailedDiagnostics;
+            breakdown = other.breakdown;
+            linguistics = other.linguistics;
+        }
+
+        public boolean equals(Context other) {
+            // equals() needs to be cheap, that's yet another reason we can only
+            // allow immutables and frozen objects in the context
+            return other.indexFacts == indexFacts
+                    && other.rendererRegistry == rendererRegistry
+                    && other.tokenRegistry == tokenRegistry
+                    && other.searchChainRegistry == searchChainRegistry
+                    && other.detailedDiagnostics == detailedDiagnostics
+                    && other.breakdown == breakdown
+                    && other.linguistics == linguistics;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (other == null) {
+                return false;
+            }
+            if (other.getClass() != Context.class) {
+                return false;
+            } else {
+                return equals((Context) other);
+            }
+        }
+
+        /**
+         * Standard shallow copy, the new instance will carry the same
+         * references as this.
+         *
+         * @return a new instance which is a shallow copy
+         */
+        public Context shallowCopy() {
+            Context c = new Context();
+            c.fill(this);
+            return c;
+        }
+
+        /**
+         * This is used when building the Context stack. If Context has been
+         * changed since last time, build a new object. Otherwise simply return
+         * the previous snapshot.
+         *
+         * @param previous another Context instance to compare with
+         * @return a copy of this, or previous
+         */
+        Context copyIfChanged(Context previous) {
+            if (equals(previous)) {
+                return previous;
+            } else {
+                return shallowCopy();
+            }
+        }
+
+        /**
+         * Returns information about the indexes specified by the search definitions
+         * used in this system, or null if not know.
+         */
+        // TODO: Make a null default instance
+        public IndexFacts getIndexFacts() {
+            return indexFacts;
+        }
+
+        /**
+         * Use this to override index settings for the searchers below
+         * a given searcher, the easiest way to do this is to wrap the incoming
+         * IndexFacts instance in a subclass. E.g.
+         * execution.context().setIndexFacts(new WrapperClass(execution.context().getIndexFacts())).
+         *
+         * @param indexFacts
+         *            an instance to override the following searcher's view of
+         *            the indexes.
+         */
+        public void setIndexFacts(IndexFacts indexFacts) {
+            this.indexFacts = indexFacts;
+        }
+
+        /**
+         * Returns the search chain registry to use with this execution. This is
+         * a snapshot taken at creation of this execution, use
+         * Context.shallowCopy() to get a correctly instantiated Context if
+         * making a custom Context instance.
+         */
+        public SearchChainRegistry searchChainRegistry() {
+            return searchChainRegistry;
+        }
+
+        /**
+         * Returns the template registry to use with this execution. This is
+         * a snapshot taken at creation of this execution.
+         */
+        public RendererRegistry rendererRegistry() {
+            return rendererRegistry;
+        }
+
+        /**
+         * @return the current set of special strings for the query tokenizer
+         */
+        public SpecialTokenRegistry getTokenRegistry() {
+            return tokenRegistry;
+        }
+
+        /**
+         * Wrapping the incoming special token registry and then setting the
+         * wrapper as the token registry, can be used for changing the set of
+         * special tokens used by succeeding searchers. E.g.
+         * execution.context().setTokenRegistry(new WrapperClass(execution.context().getTokenRegistry())).
+         *
+         * @param tokenRegistry a new registry for overriding behavior of following searchers
+         */
+        public void setTokenRegistry(SpecialTokenRegistry tokenRegistry) {
+            this.tokenRegistry = tokenRegistry;
+        }
+
+        public void setDetailedDiagnostics(boolean breakdown) {
+            this.detailedDiagnostics = breakdown;
+        }
+
+        /**
+         * The container has some internal diagnostics mechanisms which may be
+         * costly, and therefore not active by default. Any general diagnostic
+         * mechanism which should not be active be default, may inspect that
+         * state here. If breakdown is assumed, a certain percentage of queries
+         * will have this set automatically.
+         *
+         * @return whether components exposing different level of diagnostics
+         *         should go for the most detailed level
+         */
+        public boolean getDetailedDiagnostics() {
+            return detailedDiagnostics;
+        }
+
+        /**
+         * If too many queries time out, the search handler will assume the
+         * system is in a breakdown state. This state is propagated here.
+         *
+         * @return whether the system is assumed to be in a breakdown state
+         */
+        public boolean getBreakdown() {
+            return breakdown;
+        }
+
+        public void setBreakdown(boolean breakdown) {
+            this.breakdown = breakdown;
+        }
+
+        /**
+         * Returns the {@link Linguistics} object assigned to this Context. This object provides access to all the
+         * linguistic-related APIs, and comes pre-configured with the Execution given.
+         *
+         * @return The current Linguistics.
+         */
+        public Linguistics getLinguistics() {
+            return linguistics;
+        }
+
+        public void setLinguistics(Linguistics linguistics) {
+            this.linguistics = linguistics;
+        }
+
+        /** Creates a child trace if this has an owner, or a root trace otherwise */
+        private Trace createChildTrace() {
+            return owner!=null ? owner.trace().createChild() : Trace.createRoot(0);
+        }
+
+        /** Creates a child environment if this has an owner, or a root environment otherwise */
+        private Environment createChildEnvironment() {
+            return owner!=null ? owner.environment().nested() : Execution.Environment.<Searcher>createEmpty();
+        }
+
+    }
+
+    /**
+     * The index of where in the chain this Execution has its initial entry point.
+     * This is needed because executions can be started from the middle of other executions.
+     */
+    private final int entryIndex;
+
+    /** Time spent in each state of filling, searching or pinging. */
+    private final TimeTracker timer;
+
+    /** A searcher's view of state external to the search chain. */
+    // Note that the context plays the same role as the Environment of the super.Execution
+    // (although complicated by the need for stack-like behavior on changes).
+    // We might want to unify those at some point.
+    private final Context context = new Context(this);
+
+    /**
+     * Array for hiding context changes done in search by searcher following
+     * another.
+     */
+    private final Context[] contextCache;
+
+    private static final Logger log = Logger.getLogger(Execution.class.getName());
+
+    /**
+     * <p>
+     * Creates an execution from another. This execution will start at the
+     * <b>current next searcher</b> in the given execution, rather than at the
+     * start.
+     * </p>
+     *
+     * <p>
+     * The relevant state of the given execution is copied before this method
+     * returns - the argument execution can then be reused for any other
+     * purpose.
+     * </p>
+     */
+    public Execution(Execution execution) {
+        this(execution.chain(), execution.context, execution.nextIndex());
+    }
+
+    /** Creates an which executes nothing */
+    public Execution(Context context) {
+        this(new Chain<>(), context);
+    }
+
+    /**
+     * The usually best way of creating a new execution for a search chain. This
+     * is the one suitable for a production environment. It is safe to use the
+     * incoming context from the search directly:
+     *
+     * <pre>
+     * public Result search(Query query, Execution execution) {
+     *     SearchChain searchChain = fancyChainSelectionRoutine(query);
+     *     if (searchChain != null) {
+     *         return new Execution(searchChain, execution.context());
+     *     else {
+     *         return execution.search(query);
+     *     }
+     * }
+     * </pre>
+     *
+     * @param searchChain
+     *            the search chain to execute
+     * @param context
+     *            the execution context from which this is populated (the given
+     *            context is not changed nor retained by this), or null to not
+     *            populate from a context
+     * @throws IllegalArgumentException
+     *             if searchChain is null
+     */
+    public Execution(Chain<? extends Searcher> searchChain, Context context) {
+        this(searchChain, context, 0);
+    }
+
+    /** Creates an execution from a single searcher */
+    public Execution(Searcher searcher, Context context) {
+        this(new Chain<>(searcher), context, 0);
+    }
+
+    /**
+     * Creates a new execution for a search chain or a single searcher. private
+     * to ensure only searchChain or searcher is null (and because it's long and
+     * cumbersome).
+     *
+     * @param searchChain
+     *            the search chain to execute, must be null if searcher is set
+     * @param context
+     *            execution context for the search
+     * @param searcherIndex
+     *            index of the first searcher to invoke, see
+     *            Execution(Execution)
+     * @throws IllegalArgumentException
+     *             if searchChain is null
+     */
+    @SuppressWarnings("unchecked")
+    private Execution(Chain<? extends Processor> searchChain,Context context, int searcherIndex) {
+        // Create a new Execution which is placed in the context of the execution of the given Context if any
+        // "if any" because a context may, or may not, belong to an execution.
+        // This is decided at the creation time of the Context - Context instances which do not belong
+        // to an execution plays the role of data carriers between executions.
+        super(searchChain,searcherIndex,context.createChildTrace(),context.createChildEnvironment());
+        this.context.fill(context);
+        contextCache = new Context[searchChain.components().size()];
+        entryIndex=searcherIndex;
+        timer = new TimeTracker(searchChain, searcherIndex);
+    }
+
+    /** Does return search(((Query)request) */
+    @Override
+    public final Response process(Request request) {
+        return search((Query)request);
+    }
+
+    /** Calls search on the next searcher in this chain. If there is no next, an empty result is returned. */
+    public Result search(Query query) {
+        timer.sampleSearch(nextIndex(), context.getDetailedDiagnostics());
+
+        // Transfer state between query and execution as the execution constructors does not do that completely
+        query.getModel().setExecution(this);
+        trace().setTraceLevel(query.getTraceLevel());
+
+        return (Result)super.process(query);
+    }
+
+    @Override
+    protected void onInvoking(Request request, Processor processor) {
+        super.onInvoking(request,processor);
+        final int traceDependencies = 6;
+        Query query = (Query) request;
+        if (query.getTraceLevel() >= traceDependencies) {
+            query.trace(new StringBuilder().append(processor.getId())
+                    .append(" ").append(processor.getDependencies().toString())
+                    .toString(), traceDependencies);
+        }
+    }
+
+    /**
+     * The default response returned from this kind of execution when there are not further processors
+     * - an empty Result
+     */
+    @Override
+    protected Response defaultResponse(Request request) {
+        return new Result((Query)request);
+    }
+
+    /**
+     * Fill hit properties with values from all in-memory attributes.
+     * This can be done with good performance on many more hits than
+     * those for which fill is called with the final summary class, so
+     * if filtering can be done using only in-memory attribute data,
+     * this method should be preferred over {@link #fill} to get that data for filtering.
+     * <p>
+     * Calling this on already filled results has no cost.
+     *
+     * @param result the result to fill
+     */
+    @SuppressWarnings("deprecation")
+    public void fillAttributes(Result result) {
+        fill(result, ATTRIBUTEPREFETCH);
+    }
+
+    /**
+     * Fill hit properties with data using the default summary
+     * class, possibly overridden with the 'summary' request parameter.
+     * <p>
+     * Fill <b>must</b> be called before any property (accessed by
+     * getProperty/getField) is accessed on the hit. It should be done
+     * as late as possible for performance reasons.
+     * <p>
+     * Calling this on already filled results has no cost.
+     *
+     * @param result the result to fill
+     */
+    public void fill(Result result) {
+        fill(result, result.getQuery().getPresentation().getSummary());
+    }
+
+    /** Calls fill on the next searcher in this chain. If there is no next, nothing is done. */
+    public void fill(Result result,String summaryClass) {
+        timer.sampleFill(nextIndex(), context.getDetailedDiagnostics());
+        Searcher next = (Searcher)next(); // TODO: Allow but skip processors which are not searchers
+        if (next==null) return;
+
+        try {
+            nextProcessor();
+            next.ensureFilled(result, summaryClass, this);
+        }
+        finally {
+            previousProcessor();
+            timer.sampleFillReturn(nextIndex(), context.getDetailedDiagnostics(), result);
+        }
+    }
+
+    /** Calls ping on the next search in this chain. If there is no next, a Pong is created and returned. */
+    public Pong ping(Ping ping) {
+        // return this reference, not directly. It's needed for adding time data
+        Pong annotationReference = null;
+
+        timer.samplePing(nextIndex(), context.getDetailedDiagnostics());
+        Searcher next = (Searcher)next(); // TODO: Allow but skip processors which are not searchers
+        if (next==null) {
+            annotationReference = new Pong();
+            return annotationReference;
+        }
+
+        try {
+            nextProcessor();
+            annotationReference = invokePing(ping, next);
+            return annotationReference;
+        }
+        finally {
+            previousProcessor();
+            timer.samplePingReturn(nextIndex(), context.getDetailedDiagnostics(), annotationReference);
+        }
+    }
+
+    @Override
+    protected void onReturning(Request request, Processor processor,Response response) {
+        super.onReturning(request, processor, response);
+        timer.sampleSearchReturn(nextIndex(), context.getDetailedDiagnostics(), (Result)response);
+    }
+
+    @Override
+    protected void previousProcessor() {
+        super.previousProcessor();
+        popContext();
+    }
+
+    @Override
+    protected void nextProcessor() {
+        pushContext();
+        super.nextProcessor();
+    }
+
+    private void popContext() {
+        context.fill(contextCache[nextIndex()]);
+        contextCache[nextIndex()] = null;
+    }
+
+    private void pushContext() {
+        final Context contextToPush;
+        // Do note: Never put this.context in the cache. It would be totally
+        // meaningless, since it's a final.
+        if (nextIndex() == entryIndex) {
+            contextToPush = context.shallowCopy();
+        } else {
+            contextToPush = context.copyIfChanged(contextCache[nextIndex() - 1]);
+        }
+        contextCache[nextIndex()] = contextToPush;
+    }
+
+    private Pong invokePing(Ping ping, Searcher next) {
+        Pong annotationReference;
+        if (next instanceof PingableSearcher) {
+            annotationReference = ((PingableSearcher) next).ping(ping, this);
+        } else {
+            annotationReference = ping(ping);
+        }
+        return annotationReference;
+    }
+
+    /**
+     * Returns the search chain registry to use with this execution. This is a
+     * snapshot taken at creation of this execution if available.
+     */
+    public SearchChainRegistry searchChainRegistry() {
+        return context.searchChainRegistry();
+    }
+
+    /**
+     * Returns the context of this execution, which contains various objects
+     * which are looked up through a memory barrier at the point this is created
+     * and which is guaranteed to be frozen during the execution of this query.
+     * <p>
+     * Note that the context itself can be changed. Such changes will be visible
+     * to downstream searchers, but not after returning from the modifying
+     * searcher. In other words, a change in the context will not be visible to
+     * the preceding searchers when the result is returned from the searcher
+     * which modified the context.
+     */
+    public Context context() {
+        return context;
+    }
+
+    /**
+     * @return the TimeTracker instance associated with this Execution
+     */
+    public TimeTracker timer() {
+        return timer;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ForkingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/ForkingSearcher.java
new file mode 100644
index 00000000000..cae1ba36e6c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/ForkingSearcher.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.Chain;
+import com.yahoo.search.Searcher;
+
+import java.util.Collection;
+
+/**
+ * Searchers which invokes other search chains should override this.
+ *
+ * @author bratseth
+ */
+public abstract class ForkingSearcher extends Searcher {
+
+    public ForkingSearcher() {}
+
+    /** A search chain with a comment about when it is used. */
+    public static class CommentedSearchChain {
+        public final String comment;
+        public final Chain<Searcher> searchChain;
+
+        public CommentedSearchChain(String comment, Chain<Searcher> searchChain) {
+            this.comment = comment;
+            this.searchChain = searchChain;
+        }
+    }
+
+    /** Returns which searchers this searcher may forward to, for debugging and tracing */
+    public abstract Collection<CommentedSearchChain> getSearchChainsForwarded(SearchChainRegistry registry);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/FutureResult.java b/container-search/src/main/java/com/yahoo/search/searchchain/FutureResult.java
new file mode 100644
index 00000000000..877252f07e6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/FutureResult.java
@@ -0,0 +1,86 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.yahoo.yolean.Exceptions;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.ErrorMessage;
+
+/**
+ * Extends a {@code FutureTask<Result>}, with some added error handling
+ */
+public class FutureResult extends FutureTask<Result> {
+
+    private final Query query;
+
+    /** Only used for generating messages */
+    private final Execution execution;
+
+    private final static Logger log = Logger.getLogger(FutureResult.class.getName());
+
+    FutureResult(Callable<Result> callable, Execution execution, final Query query) {
+        super(callable);
+        this.query = query;
+        this.execution = execution;
+    }
+
+    @Override
+    public Result get() {
+        Result result;
+        try {
+            result = super.get();
+        }
+        catch (InterruptedException e) {
+            result = new Result(getQuery(), ErrorMessage.createUnspecifiedError(
+                    "'" + execution + "' was interrupted while executing: " + Exceptions.toMessageString(e)));
+        }
+        catch (ExecutionException e) {
+            log.log(Level.WARNING,"Exception on executing " + execution + " for " + query,e);
+            result = new Result(getQuery(), ErrorMessage.createErrorInPluginSearcher(
+                    "Error in '" + execution + "': " + Exceptions.toMessageString(e),
+                    e.getCause()));
+        }
+        return result;
+    }
+
+    @Override
+    public Result get(long timeout, TimeUnit timeunit) {
+        Result result;
+        try {
+            result = super.get(timeout, timeunit);
+        }
+        catch (InterruptedException e) {
+            result = new Result(getQuery(), ErrorMessage.createUnspecifiedError(
+                    "'" + execution + "' was interrupted while executing: " + Exceptions.toMessageString(e)));
+        }
+        catch (ExecutionException e) {
+            log.log(Level.WARNING,"Exception on executing " + execution + " for " + query, e);
+            result = new Result(getQuery(), ErrorMessage.createErrorInPluginSearcher(
+                    "Error in '" + execution + "': " + Exceptions.toMessageString(e),
+                    e.getCause()));
+        }
+        catch (TimeoutException e) {
+            result = new Result(getQuery(), createTimeoutError());
+        }
+        return result;
+    }
+
+    /** Returns the query used in this execution, never null */
+    public Query getQuery() {
+        return query;
+    }
+
+    ErrorMessage createTimeoutError() {
+        return ErrorMessage.createTimeout(
+                "Error executing '" + execution + "': " + " Chain timed out.");
+
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java
new file mode 100644
index 00000000000..96bef503e0e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java
@@ -0,0 +1,46 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+
+/**
+ * Helper class for ordering searchers. Searchers may use these names in their
+ * {@literal @}Before and {@literal @}After annotations, though in general
+ * a searcher should depend on some explicit functionality, not these
+ * checkpoints.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public final class PhaseNames {
+    private PhaseNames() {
+    }
+
+    /**
+     * A checkpoint where the query is not yet transformed in any way. RAW_QUERY
+     * is the first checkpoint not provided by some searcher.
+     */
+    public static final String RAW_QUERY = "rawQuery";
+
+    /**
+     * A checkpoint where as many query transformers as practically possible has
+     * been run. TRANSFORMED_QUERY is the first checkpoint after RAW_QUERY.
+     */
+    public static final String TRANSFORMED_QUERY = "transformedQuery";
+
+    /**
+     * A checkpoint where results from different backends have been flattened
+     * into a single result. BLENDED_RESULT is the first checkpoint after
+     * TRANSFORMED_QUERY.
+     */
+    public static final String BLENDED_RESULT = "blendedResult";
+
+    /**
+     * A checkpoint where data from different backends are not yet merged.
+     * UNBLENDED_RESULT is the first checkpoint after BLENDED_RESULT.
+     */
+    public static final String UNBLENDED_RESULT = "unblendedResult";
+
+    /**
+     * The last checkpoint in a search chain not provided by any searcher.
+     */
+    public static final String BACKEND = "backend";
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/SearchChain.java b/container-search/src/main/java/com/yahoo/search/searchchain/SearchChain.java
new file mode 100644
index 00000000000..457604f7ce8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/SearchChain.java
@@ -0,0 +1,85 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.Chain;
+import com.yahoo.component.chain.Phase;
+import com.yahoo.search.Searcher;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A named collection of searchers.
+ * <p>
+ * The searchers may have dependencies which define an ordering
+ * of the searchers of this chain.
+ * <p>
+ * Search chains may inherit the searchers of other chains and modify
+ * the inherited set of searchers.
+ * <p>
+ * Search chains may be versioned. The version and name string combined
+ * is an unique identifier of a search chain.
+ * <p>
+ * A search chain cannot be modified once constructed.
+ *
+ * @author bratseth
+ */
+public class SearchChain extends Chain<Searcher> {
+
+    public SearchChain(ComponentId id) {
+        this(id, null, null);
+    }
+
+    public SearchChain(ComponentId id, Searcher... searchers) {
+        this(id, Arrays.asList(searchers));
+    }
+
+    public SearchChain(ComponentId id, Collection<Searcher> searchers) {
+        this(id, searchers, null);
+    }
+
+    /**
+     * Creates a search chain.
+     * <p>
+     * This search chain makes a copy of the given lists before return and does not modify the argument lists.
+     * <p>
+     * The total set of searchers included in this chain will be
+     * <ul>
+     * <li>The searchers given in <code>searchers</code>.
+     * <li>Plus all searchers returned by {@link #searchers} on all search chains in <code>inherited</code>.
+     * If a searcher with a given name is present in the <code>searchers</code> list in any version, that
+     * version will be used, and a searcher with that name will never be included from an inherited search chain.
+     * If the same searcher exists in multiple inherited chains, the highest version will be used.
+     * <li>Minus all searchers, of any version, whose name exists in the <code>excluded</code> list.
+     * </ul>
+     *
+     * @param id        the id of this search chain
+     * @param searchers the searchers of this chain, or null if none
+     * @param phases    the phases of this chain
+     */
+    public SearchChain(ComponentId id, Collection<Searcher> searchers, Collection<Phase> phases) {
+        super(id, searchers, phases);
+    }
+
+    /** For internal use only! */
+    public SearchChain(Chain<Searcher> chain) {
+        super(chain.getId(), chain.components());
+    }
+
+    /**
+     * Returns an unmodifiable list of the searchers this search chain executs, in resolved execution order.
+     * This includes all inherited (and not excluded) searchers.
+     */
+    public List<Searcher> searchers() {
+        return components();
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder b = new StringBuilder("search ");
+        b.append(super.toString());
+        return b.toString();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/SearchChainRegistry.java b/container-search/src/main/java/com/yahoo/search/searchchain/SearchChainRegistry.java
new file mode 100644
index 00000000000..9513394bc9f
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/SearchChainRegistry.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.Chain;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.processing.execution.chain.ChainRegistry;
+import com.yahoo.search.Searcher;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Contains a reference to all currently known search chains.
+ * Searchers can be fetched from this from multiple threads.
+ * <p>
+ * A registry can exist in two states:
+ * <ul>
+ * <li>not frozen - in this state it can be edited freely by calling {@link #register}
+ * <li>frozen - in this state any attempt at modification throws an IlegalStateException
+ * </ul>
+ * Registries start in the first state, moves to the second on calling freeze and stays in that
+ * state for the rest of their lifetime.
+ *
+ * @author bratseth
+ */
+public class SearchChainRegistry extends ChainRegistry<Searcher> {
+
+    private final SearcherRegistry searcherRegistry;
+
+    @Override
+    public void freeze() {
+        super.freeze();
+        getSearcherRegistry().freeze();
+    }
+
+    public SearchChainRegistry() {
+        searcherRegistry = new SearcherRegistry();
+        searcherRegistry.freeze();
+    }
+
+    public SearchChainRegistry(ComponentRegistry<? extends AbstractComponent> allComponentRegistry) {
+        this.searcherRegistry = setupSearcherRegistry(allComponentRegistry);
+    }
+
+    public void register(Chain<Searcher> component) {
+        super.register(component.getId(), component);
+    }
+
+    public Chain<Searcher> unregister(Chain<Searcher> component) {
+        return super.unregister(component.getId());
+    }
+
+    private SearcherRegistry setupSearcherRegistry(ComponentRegistry<? extends AbstractComponent> allComponents) {
+        SearcherRegistry registry = new SearcherRegistry();
+        for (AbstractComponent component : allComponents.allComponents()) {
+            if (component instanceof Searcher) {
+                registry.register((Searcher) component);
+            }
+        }
+        //just freeze this right away
+        registry.freeze();
+        return registry;
+    }
+
+    public SearcherRegistry getSearcherRegistry() {
+        return searcherRegistry;
+    }
+
+    @Override
+    public SearchChain getComponent(ComponentId id) {
+        Chain<Searcher> chain = super.getComponent(id);
+        return asSearchChain(chain);
+    }
+
+    @Override
+    public SearchChain getComponent(ComponentSpecification specification) {
+        return asSearchChain(super.getComponent(specification));
+    }
+
+    public final Chain<Searcher> getChain(String componentSpecification) {
+        return super.getComponent(new ComponentSpecification(componentSpecification));
+    }
+
+    public final Chain<Searcher> getChain(ComponentId id) {
+         return super.getComponent(id);
+     }
+
+
+    @Override
+    public SearchChain getComponent(String componentSpecification) {
+        return getComponent(new ComponentSpecification(componentSpecification));
+    }
+
+    private SearchChain asSearchChain(Chain<Searcher> chain) {
+        if (chain == null) {
+            return null;
+        } else if (chain instanceof SearchChain) {
+            return (SearchChain) chain;
+        } else {
+            return new SearchChain(chain);
+        }
+    }
+
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/SearcherRegistry.java b/container-search/src/main/java/com/yahoo/search/searchchain/SearcherRegistry.java
new file mode 100644
index 00000000000..d1a4c1743d6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/SearcherRegistry.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.pagetemplates.engine.Resolver;
+
+/**
+ * A registry of searchers. This is instantiated and recycled in the context of an owning search chain registry.
+ * This class exists for legacy purposes only, to preserve the public API for retrieving searchers from Vespa 4.2.
+ *
+ * @author bratseth
+ */
+public class SearcherRegistry extends ComponentRegistry<Searcher> {
+
+    public void register(Searcher searcher) {
+        super.register(searcher.getId(), searcher);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/example/ExampleSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/example/ExampleSearcher.java
new file mode 100644
index 00000000000..06a4096dc68
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/example/ExampleSearcher.java
@@ -0,0 +1,23 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain.example;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * An example searcher which adds a hit
+ *
+ * @author bratseth
+ */
+public class ExampleSearcher extends Searcher {
+
+    public @Override Result search(Query query,Execution execution) {
+        Result result=execution.search(query);
+        result.hits().add(new Hit("example",1.0,"examplesearcher"));
+        return result;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java
new file mode 100644
index 00000000000..1a3790e1012
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain.model;
+
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.model.federation.FederationSearcherModel;
+import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec;
+import org.apache.commons.collections.CollectionUtils;
+
+import java.util.*;
+
+
+/**
+ * Defines the searcher models used in the vespa and native search chains, except for federation.
+ *
+ * @author tonytv
+ */
+@SuppressWarnings({"rawtypes", "deprecation", "unchecked"})
+public class VespaSearchers {
+    public static final Collection<ChainedComponentModel> vespaSearcherModels =
+            toSearcherModels(
+                    com.yahoo.prelude.querytransform.IndexCombinatorSearcher.class,
+                    //com.yahoo.prelude.querytransform.LocalitySearcher.class,
+                    com.yahoo.prelude.querytransform.PhrasingSearcher.class,
+                    com.yahoo.prelude.searcher.FieldCollapsingSearcher.class,
+                    com.yahoo.search.yql.MinimalQueryInserter.class,
+                    com.yahoo.search.yql.FieldFilter.class,
+                    com.yahoo.prelude.searcher.JuniperSearcher.class,
+                    com.yahoo.prelude.searcher.BlendingSearcher.class,
+                    com.yahoo.prelude.searcher.PosSearcher.class,
+                    com.yahoo.prelude.semantics.SemanticSearcher.class,
+                    com.yahoo.search.grouping.GroupingQueryParser.class);
+
+
+    public static final Collection<ChainedComponentModel> nativeSearcherModels;
+
+    static {
+        nativeSearcherModels = new LinkedHashSet<>();
+        nativeSearcherModels.add(federationSearcherModel());
+        nativeSearcherModels.addAll(toSearcherModels(com.yahoo.prelude.statistics.StatisticsSearcher.class));
+
+        //ensure that searchers in the native search chain are not overridden by searchers in the vespa search chain,
+        //and that all component ids in each chain are unique.
+        assert(allComponentIdsDifferent(vespaSearcherModels, nativeSearcherModels));
+    }
+
+    private static boolean allComponentIdsDifferent(Collection<ChainedComponentModel> vespaSearcherModels,
+                                                    Collection<ChainedComponentModel> nativeSearcherModels) {
+        Set<ComponentId> componentIds = new LinkedHashSet<>();
+        return
+                allAdded(vespaSearcherModels, componentIds) &&
+                allAdded(nativeSearcherModels, componentIds);
+
+    }
+
+    private static FederationSearcherModel federationSearcherModel() {
+        return new FederationSearcherModel(new ComponentSpecification("federation"),
+                Dependencies.emptyDependencies(),
+                Collections.<TargetSpec>emptyList(), true);
+    }
+
+    private static boolean allAdded(Collection<ChainedComponentModel> searcherModels, Set<ComponentId> componentIds) {
+        for (ChainedComponentModel model : searcherModels) {
+            if (!componentIds.add(model.getComponentId()))
+                return false;
+        }
+        return true;
+    }
+
+    private static Collection<ChainedComponentModel> toSearcherModels(Class<? extends Searcher>... searchers) {
+        List<ChainedComponentModel> searcherModels = new ArrayList<>();
+        for (Class c : searchers) {
+            searcherModels.add(
+                    new ChainedComponentModel(
+                            BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(c.getName(), null),
+                            Dependencies.emptyDependencies()));
+        }
+        return searcherModels;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationOptions.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationOptions.java
new file mode 100644
index 00000000000..ec6bf9661c6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationOptions.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain.model.federation;
+
+import net.jcip.annotations.Immutable;
+
+/**
+ * Options for controlling federation to a single source.
+ *
+ * @author tonytv
+ */
+@Immutable
+public class FederationOptions implements Cloneable {
+
+    private final Boolean optional;
+    private final Integer timeoutInMilliseconds;
+    private final Integer requestTimeoutInMilliseconds;
+    private final Boolean useByDefault;
+
+    /**
+     * Creates a request with no separate requestTimeoutInMilliseconds
+     */
+    public FederationOptions(Boolean optional, Integer timeoutInMilliseconds, Boolean useByDefault) {
+        this(optional, timeoutInMilliseconds, null, useByDefault);
+    }
+
+    /**
+     * Creates a fully specified set of options
+     *
+     * @param optional whether this should be optional
+     * @param timeoutInMilliseconds the max time to wait for a result from this source, or null to not specify a limit
+     * @param requestTimeoutInMilliseconds the max time to allow this request to live, or null to make this the same as
+     *                                     timeoutInMilliseconds. Setting this higher than timeoutInMilliseconds is
+     *                                     useful to use queries to populate the cache of slow sources
+     * @param useByDefault whether this should be invoked by default
+     */
+    public FederationOptions(Boolean optional, Integer timeoutInMilliseconds, Integer requestTimeoutInMilliseconds, Boolean useByDefault) {
+        this.optional = optional;
+        this.timeoutInMilliseconds = timeoutInMilliseconds;
+        this.requestTimeoutInMilliseconds = requestTimeoutInMilliseconds;
+        this.useByDefault = useByDefault;
+    }
+
+    /** Creates a set of default options: Mandatory, no timeout restriction and not used by default */
+    public FederationOptions() {
+        this(null, null, null, null);
+    }
+
+    /** Returns a set of options which are the same of this but with optional set to the given value */
+    public FederationOptions setOptional(Boolean newOptional) {
+        return new FederationOptions(newOptional, timeoutInMilliseconds, requestTimeoutInMilliseconds, useByDefault);
+    }
+
+    /** Returns a set of options which are the same of this but with timeout set to the given value */
+    public FederationOptions setTimeoutInMilliseconds(Integer newTimeoutInMilliseconds) {
+        return new FederationOptions(optional, newTimeoutInMilliseconds, requestTimeoutInMilliseconds, useByDefault);
+    }
+
+    /** Returns a set of options which are the same of this but with request timeout set to the given value */
+    public FederationOptions setRequestTimeoutInMilliseconds(Integer newRequestTimeoutInMilliseconds) {
+        return new FederationOptions(optional, timeoutInMilliseconds, newRequestTimeoutInMilliseconds, useByDefault);
+    }
+
+    /** Returns a set of options which are the same of this but with default set to the given value */
+    public FederationOptions setUseByDefault(Boolean newUseByDefault) {
+        return new FederationOptions(optional, timeoutInMilliseconds, requestTimeoutInMilliseconds, newUseByDefault);
+    }
+
+    public boolean getOptional() {
+        return (optional != null) ? optional : false;
+    }
+
+    /** Returns the amount of time we should wait for this target, or -1 to use default */
+    public int getTimeoutInMilliseconds() {
+        return (timeoutInMilliseconds != null) ? timeoutInMilliseconds : -1;
+    }
+
+    /** Returns the amount of time we should allow this target execution to run, or -1 to use default */
+    public int getRequestTimeoutInMilliseconds() {
+        return (requestTimeoutInMilliseconds != null) ? requestTimeoutInMilliseconds : -1;
+    }
+
+    public long getSearchChainExecutionTimeoutInMilliseconds(long queryTimeout) {
+        return getTimeoutInMilliseconds() >= 0 ?
+                getTimeoutInMilliseconds() :
+                queryTimeout;
+    }
+
+    public boolean getUseByDefault() {
+        return useByDefault != null ? useByDefault : false;
+    }
+
+    public FederationOptions inherit(FederationOptions parent) {
+        return new FederationOptions(
+                inherit(optional, parent.optional),
+                inherit(timeoutInMilliseconds, parent.timeoutInMilliseconds),
+                inherit(requestTimeoutInMilliseconds, parent.requestTimeoutInMilliseconds),
+                inherit(useByDefault, parent.useByDefault));
+    }
+
+    private static <T> T inherit(T child, T parent) {
+        return (child != null) ? child : parent;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return (other instanceof FederationOptions) &&
+                equals((FederationOptions) other);
+    }
+
+    public boolean equals(FederationOptions other) {
+        return getOptional() == other.getOptional() &&
+                getTimeoutInMilliseconds() == other.getTimeoutInMilliseconds();
+    }
+
+    @Override
+    public String toString() {
+        return "FederationOptions{" +
+                "optional=" + optional +
+                ", timeoutInMilliseconds=" + timeoutInMilliseconds +
+                ", useByDefault=" + useByDefault +
+                '}';
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java
new file mode 100644
index 00000000000..99293cb611b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationSearcherModel.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain.model.federation;
+
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import net.jcip.annotations.Immutable;
+
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.search.federation.FederationSearcher;
+
+/**
+ * Specifies how a federation searcher is to be set up.
+ *
+ * @author tonytv
+ */
+@Immutable
+public class FederationSearcherModel extends ChainedComponentModel {
+
+    /**
+     * Specifies one or more search chains that can be addressed
+     * as a single source.
+     */
+    public static class TargetSpec {
+        public final ComponentSpecification sourceSpec;
+        public final FederationOptions federationOptions;
+
+        public TargetSpec(ComponentSpecification sourceSpec, FederationOptions federationOptions) {
+            this.sourceSpec = sourceSpec;
+            this.federationOptions = federationOptions;
+        }
+    }
+
+    private static ComponentSpecification federationSearcherComponentSpecification =
+            new ComponentSpecification(FederationSearcher.class.getName());
+
+    public final List<TargetSpec> targets;
+    public final boolean inheritDefaultSources;
+
+    public FederationSearcherModel(ComponentSpecification componentId, Dependencies dependencies,
+                                   List<TargetSpec> targets, boolean inheritDefaultSources) {
+        super(BundleInstantiationSpecification.getInternalSearcherSpecification(componentId, federationSearcherComponentSpecification),
+              dependencies);
+        this.inheritDefaultSources = inheritDefaultSources;
+        this.targets = ImmutableList.copyOf(targets);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java
new file mode 100644
index 00000000000..33bdb54b00e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java
@@ -0,0 +1,121 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain.model.federation;
+
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import net.jcip.annotations.Immutable;
+
+import com.yahoo.search.federation.http.HTTPProviderSearcher;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Specifies how a http provider is to be set up.
+ *
+ * @author tonytv
+ */
+@Immutable
+public class HttpProviderSpec {
+    public enum Type {
+        vespa(com.yahoo.search.federation.vespa.VespaSearcher.class);
+
+        Type(Class<? extends HTTPProviderSearcher> searcherClass) {
+            className = searcherClass.getName();
+        }
+
+        final String className;
+    }
+
+    // The default connection parameter values come from the config definition
+    public static class ConnectionParameters {
+        public final Double readTimeout;
+        public final Double connectionTimeout;
+        public final Double connectionPoolTimeout;
+        public final Integer retries;
+
+        public ConnectionParameters(Double readTimeout, Double connectionTimeout,
+                                    Double connectionPoolTimeout, Integer retries) {
+            this.readTimeout = readTimeout;
+            this.connectionTimeout = connectionTimeout;
+            this.connectionPoolTimeout = connectionPoolTimeout;
+            this.retries = retries;
+        }
+    }
+
+    public static class Node {
+        public final String host;
+        public final int port;
+
+        public Node(String host, int port) {
+            this.host = host;
+            this.port = port;
+        }
+
+        @Override
+        public String toString() {
+            return "Node{" +
+                    "host='" + host + '\'' +
+                    ", port=" + port +
+                    '}';
+        }
+    }
+
+    public final ConnectionParameters connectionParameters;
+
+    public final Integer cacheSizeMB;
+
+    public final String path;
+    public final List<Node> nodes;
+    public final String ycaApplicationId;
+    public final Integer ycaCertificateTtl;
+    public final Integer ycaRetryWait;
+    public final Node ycaProxy;
+
+    //TODO:remove this
+    public final double cacheWeight;
+
+
+    public static BundleInstantiationSpecification toBundleInstantiationSpecification(Type type) {
+        return BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(type.className, null);
+    }
+
+    public static boolean includesType(String typeString) {
+        for (Type type : Type.values()) {
+            if (type.name().equals(typeString)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public HttpProviderSpec(Double cacheWeight,
+                            String path,
+                            List<Node> nodes,
+                            String ycaApplicationId,
+                            Integer ycaCertificateTtl,
+                            Integer ycaRetryWait,
+                            Node ycaProxy,
+                            Integer cacheSizeMB,
+                            ConnectionParameters connectionParameters) {
+
+        final double defaultCacheWeight = 1.0d;
+        this.cacheWeight = (cacheWeight != null) ? cacheWeight : defaultCacheWeight;
+
+        this.path = path;
+        this.nodes = unmodifiable(nodes);
+        this.ycaApplicationId = ycaApplicationId;
+        this.ycaProxy = ycaProxy;
+        this.ycaCertificateTtl = ycaCertificateTtl;
+        this.ycaRetryWait = ycaRetryWait;
+        this.cacheSizeMB = cacheSizeMB;
+
+        this.connectionParameters = connectionParameters;
+    }
+
+    private List<HttpProviderSpec.Node> unmodifiable(List<HttpProviderSpec.Node> nodes) {
+        return nodes == null ?
+                Collections.<HttpProviderSpec.Node>emptyList() :
+                Collections.unmodifiableList(new ArrayList<>(nodes));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java
new file mode 100644
index 00000000000..c8847507039
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain.model.federation;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.component.chain.dependencies.Dependencies;
+import com.yahoo.component.chain.model.ChainedComponentModel;
+import com.yahoo.search.Searcher;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import net.jcip.annotations.Immutable;
+
+/**
+ * Specifies how a local provider is to be set up.
+ *
+ * @author tonytv
+ */
+@Immutable
+public class LocalProviderSpec {
+    @SuppressWarnings("unchecked")
+    public static final Collection<ChainedComponentModel> searcherModels =
+                    toSearcherModels(
+                            com.yahoo.prelude.querytransform.CJKSearcher.class,
+                            com.yahoo.search.querytransform.NGramSearcher.class,
+                            com.yahoo.prelude.querytransform.LiteralBoostSearcher.class,
+                            com.yahoo.prelude.querytransform.NormalizingSearcher.class,
+                            com.yahoo.prelude.querytransform.StemmingSearcher.class,
+                            com.yahoo.search.querytransform.VespaLowercasingSearcher.class,
+                            com.yahoo.search.querytransform.DefaultPositionSearcher.class,
+                            com.yahoo.search.querytransform.RangeQueryOptimizer.class,
+                            com.yahoo.search.querytransform.SortingDegrader.class,
+                            com.yahoo.prelude.searcher.ValidateSortingSearcher.class,
+                            com.yahoo.prelude.cluster.ClusterSearcher.class,
+                            com.yahoo.search.grouping.GroupingValidator.class,
+                            com.yahoo.search.grouping.vespa.GroupingExecutor.class,
+                            com.yahoo.prelude.querytransform.RecallSearcher.class,
+                            com.yahoo.search.querytransform.WandSearcher.class,
+                            com.yahoo.search.querytransform.BooleanSearcher.class,
+                            com.yahoo.prelude.searcher.ValidatePredicateSearcher.class,
+                            com.yahoo.search.searchers.ValidateMatchPhaseSearcher.class,
+                            com.yahoo.search.yql.FieldFiller.class,
+                            com.yahoo.search.searchers.InputCheckingSearcher.class);
+
+    public final String clusterName;
+
+    //TODO: make this final
+    public Integer cacheSize;
+
+    public LocalProviderSpec(String clusterName, Integer cacheSize) {
+        this.clusterName = clusterName;
+        this.cacheSize = cacheSize;
+
+        if (clusterName == null)
+            throw new IllegalArgumentException("Missing cluster name.");
+    }
+
+    public static boolean includesType(String type) {
+        return "local".equals(type);
+    }
+
+    @SafeVarargs
+    private static final Collection<ChainedComponentModel> toSearcherModels(Class<? extends Searcher>... searchers) {
+        List<ChainedComponentModel> searcherModels = new ArrayList<>();
+
+        for (Class<? extends Searcher> c : searchers) {
+            searcherModels.add(
+                    new ChainedComponentModel(
+                            BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(
+                                    c.getName(),
+                                    null),
+                            Dependencies.emptyDependencies()));
+        }
+
+        return ImmutableList.copyOf(searcherModels);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/package-info.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/package-info.java
new file mode 100644
index 00000000000..9642d389661
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.search.searchchain.model.federation;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/package-info.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/package-info.java
new file mode 100644
index 00000000000..9219eb36094
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.search.searchchain.model;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/package-info.java b/container-search/src/main/java/com/yahoo/search/searchchain/package-info.java
new file mode 100644
index 00000000000..0b1ec05abef
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Classes for composition of searchers into search chains, which are executed to produce Results for Queries.
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.searchchain;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java
new file mode 100644
index 00000000000..a5b9c58f084
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java
@@ -0,0 +1,190 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchchain.testutil;
+
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.List;
+
+import com.yahoo.net.URI;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * <p>Implements a document source.  You pass in a query and a Result
+ * set.  When this Searcher is called with that query it will return
+ * that result set.</p>
+ *
+ * <p>This supports multi-phase search.</p>
+ *
+ * <p>To avoid having to add type information for the fields, a quck hack is used to
+ * support testing of attribute prefetching.
+ * Any field in the configured hits which has a name starting by attribute
+ * will be returned when attribute prefetch filling is requested.</p>
+ *
+ * @author  <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public class DocumentSourceSearcher extends Searcher {
+
+    // using null as name in the API would just be a horrid headache
+    public static final String DEFAULT_SUMMARY_CLASS = "default";
+
+    // TODO: update tests to explicitly set hits, so that the default results can be removed entirely.
+    private Result defaultFilledResult;
+
+    private Map<Query, Result> completelyFilledResults = new HashMap<>();
+    private Map<Query, Result> unFilledResults = new HashMap<>();
+    private Map<String, Set<String>> summaryClasses = new HashMap<>();
+
+    private int queryCount;
+
+    public DocumentSourceSearcher() {
+        addDefaultResults();
+    }
+
+    /**
+     * Adds a result which can be searched for and filled.
+     * Summary fields starting by "a" are attributes, others are not.
+     *
+     * @return true when replacing an existing &lt;query, result&gt; pair.
+     */
+    public boolean addResult(Query query, Result fullResult) {
+        Result emptyResult = new Result(query.clone());
+        emptyResult.setTotalHitCount(fullResult.getTotalHitCount());
+        for (Hit fullHit : fullResult.hits().asList()) {
+            Hit emptyHit = fullHit.clone();
+            emptyHit.clearFields();
+            emptyHit.setFillable();
+            emptyHit.setRelevance(fullHit.getRelevance());
+
+            emptyResult.hits().add(emptyHit);
+        }
+        unFilledResults.put(getQueryKeyClone(query), emptyResult);
+
+        if (completelyFilledResults.put(getQueryKeyClone(query), fullResult.clone()) != null) {
+           // TODO: throw exception if the key exists from before, change the method to void
+           return true;
+        }
+        return false;
+    }
+
+    public void addSummaryClass(String name, Set<String> fields) {
+        summaryClasses.put(name,fields);
+    }
+
+    public void addSummaryClassByCopy(String name, Collection<String> fields) {
+        addSummaryClass(name, new HashSet<>(fields));
+    }
+
+    private void addDefaultResults() {
+        Query q = new Query("?query=default");
+        Result r = new Result(q);
+        // These four used to assign collapseId 1,2,3,4 - re-add that if needed
+        r.hits().add(new Hit("http://default-1.html", 0));
+        r.hits().add(new Hit("http://default-2.html", 0));
+        r.hits().add(new Hit("http://default-3.html", 0));
+        r.hits().add(new Hit("http://default-4.html", 0));
+        defaultFilledResult = r;
+        addResult(q, r);
+    }
+
+    public @Override Result search(Query query, Execution execution)  {
+        queryCount++;
+        Result r;
+        r = unFilledResults.get(getQueryKeyClone(query));
+        if (r == null) {
+            r = defaultFilledResult.clone();
+        } else {
+            r = r.clone();
+        }
+
+        r.setQuery(query);
+        r.hits().trim(query.getOffset(), query.getHits());
+        return r;
+    }
+
+    /**
+     * Returns a query clone which has offset and hits set to null. This is used by access to
+     * the maps using the query as key to achieve lookup independent of offset/hits value
+     */
+    private Query getQueryKeyClone(Query query) {
+        Query key=query.clone();
+        key.setWindow(0,0);
+        return key;
+    }
+
+    public @Override void fill(Result result, String summaryClass, Execution execution) {
+        Result filledResult;
+        filledResult = completelyFilledResults.get(getQueryKeyClone(result.getQuery()));
+
+        if (filledResult == null) {
+            filledResult = defaultFilledResult;
+        }
+        fillHits(filledResult,result,summaryClass);
+    }
+
+    private void fillHits(Result filledHits, Result hitsToFill, String summaryClass) {
+        Set<String> fieldsToFill = summaryClasses.get(summaryClass);
+
+        if (fieldsToFill == null ) {
+            fieldsToFill = summaryClasses.get(DEFAULT_SUMMARY_CLASS);
+        }
+
+        for (Hit hitToFill : hitsToFill.hits()) {
+            Hit filledHit = getMatchingFilledHit(hitToFill.getId(), filledHits);
+
+            if (filledHit != null) {
+                if (fieldsToFill != null) {
+                    copyFieldValuesThatExist(filledHit,hitToFill,fieldsToFill);
+                } else {
+                    // TODO: remove this block and update fieldsToFill above to throw an exception if no appropriate summary class is found
+                    for (Map.Entry<String,Object> propertyEntry : filledHit.fields().entrySet()) {
+                        hitToFill.setField(propertyEntry.getKey(),
+                                propertyEntry.getValue());
+                    }
+                }
+                hitToFill.setFilled(summaryClass == null ? DEFAULT_SUMMARY_CLASS : summaryClass);
+            }
+        }
+        hitsToFill.analyzeHits();
+    }
+
+    private Hit getMatchingFilledHit(URI hitToFillId, Result filledHits) {
+        Hit filledHit = null;
+
+        for ( Hit filledHitCandidate : filledHits.hits()) {
+            if ( hitToFillId == filledHitCandidate.getId() ) {
+                filledHit = filledHitCandidate;
+                break;
+            }
+        }
+        return filledHit;
+    }
+
+    private void copyFieldValuesThatExist(Hit filledHit, Hit hitToFill, Set<String> fieldsToFill) {
+        for (String fieldToFill : fieldsToFill ) {
+            if ( filledHit.getField(fieldToFill) != null ) {
+                hitToFill.setField(fieldToFill, filledHit.getField(fieldToFill));
+            }
+        }
+    }
+
+    /**
+     * Returns the number of queries made to this searcher since the last
+     * reset. For testing - not reliable if multiple threads makes
+     * queries simultaneously
+     */
+    public int getQueryCount() {
+        return queryCount;
+    }
+
+    public void resetQueryCount() {
+        queryCount=0;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java
new file mode 100644
index 00000000000..064e38d91fc
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java
@@ -0,0 +1,75 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * Searcher that sets cache control HTTP headers in response based on query/GET parameters to
+ * control caching done by proxy/caches such as YSquid and YTS:
+ * <ul>
+ *    <li>max-age=XXX - set with &amp;cachecontrol.maxage parameter
+ *    <li>stale-while-revalidate=YYY - set with &amp;cachecontrol.staleage
+ *    <li>no-cache - if Vespa &amp;noCache or &amp;cachecontrol.nocache parameter is set to true
+ * </ul>
+ *
+ * <p>This is controlled through the three query parameters <code>cachecontrol.maxage</code>,
+ * <code>cachecontrol.staleage</code> and <code>cachecontrol.nocache</code>, with the obvious meanings.</p>
+ *
+ * Example:
+ * <ul>
+ *    <li>Request: "?query=foo&amp;cachecontrol.maxage=60&amp;cachecontrol.staleage=3600"
+ *    <li>Response HTTP header: "Cache-Control: max-age=60, revalidate-while-stale=3600"
+ * </ul>
+ *
+ * Further documentation on use of Cache-Control headers:
+ * http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
+ *
+ * @author Frode Lundgren
+ */
+public class CacheControlSearcher extends Searcher {
+
+    private static final CompoundName cachecontrolNocache=new CompoundName("cachecontrol.nocache");
+    private static final CompoundName cachecontrolMaxage=new CompoundName("cachecontrol.maxage");
+    private static final CompoundName cachecontrolStaleage=new CompoundName("cachecontrol.staleage");
+
+    public static final String CACHE_CONTROL_HEADER = "Cache-Control";
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        query.trace("CacheControlSearcher: Running version $Revision$", false, 6);
+        Result result = execution.search(query);
+        query = result.getQuery();
+
+        if (result.getHeaders(true) == null) {
+            query.trace("CacheControlSearcher: No HTTP header map available - skipping searcher.", false, 5);
+            return result;
+        }
+
+        // If you specify no-cache, no further cache control headers make sense
+        if (query.properties().getBoolean(cachecontrolNocache, false) || query.getNoCache()) {
+            result.getHeaders(true).put(CACHE_CONTROL_HEADER, "no-cache");
+            query.trace("CacheControlSearcher: Added no-cache header", false, 4);
+            return result;
+        }
+
+        // Handle max-age header
+        int maxage = query.properties().getInteger(cachecontrolMaxage, -1);
+        if (maxage > 0) {
+            result.getHeaders(true).put(CACHE_CONTROL_HEADER, "max-age=" + maxage);
+            query.trace("CacheControlSearcher: Set max-age header to " + maxage, false, 4);
+        }
+
+        // Handle stale-while-revalidate header
+        int staleage = query.properties().getInteger(cachecontrolStaleage, -1);
+        if (staleage > 0) {
+            result.getHeaders(true).put(CACHE_CONTROL_HEADER, "stale-while-revalidate=" + staleage);
+            query.trace("CacheControlSearcher: Set stale-while-revalidate header to " + maxage, false, 4);
+        }
+
+        return result;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java
new file mode 100644
index 00000000000..cdbf864f7fd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongSupplier;
+
+/**
+ * Searcher which can enforce HTTP connection close based on query properties.
+ *
+ * <p>
+ * This searcher informs the client to close a persistent HTTP connection if the
+ * connection is older than the configured max lifetime. This is done by adding
+ * the "Connection" HTTP header with the value "Close" to the result.
+ * </p>
+ *
+ * <p>
+ * The searcher reads the query property "connectioncontrol.maxlifetime", which
+ * is an integer number of seconds, to get the value for maximum connection
+ * lifetime. Setting it to zero will enforce connection close independently of
+ * the age of the connection. Typical usage would be as follows:
+ * </p>
+ *
+ * <ol>
+ * <li>Add the ConnectionControlSearcher to the default search chain of your
+ * application. (It has no special ordering considerations.)</li>
+ *
+ * <li>For the default query profile of your application, set a reasonable value
+ * for "connectioncontrol.maxlifetime". The definition of reasonable will be
+ * highly application dependent, but it should always be less than the grace
+ * period when taking the container out of production traffic.</li>
+ *
+ * <li>Deploy application. The container will now inform clients to close
+ * connections/reconnect within the configured time limit.
+ * </ol>
+ *
+ * @author frodelu
+ * @author Steinar Knutsen
+ */
+public class ConnectionControlSearcher extends Searcher {
+
+    private final String simpleName = this.getClass().getSimpleName();
+
+    private final LongSupplier clock;
+
+    private static final CompoundName KEEPALIVE_MAXLIFETIMESECONDS = new CompoundName("connectioncontrol.maxlifetime");
+    private static final String HTTP_CONNECTION_HEADER_NAME = "Connection";
+    private static final String HTTP_CONNECTION_CLOSE_ARGUMENT = "Close";
+
+    public ConnectionControlSearcher() {
+        this(() -> System.currentTimeMillis());
+    }
+
+    private ConnectionControlSearcher(LongSupplier clock) {
+        this.clock = clock;
+    }
+
+    /**
+     * Create a searcher instance suitable for unit tests.
+     *
+     * @param clock a simulated or real clock behaving similarly to System.currentTimeMillis()
+     * @return a fully initialised instance
+     */
+    public static ConnectionControlSearcher createTestInstance(LongSupplier clock) {
+        return new ConnectionControlSearcher(clock);
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        Result result = execution.search(query);
+
+        query.trace(false, 3, simpleName, " updating headers.");
+        keepAliveProcessing(query, result);
+        return result;
+    }
+
+    /**
+     * If the HTTP connection has been alive for too long, set the header
+     * "Connection: Close" to tell the client to close the connection after this
+     * request.
+     */
+    private void keepAliveProcessing(Query query, Result result) {
+        int maxLifetimeSeconds = query.properties().getInteger(KEEPALIVE_MAXLIFETIMESECONDS, -1);
+
+        if (maxLifetimeSeconds < 0) {
+            return;
+        } else if (maxLifetimeSeconds == 0) {
+            result.getHeaders(true).put(HTTP_CONNECTION_HEADER_NAME, HTTP_CONNECTION_CLOSE_ARGUMENT);
+            query.trace(false, 5, simpleName, ": Max HTTP connection lifetime set to 0; adding \"", HTTP_CONNECTION_HEADER_NAME,
+                    ": ", HTTP_CONNECTION_CLOSE_ARGUMENT, "\" header");
+        } else {
+            setCloseIfLifetimeExceeded(query, result, maxLifetimeSeconds);
+        }
+    }
+
+    private void setCloseIfLifetimeExceeded(Query query, Result result, int maxLifetimeSeconds) {
+        final HttpRequest httpRequest = query.getHttpRequest();
+        if (httpRequest == null) {
+            query.trace(false, 5, simpleName, " got max lifetime = ", maxLifetimeSeconds,
+                    ", but got no JDisc request. Setting no header.");
+            return;
+        }
+
+        final long connectedAtMillis = httpRequest.getJDiscRequest().getConnectedAt(TimeUnit.MILLISECONDS);
+        final long maxLifeTimeMillis = maxLifetimeSeconds * 1000L;
+        if (connectedAtMillis + maxLifeTimeMillis < clock.getAsLong()) {
+            result.getHeaders(true).put(HTTP_CONNECTION_HEADER_NAME, HTTP_CONNECTION_CLOSE_ARGUMENT);
+            query.trace(false, 5, simpleName, ": Max HTTP connection lifetime (", maxLifetimeSeconds, ") exceeded; adding \"",
+                    HTTP_CONNECTION_HEADER_NAME, ": ", HTTP_CONNECTION_CLOSE_ARGUMENT, "\" header");
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java
new file mode 100644
index 00000000000..d99cb72f5a3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java
@@ -0,0 +1,191 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.metrics.simple.Counter;
+import com.yahoo.metrics.simple.MetricReceiver;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.TermItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * Check whether the query tree seems to be "well formed". In other words, run heurestics against
+ * the input data to see whether the query should sent to the search backend.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class InputCheckingSearcher extends Searcher {
+
+    private final Counter utfRejections;
+    private final Counter repeatedConsecutiveTermsInPhraseRejections;
+    private final Counter repeatedTermsInPhraseRejections;
+    private static final Logger log = Logger.getLogger(InputCheckingSearcher.class.getName());
+    private final int MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE = 5;
+    private final int MAX_REPEATED_TERMS_IN_PHRASE=10;
+
+    public InputCheckingSearcher(MetricReceiver metrics) {
+        utfRejections = metrics.declareCounter("double_encoded_utf8_rejections");
+        repeatedTermsInPhraseRejections = metrics.declareCounter("repeated_terms_in_phrase_rejections");
+        repeatedConsecutiveTermsInPhraseRejections = metrics.declareCounter("repeated_consecutive_terms_in_phrase_rejections");
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        try {
+            checkQuery(query);
+        } catch (IllegalArgumentException e) {
+            if (log.isLoggable(LogLevel.DEBUG)) {
+                log.log(LogLevel.DEBUG, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage());
+            }
+            return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage()));
+        }
+        return execution.search(query);
+    }
+
+    private void checkQuery(Query query) {
+        doubleEncodedUtf8(query);
+        checkPhrases(query.getModel().getQueryTree().getRoot());
+        // add new heuristics here
+    }
+
+    private void checkPhrases(Item queryItem) {
+        if (queryItem instanceof PhraseItem) {
+            PhraseItem phrase = (PhraseItem) queryItem;
+            repeatedConsecutiveTermsInPhraseCheck(phrase);
+            repeatedTermsInPhraseCheck(phrase);
+        } else  if (queryItem instanceof CompositeItem) {
+            CompositeItem asComposite = (CompositeItem) queryItem;
+            for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) {
+                checkPhrases(i.next());
+            }
+        }
+    }
+
+    private void repeatedConsecutiveTermsInPhraseCheck(PhraseItem phrase) {
+        if (phrase.getItemCount() > MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) {
+            String prev = null;
+            int repeatedCount = 0;
+            for (int i = 0; i < phrase.getItemCount(); ++i) {
+                Item item = phrase.getItem(i);
+                if (item instanceof TermItem) {
+                    TermItem term = (TermItem) item;
+                    String current = term.getIndexedString();
+                    if (prev != null) {
+                        if (prev.equals(current)) {
+                            repeatedCount++;
+                            if (repeatedCount >= MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE) {
+                                repeatedConsecutiveTermsInPhraseRejections.add();
+                                throw new IllegalArgumentException("More than " + MAX_REPEATED_CONSECUTIVE_TERMS_IN_PHRASE +
+                                        " ocurrences of term '" + current + "' in a row detected in phrase : " + phrase.toString());
+                            }
+                        } else {
+                            repeatedCount = 0;
+                        }
+                    }
+                    prev = current;
+                } else {
+                    prev = null;
+                    repeatedCount = 0;
+                }
+            }
+        }
+    }
+    private static final class Count {
+        private int v;
+        Count(int initial) { v = initial; }
+        void inc() { v++; }
+        int get() { return v; }
+    }
+    private void repeatedTermsInPhraseCheck(PhraseItem phrase) {
+        if (phrase.getItemCount() > MAX_REPEATED_TERMS_IN_PHRASE) {
+            Map<String, Count> repeatedCount = new HashMap<>();
+            for (int i = 0; i < phrase.getItemCount(); ++i) {
+                Item item = phrase.getItem(i);
+                if (item instanceof TermItem) {
+                    TermItem term = (TermItem) item;
+                    String current = term.getIndexedString();
+                    Count count = repeatedCount.get(current);
+                    if (count != null) {
+                        if (count.get() >= MAX_REPEATED_TERMS_IN_PHRASE) {
+                            repeatedTermsInPhraseRejections.add();
+                            throw new IllegalArgumentException("Phrase contains more than " + MAX_REPEATED_TERMS_IN_PHRASE +
+                                    " occurrences of term '" + current + "' in phrase : " + phrase.toString());
+                        }
+                        count.inc();
+                    } else {
+                        repeatedCount.put(current, new Count(1));
+                    }
+                }
+            }
+        }
+    }
+
+
+    private void doubleEncodedUtf8(Query query) {
+        int singleCharacterTerms = countSingleCharacterUserTerms(query.getModel().getQueryTree());
+        if (singleCharacterTerms <= 4) {
+            return;
+        }
+        String userInput = query.getModel().getQueryString();
+        ByteBuffer asOctets = ByteBuffer.allocate(userInput.length());
+        boolean asciiOnly = true;
+        for (int i = 0; i < userInput.length(); ++i) {
+            char c = userInput.charAt(i);
+            if (c > 255) {
+                return; // not double (or more) encoded
+            }
+            if (c > 127) {
+                asciiOnly = false;
+            }
+            asOctets.put((byte) c);
+        }
+        if (asciiOnly) {
+            return;
+        }
+        asOctets.flip();
+        CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
+                .onUnmappableCharacter(CodingErrorAction.REPORT);
+        // OK, unmappable character is sort of theoretical, but added to be explicit
+        try {
+            decoder.decode(asOctets);
+        } catch (CharacterCodingException e) {
+            return;
+        }
+        utfRejections.add();
+        throw new IllegalArgumentException("The user input has been determined to be double encoded UTF-8."
+                + " Please investigate whether this is a false positive.");
+    }
+
+    private int countSingleCharacterUserTerms(Item queryItem) {
+        if (queryItem instanceof CompositeItem) {
+            int sum = 0;
+            CompositeItem asComposite = (CompositeItem) queryItem;
+            for (ListIterator<Item> i = asComposite.getItemIterator(); i.hasNext();) {
+                sum += countSingleCharacterUserTerms(i.next());
+            }
+            return sum;
+        } else if (queryItem instanceof WordItem) {
+            WordItem word = (WordItem) queryItem;
+            return (word.isFromQuery() && word.stringValue().length() == 1) ? 1 : 0;
+        } else {
+            return 0;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java
new file mode 100755
index 00000000000..95cec1d0960
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java
@@ -0,0 +1,219 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ClusterInfoConfig;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.config.RateLimitingConfig;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.yolean.chain.Provides;
+
+import java.time.Clock;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * A simple rate limiter.
+ * <p>
+ * This takes these query parameter arguments:
+ * <ul>
+ *     <li>rate.id - (String) the id of the client from rate limiting perspective
+ *     <li>rate.cost - (Double) the cost Double of this query. This is read after executing the query and hence can be set
+ *     by downstream searchers inspecting the result to allow differencing the cost of various queries. Default is 1.
+ *     <li>rate.quota - (Double) the cost per second a particular id is allowed to consume in this system.
+ *     <li>rate.idDimension - (String) the name of the rate-id dimension used when logging metrics.
+ *                                 If this is not specified, the metric will be logged without dimensions.
+ *     <li>rate.dryRun - (Boolean) emit metrics on rejected requests but don't actually reject them
+ * </ul>
+ * <p>
+ * Whenever quota is exceeded for an id this searcher will reject queries from that id by
+ * returning a result containing a status 429 error.
+ * <p>
+ * If rate.id or rate.quota is not set in Query.properties this searcher will do nothing.
+ * <p>
+ * Metrics: This will emit the count metric requestsOverQuota with the dimension [rate.idDimension=rate.id]
+ * counting rejected requests.
+ * <p>
+ * Ordering: This searcher Provides rateLimiting
+ *
+ * @author bratseth
+ */
+@Provides(RateLimitingSearcher.RATE_LIMITING)
+public class RateLimitingSearcher extends Searcher {
+
+    /** Constant containing the name this Provides - "rateLimiting", for ordering constraints */
+    public static final String RATE_LIMITING = "rateLimiting";
+
+    public static final CompoundName idKey = new CompoundName("rate.id");
+    public static final CompoundName costKey = new CompoundName("rate.cost");
+    public static final CompoundName quotaKey = new CompoundName("rate.quota");
+    public static final CompoundName idDimensionKey = new CompoundName("rate.idDimension");
+    public static final CompoundName dryRunKey = new CompoundName("rate.dryRun");
+
+    private static final String requestsOverQuotaMetricName = "requestsOverQuota";
+
+    /** Used to divide quota by nodes. Assumption: All nodes get the same share of traffic. */
+    private final int nodeCount;
+
+    /** Shared capacity across all threads. Each thread will ask for more capacity from here when they run out. */
+    private final AvailableCapacity availableCapacity;
+
+    /** Capacity already allocated to this thread */
+    private final ThreadLocal<Map<String, Double>> allocatedCapacity = new ThreadLocal<>();
+
+    /** For emitting metrics */
+    private final Metric metric;
+
+    /**
+     * How much capacity to allocate to a thread each time it runs out.
+     * A higher value means less contention and less accuracy.
+     */
+    private final double capacityIncrement;
+
+    /** How often to check for new capacity if we have run out */
+    private final double recheckForCapacityProbability;
+
+    @Inject
+    public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, ClusterInfoConfig clusterInfoConfig, Metric metric) {
+        this(rateLimitingConfig, clusterInfoConfig, metric, Clock.systemUTC());
+    }
+
+    /** For testing - allows injection of a timer to avoid depending on the system clock */
+    public RateLimitingSearcher(RateLimitingConfig rateLimitingConfig, ClusterInfoConfig clusterInfoConfig, Metric metric, Clock clock) {
+        this.capacityIncrement = rateLimitingConfig.capacityIncrement();
+        this.recheckForCapacityProbability = rateLimitingConfig.recheckForCapacityProbability();
+        this.availableCapacity = new AvailableCapacity(rateLimitingConfig.maxAvailableCapacity(), clock);
+
+        this.nodeCount = clusterInfoConfig.nodeCount();
+
+        this.metric = metric;
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        String id = query.properties().getString(idKey);
+        Double rate = query.properties().getDouble(quotaKey);
+        if (id == null || rate == null) {
+            query.trace(false, 6, "Skipping rate limiting check. Need both " + idKey + " and " + quotaKey + " set");
+            return execution.search(query);
+        }
+
+        rate = rate / nodeCount;
+
+        if (allocatedCapacity.get() == null) // new thread
+            allocatedCapacity.set(new HashMap<>());
+        if (allocatedCapacity.get().get(id) == null) // new id in this thread
+            requestCapacity(id, rate);
+
+        // Check if there is capacity available. Cannot check for exact cost as it may be computed after execution
+        // no capacity means we're over rate. Only recheck occasionally to limit synchronization.
+        if (getAllocatedCapacity(id) <= 0 && ThreadLocalRandom.current().nextDouble() < recheckForCapacityProbability) {
+            requestCapacity(id, rate);
+        }
+
+        if (rate==0 || getAllocatedCapacity(id) <= 0) { // we are still over rate: reject
+            metric.add(requestsOverQuotaMetricName, 1, createContext(query.properties().getString(idDimensionKey, ""), id));
+            if ( ! query.properties().getBoolean(dryRunKey, false))
+                return new Result(query, new ErrorMessage(429, "Too many requests", "Allowed rate: " + rate + "/s"));
+        }
+
+        Result result = execution.search(query);
+        addAllocatedCapacity(id, - query.properties().getDouble(costKey, 1.0));
+
+        if (getAllocatedCapacity(id) <= 0) // make sure we ask for more with 100% probability when first running out
+            requestCapacity(id, rate);
+
+        return result;
+    }
+
+    private Metric.Context createContext(String dimensionName, String dimensionValue) {
+        if (dimensionName.isEmpty())
+            return metric.createContext(Collections.emptyMap());
+        return metric.createContext(Collections.singletonMap(dimensionName, dimensionValue));
+    }
+
+    private double getAllocatedCapacity(String id) {
+        Double value = allocatedCapacity.get().get(id);
+        if (value == null) return 0;
+        return value;
+    }
+
+    private void addAllocatedCapacity(String id, double newCapacity) {
+        Double capacity = allocatedCapacity.get().get(id);
+        if (capacity != null)
+            newCapacity += capacity;
+        allocatedCapacity.get().put(id, newCapacity);
+    }
+
+    private void requestCapacity(String id, double rate) {
+        double minimumRequested = Math.max(0, -getAllocatedCapacity(id)); // If we are below, make sure we reach 0
+        double preferredRequested = Math.max(capacityIncrement, -getAllocatedCapacity(id));
+        addAllocatedCapacity(id, availableCapacity.request(id, minimumRequested, preferredRequested, rate));
+    }
+
+    /**
+     * This keeps track of the current "capacity" (total cost) available to each client (rate id)
+     * across all threads. Capacity is supplied at the rate per second given by the clients quota.
+     * When all the capacity is spent, no further capacity will be handed out, leading to request rejection.
+     * Capacity has a max value it will never exceed to avoid clients saving capacity for future overspending.
+     */
+    private static class AvailableCapacity {
+
+        private final double maxAvailableCapacity;
+        private final Clock clock;
+
+        private final Map<String, CapacityAllocation> available = new HashMap<>();
+
+        public AvailableCapacity(double maxAvailableCapacity, Clock clock) {
+            this.maxAvailableCapacity = maxAvailableCapacity;
+            this.clock = clock;
+        }
+
+        /** Returns an amount of capacity between 0 and the requested amount based on availability for this id */
+        public synchronized double request(String id, double minimumRequested, double preferredRequested, double rate) {
+            CapacityAllocation allocation = available.get(id);
+            if (allocation == null) {
+                allocation = new CapacityAllocation(rate, clock);
+                available.put(id, allocation);
+            }
+            return allocation.request(minimumRequested, preferredRequested, rate, maxAvailableCapacity);
+        }
+
+    }
+
+    private static class CapacityAllocation {
+
+        private double capacity;
+        private final Clock clock;
+        private long lastAllocatedTime;
+
+        public CapacityAllocation(double initialCapacity, Clock clock) {
+            this.capacity = initialCapacity;
+            this.clock = clock;
+            lastAllocatedTime = clock.millis();
+        }
+
+        public double request(double minimumRequested, double preferredRequested, double rate, double maxAvailableCapacity) {
+            if ( preferredRequested > capacity) { // attempt to allocate more
+                // rate is per second so we get rate/1000 per millisecond
+                long currentTime = clock.millis();
+                capacity += Math.min(maxAvailableCapacity, rate/1000d * (Math.max(0, currentTime - lastAllocatedTime)));
+                lastAllocatedTime = currentTime;
+            }
+            double grantedCapacity = Math.min(capacity/10, preferredRequested); // /10 to avoid stealing all capacity when low
+            if (grantedCapacity < minimumRequested)
+                grantedCapacity = Math.min(minimumRequested, capacity);
+            capacity = capacity - grantedCapacity;
+            return grantedCapacity;
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java
new file mode 100644
index 00000000000..ff00c8edb9b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateMatchPhaseSearcher.java
@@ -0,0 +1,69 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.searchers;
+
+import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vespa.config.search.AttributesConfig;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Validates that the attribute given as match-phase override is actually a valid numeric attribute
+ * with fast-search enabled.
+ * Created by balder on 1/21/15.
+ */
+public class ValidateMatchPhaseSearcher extends Searcher {
+    private Set<String> validMatchPhaseAttributes = new HashSet<>();
+    private Set<String> validDiversityAttributes = new HashSet<>();
+    public ValidateMatchPhaseSearcher(AttributesConfig attributesConfig) {
+        for (AttributesConfig.Attribute a : attributesConfig.attribute()) {
+            if (a.fastsearch() &&
+                (a.collectiontype() == AttributesConfig.Attribute.Collectiontype.SINGLE) &&
+                isNumeric(a.datatype()))
+            {
+                validMatchPhaseAttributes.add(a.name());
+            }
+        }
+        for (AttributesConfig.Attribute a : attributesConfig.attribute()) {
+            if ((a.collectiontype() == AttributesConfig.Attribute.Collectiontype.SINGLE) &&
+                ((a.datatype() == AttributesConfig.Attribute.Datatype.STRING) || isNumeric(a.datatype())))
+            {
+                validDiversityAttributes.add(a.name());
+            }
+        }
+    }
+    private boolean isNumeric(AttributesConfig.Attribute.Datatype.Enum dt) {
+        return  dt == AttributesConfig.Attribute.Datatype.DOUBLE ||
+                dt == AttributesConfig.Attribute.Datatype.FLOAT ||
+                dt == AttributesConfig.Attribute.Datatype.INT8 ||
+                dt == AttributesConfig.Attribute.Datatype.INT16 ||
+                dt == AttributesConfig.Attribute.Datatype.INT32 ||
+                dt == AttributesConfig.Attribute.Datatype.INT64;
+    }
+    @Override
+    public Result search(Query query, Execution execution) {
+        ErrorMessage e = validate(query);
+        return (e != null)
+                ? new Result(query, e)
+                : execution.search(query);
+    }
+
+    private ErrorMessage validate(Query query) {
+        String attribute = query.getRanking().getMatchPhase().getAttribute();
+        if ( attribute != null && ! validMatchPhaseAttributes.contains(attribute) ) {
+            return ErrorMessage.createInvalidQueryParameter("The attribute '" + attribute + "' is not available for match-phase. " +
+                                                            "It must be a single value numeric attribute with fast-search.");
+        }
+        attribute = query.getRanking().getMatchPhase().getDiversity().getAttribute();
+        if (attribute != null && ! validDiversityAttributes.contains(attribute)) {
+            return ErrorMessage.createInvalidQueryParameter("The attribute '" + attribute + "' is not available for match-phase diversification. " +
+                    "It must be a single value numeric or string attribute.");
+        }
+        return null;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/package-info.java b/container-search/src/main/java/com/yahoo/search/searchers/package-info.java
new file mode 100644
index 00000000000..78f1e5940a6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/package-info.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Various useful searchers
+ */
+@ExportPackage
+@PublicApi
+package com.yahoo.search.searchers;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/statistics/ElapsedTime.java b/container-search/src/main/java/com/yahoo/search/statistics/ElapsedTime.java
new file mode 100644
index 00000000000..8cf159f5ad8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/statistics/ElapsedTime.java
@@ -0,0 +1,235 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.statistics;
+
+import com.yahoo.collections.TinyIdentitySet;
+import com.yahoo.search.statistics.TimeTracker.Activity;
+import com.yahoo.search.statistics.TimeTracker.SearcherTimer;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static com.yahoo.search.statistics.TimeTracker.Activity.*;
+
+/**
+ * Basically a collection of TimeTracker instances.
+ *
+ * <p>This class may need a lot of restructuring as actual
+ * needs are mapped out.
+ *
+ * @author <a href="steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class ElapsedTime {
+
+    // An identity set is used to make it safe to do multiple merges. This may happen if
+    // user calls Result.mergeWith() and Result.mergeWithAfterFill() on the same result
+    // with the same result as an argument too. This is slightly pathological, but better
+    // safe than sorry. It also covers in SearchHandler where the same Execution instance
+    // is used for search and fill.
+    /** A map used as a set to store the time track of all the Execution instances for a Result */
+    private Set<TimeTracker> tracks = new TinyIdentitySet<>(8);
+
+    public void add(TimeTracker track) {
+        tracks.add(track);
+    }
+
+    private long fetcher(Activity toFetch, TimeTracker fetchFrom) {
+        switch (toFetch) {
+        case SEARCH:
+            return fetchFrom.searchTime();
+        case FILL:
+            return fetchFrom.fillTime();
+        case PING:
+            return fetchFrom.pingTime();
+        default:
+            return 0L;
+        }
+
+    }
+
+    /**
+     * Give an estimate on how much of the time tracked by this
+     * instance was used fetching document contents. This will
+     * by definition be smaller than last() - first().
+     */
+    public long weightedFillTime() {
+        return weightedTime(FILL);
+    }
+
+    private long weightedTime(Activity kind) {
+        long total = 0L;
+        long elapsed = 0L;
+        long first = Long.MAX_VALUE;
+        long last = 0L;
+
+        if (tracks.isEmpty()) {
+            return 0L;
+        }
+        for (TimeTracker track : tracks) {
+            total += track.totalTime();
+            elapsed += fetcher(kind, track);
+            last = Math.max(last, track.last());
+            first = Math.min(first, track.first());
+        }
+        if (total == 0L) {
+            return 0L;
+        } else {
+            return ((last - first) * elapsed) / total;
+        }
+    }
+
+    private long absoluteTime(Activity kind) {
+        long elapsed = 0L;
+
+        if (tracks.isEmpty()) {
+            return 0L;
+        }
+        for (TimeTracker track : tracks) {
+            elapsed += fetcher(kind, track);
+        }
+        return elapsed;
+    }
+
+    /**
+     * Total amount of time spent in all threads for this Execution while
+     * fetching document contents, or preparing to fetch them.
+     */
+    public long fillTime() {
+        return absoluteTime(FILL);
+    }
+
+    /**
+     * Total amount of time spent for this ElapsedTime instance.
+     */
+    public long totalTime() {
+        long total = 0L;
+        for (TimeTracker track : tracks) {
+            total  += track.totalTime();
+        }
+        return total;
+    }
+
+    /**
+     * Give a relative estimate on how much of the time tracked by this
+     * instance was used searching. This will
+     * by definition be smaller than last() - first().
+     */
+    public long weightedSearchTime() {
+        return weightedTime(SEARCH);
+    }
+
+    /**
+     * Total amount of time spent in all threads for this Execution while
+     * searching or waiting for (a) backend(s) doing (a) search(es).
+     */
+    public long searchTime() {
+        return absoluteTime(SEARCH);
+    }
+
+    /**
+     * Total amount of time spent in all threads for this Execution while
+     * pinging, or preparing to ping, a backend.
+     */
+    public long pingTime() {
+        return absoluteTime(PING);
+    }
+
+    /**
+     * Give a relative estimate on how much of the time tracked by this
+     * instance was used pinging backends. This will
+     * by definition be smaller than last() - first().
+     */
+    public long weightedPingTime() {
+        return weightedTime(PING);
+    }
+
+    /**
+     * Time stamp of start of the first event registered.
+     */
+    public long first() {
+        long first = Long.MAX_VALUE;
+        for (TimeTracker track : tracks) {
+            first = Math.min(first, track.first());
+        }
+        return first;
+    }
+
+    /**
+     * Time stamp of the end the last event registered.
+     */
+    public long last() {
+        long last = 0L;
+        for (TimeTracker track : tracks) {
+            last = Math.max(last, track.last());
+        }
+        return last;
+    }
+
+    public void merge(ElapsedTime other) {
+        for (TimeTracker t : other.tracks) {
+            add(t);
+        }
+    }
+
+    /**
+     * The time of the start of the first document fill requested.
+     */
+    public long firstFill() {
+        long first = Long.MAX_VALUE;
+        if (tracks.isEmpty()) {
+            return 0L;
+        }
+        for (TimeTracker t : tracks) {
+            long candidate = t.firstFill();
+            if (candidate == 0L) {
+                continue;
+            }
+            first = Math.min(first, t.firstFill());
+        }
+        return first;
+    }
+
+    /*
+     * Tell whether time use per searcher is available.
+     */
+    public boolean hasDetailedData() {
+        for (TimeTracker t : tracks) {
+            if (t.searcherTracking() != null) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public String detailedReport() {
+        Map<String, TimeTracker.SearcherTimer> raw = new LinkedHashMap<>();
+        StringBuilder report = new StringBuilder();
+        int preLen;
+        report.append("Time use per searcher: ");
+        for (TimeTracker t : tracks) {
+            if (t.searcherTracking() == null) {
+                continue;
+            }
+            SearcherTimer[] searchers = t.searcherTracking();
+            for (SearcherTimer s : searchers) {
+                SearcherTimer sum;
+                if (raw.containsKey(s.getName())) {
+                    sum = raw.get(s.getName());
+                } else {
+                    sum = new SearcherTimer(s.getName());
+                    raw.put(s.getName(), sum);
+                }
+                sum.merge(s);
+            }
+        }
+        preLen = report.length();
+        for (TimeTracker.SearcherTimer value : raw.values()) {
+            if (report.length() > preLen) {
+                report.append(",\n    ");
+            }
+            report.append(value.toString());
+        }
+        report.append(".");
+        return report.toString();
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java
new file mode 100644
index 00000000000..e6056659c55
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java
@@ -0,0 +1,237 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.statistics;
+
+import com.yahoo.collections.Tuple2;
+import com.yahoo.concurrent.ThreadLocalDirectory;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.statistics.Callback;
+import com.yahoo.statistics.Handle;
+import com.yahoo.statistics.Statistics;
+import com.yahoo.statistics.Value;
+
+import java.util.*;
+
+/**
+ * Aggregate peak qps and expose through meta hits and/or log events.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class PeakQpsSearcher extends Searcher {
+    private final ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> directory;
+    private final Value qpsStatistics;
+    private final CompoundName propertyName;
+    private final boolean useMetaHit;
+
+    /**
+     * Meta hit which carries the peak qps and mean qps since the last time this
+     * data was requested. The URI is always "meta:qps". The data is stored as
+     * Number subclasses in the fields named by the fields PEAK_QPS and MEAN_QPS
+     * in the QpsHit class.
+     */
+    public static class QpsHit extends Hit {
+        /**
+         * Machine generated version ID for serialization.
+         */
+        private static final long serialVersionUID = 1042868845398233889L;
+
+        /**
+         * The name of the field containing mean QPS since the last measurement.
+         */
+        public static final String MEAN_QPS = "mean_qps";
+
+        /**
+         * The name of the field containing peak QPS since the last measurement.
+         */
+        public static final String PEAK_QPS = "peak_qps";
+        public static final String SCHEME = "meta";
+
+        public QpsHit(Integer peakQps, Double meanQps) {
+            super(SCHEME + ":qps");
+            setField(PEAK_QPS, peakQps);
+            setField(MEAN_QPS, meanQps);
+        }
+
+        public boolean isMeta() {
+            return true;
+        }
+
+        @Override
+        public String toString() {
+            return "QPS hit: Peak QPS " + getField(PEAK_QPS) + ", mean QPS " + getField(MEAN_QPS) + ".";
+        }
+    }
+
+    static class QueryRatePerSecond {
+        long when;
+        int howMany;
+
+        QueryRatePerSecond(long when) {
+            this.when = when;
+            this.howMany = 0;
+        }
+
+        void add(int x) {
+            howMany += x;
+        }
+
+        void increment() {
+            howMany += 1;
+        }
+
+        @Override
+        public String toString() {
+            return "QueryRatePerSecond(" + when + ": " + howMany + ")";
+        }
+    }
+
+    static class QueryRate implements
+            ThreadLocalDirectory.Updater<Deque<QueryRatePerSecond>, Long> {
+        @Override
+        public Deque<QueryRatePerSecond> update(
+                Deque<QueryRatePerSecond> current, Long when) {
+            QueryRatePerSecond last = current.peekLast();
+            if (last == null || last.when != when) {
+                last = new QueryRatePerSecond(when);
+                current.addLast(last);
+            }
+            last.increment();
+            return current;
+        }
+
+        @Override
+        public Deque<QueryRatePerSecond> createGenerationInstance(
+                Deque<QueryRatePerSecond> previous) {
+            if (previous == null) {
+                return new ArrayDeque<>();
+            } else {
+                return new ArrayDeque<>(previous.size());
+            }
+        }
+    }
+
+    private class Fetcher implements Callback {
+        @Override
+        public void run(Handle h, boolean firstRun) {
+            List<Deque<QueryRatePerSecond>> data = directory.fetch();
+            List<QueryRatePerSecond> chewed = merge(data);
+            for (QueryRatePerSecond qps : chewed) {
+                qpsStatistics.put((double) qps.howMany);
+            }
+        }
+    }
+
+    public PeakQpsSearcher(MeasureQpsConfig config, Statistics manager) {
+        directory = createDirectory();
+        MeasureQpsConfig.Outputmethod.Enum method = config.outputmethod();
+        if (method == MeasureQpsConfig.Outputmethod.METAHIT) {
+            useMetaHit = true;
+            propertyName = new CompoundName(config.queryproperty());
+            qpsStatistics = null;
+        } else if (method == MeasureQpsConfig.Outputmethod.STATISTICS) {
+            String event = config.eventname();
+            if (event == null || event.isEmpty()) {
+                event = getId().getName();
+                event = event.replace('.', '_');
+            }
+            qpsStatistics = new Value(event, manager, new Value.Parameters()
+                    .setAppendChar('_').setLogMax(true).setLogMean(true)
+                    .setLogMin(false).setLogRaw(false).setNameExtension(true)
+                    .setCallback(new Fetcher()));
+            useMetaHit = false;
+            propertyName = null;
+        } else {
+            throw new IllegalArgumentException(
+                    "Config definition out of sync with implementation."
+                    + " No way to create output for method " + method + ".");
+        }
+    }
+
+    static ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> createDirectory() {
+        return new ThreadLocalDirectory<>(new QueryRate());
+    }
+
+    static List<QueryRatePerSecond> merge(List<Deque<QueryRatePerSecond>> measurements) {
+        List<QueryRatePerSecond> rates = new ArrayList<>();
+        while (measurements.size() > 0) {
+            Deque<Deque<QueryRatePerSecond>> consumeFrom
+                    = new ArrayDeque<>(measurements.size());
+            long current = Long.MAX_VALUE;
+            for (ListIterator<Deque<QueryRatePerSecond>> i = measurements.listIterator(); i.hasNext();) {
+                Deque<QueryRatePerSecond> deck = i.next();
+                if (deck.size() == 0) {
+                    i.remove();
+                    continue;
+                }
+                QueryRatePerSecond threadData = deck.peekFirst();
+                if (threadData.when < current) {
+                    consumeFrom.clear();
+                    current = threadData.when;
+                    consumeFrom.add(deck);
+                } else if (threadData.when == current) {
+                    consumeFrom.add(deck);
+                }
+            }
+            if (consumeFrom.size() > 0) {
+                rates.add(consume(consumeFrom));
+            }
+        }
+        return rates;
+    }
+
+    private static QueryRatePerSecond consume(Deque<Deque<QueryRatePerSecond>> consumeFrom) {
+        Deque<QueryRatePerSecond> valueQueue = consumeFrom.pop();
+        QueryRatePerSecond value = valueQueue.pop();
+        QueryRatePerSecond thisSecond = new QueryRatePerSecond(value.when);
+        thisSecond.add(value.howMany);
+        while (consumeFrom.size() > 0) {
+            valueQueue = consumeFrom.pop();
+            value = valueQueue.pop();
+            thisSecond.add(value.howMany);
+        }
+        return thisSecond;
+
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        Result r;
+        long when = query.getStartTime() / 1000L;
+        Hit meta = null;
+        directory.update(when);
+        if (useMetaHit) {
+            if (query.properties().getBoolean(propertyName, false)) {
+                List<QueryRatePerSecond> l = merge(directory.fetch());
+                Tuple2<Integer, Double> maxAndMean = maxAndMean(l);
+                meta = new QpsHit(maxAndMean.first, maxAndMean.second);
+            }
+        }
+        r = execution.search(query);
+        if (meta != null) {
+            r.hits().add(meta);
+        }
+        return r;
+    }
+
+    private Tuple2<Integer, Double> maxAndMean(List<QueryRatePerSecond> l) {
+        int max = Integer.MIN_VALUE;
+        double sum = 0.0d;
+        if (l.size() == 0) {
+            return new Tuple2<>(Integer.valueOf(0),
+                    Double.valueOf(0.0));
+        }
+        for (QueryRatePerSecond qps : l) {
+            sum += (double) qps.howMany;
+            if (qps.howMany > max) {
+                max = qps.howMany;
+            }
+        }
+        return new Tuple2<>(Integer.valueOf(max),
+                Double.valueOf(sum / (double) l.size()));
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/statistics/TimeTracker.java b/container-search/src/main/java/com/yahoo/search/statistics/TimeTracker.java
new file mode 100644
index 00000000000..6d23701b06a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/statistics/TimeTracker.java
@@ -0,0 +1,390 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.statistics;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.prelude.Pong;
+import com.yahoo.processing.Processor;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+
+/**
+ * A container for storing time stamps throughout the
+ * lifetime of an Execution instance.
+ *
+ * <p>Check state both when entering and exiting, to allow for arbitrary
+ * new queries anywhere inside a search chain.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public final class TimeTracker {
+
+    public enum Activity {
+        PING,
+        SEARCH,
+        FILL;
+    }
+
+    static class SearcherTimer {
+        // Searcher ID
+        private final String name;
+        // Time spent transforming query/producing result
+        private final EnumMap<Activity, Long> invoking = new EnumMap<>(Activity.class);
+        // Time spent transforming result
+        private final EnumMap<Activity, Long> returning = new EnumMap<>(Activity.class);
+
+        SearcherTimer(String name) {
+            this.name = name;
+        }
+
+        private void activityRepr(StringBuilder buffer, int preLen,
+                Map.Entry<Activity, Long> m) {
+            if (buffer.length() != preLen) {
+                buffer.append(", ");
+            }
+            buffer.append(m.getKey()).append(": ").append(m.getValue())
+                    .append(" ms");
+        }
+
+        void addInvoking(Activity activity, long time) {
+            Long storedTillNow = invoking.get(activity);
+            long tillNow = getTime(storedTillNow);
+            invoking.put(activity, Long.valueOf(tillNow + time));
+        }
+
+        void addReturning(Activity activity, long time) {
+            Long storedTillNow = returning.get(activity);
+            long tillNow = getTime(storedTillNow);
+            returning.put(activity, Long.valueOf(tillNow + time));
+        }
+
+        Long getInvoking(Activity activity) {
+            return invoking.get(activity);
+        }
+
+        String getName() {
+            return name;
+        }
+
+        Long getReturning(Activity activity) {
+            return returning.get(activity);
+        }
+
+        private long getTime(Long storedTillNow) {
+            long tillNow;
+            if (storedTillNow == null) {
+                tillNow = 0L;
+            } else {
+                tillNow = storedTillNow.longValue();
+            }
+            return tillNow;
+        }
+
+        public void merge(SearcherTimer other) {
+            for (Map.Entry<Activity, Long> invokingEntry : other.invoking.entrySet()) {
+                addInvoking(invokingEntry.getKey(), invokingEntry.getValue());
+            }
+            for (Map.Entry<Activity, Long> returningEntry : other.returning.entrySet()) {
+                addReturning(returningEntry.getKey(), returningEntry.getValue());
+            }
+        }
+
+        public String toString() {
+            StringBuilder buffer = new StringBuilder();
+            int preLen;
+            buffer.append(name).append("(").append("QueryProcessing(");
+            preLen = buffer.length();
+            for (Map.Entry<Activity, Long> m : invoking.entrySet()) {
+                activityRepr(buffer, preLen, m);
+            }
+            buffer.append("), ResultProcessing(");
+            preLen = buffer.length();
+            for (Map.Entry<Activity, Long> m : returning.entrySet()) {
+                activityRepr(buffer, preLen, m);
+            }
+            buffer.append("))");
+            return buffer.toString();
+        }
+    }
+
+    static class State {
+        public final long start;
+        public final Activity activity;
+
+        State(long start, Activity activity) {
+            super();
+            this.start = start;
+            this.activity = activity;
+        }
+    }
+
+    static class Tag {
+        public final long start;
+        public final long end;
+        public final Activity activity;
+
+        Tag(long start, long end, Activity activity) {
+            super();
+            this.start = start;
+            this.end = end;
+            this.activity = activity;
+        }
+    }
+
+    static class TimeSource {
+        long now() {
+            return System.currentTimeMillis();
+        }
+    }
+
+    private State state = null;
+    private List<Tag> tags = new ArrayList<>();
+
+    private SearcherTimer[] searcherTracking = null;
+    private final Chain<? extends Processor> searchChain;
+    // whether the previous state was invoking or returning
+    private boolean invoking = true;
+    private long last = 0L;
+    private final int entryIndex;
+    TimeSource timeSource = new TimeSource();
+
+    public TimeTracker(Chain<? extends Searcher> searchChain) {
+        this(searchChain, 0);
+    }
+
+    public TimeTracker(Chain<? extends Processor> searchChain, int entryIndex) {
+        this.searchChain = searchChain;
+        this.entryIndex = entryIndex;
+    }
+
+    private void concludeState(long now) {
+        if (state == null) {
+            return;
+        }
+
+        tags.add(new Tag(state.start, now, state.activity));
+        state = null;
+    }
+
+    private void concludeStateOnExit(long now) {
+        if (now != 0L) {
+            concludeState(now);
+        } else {
+            concludeState(getNow());
+        }
+    }
+
+    private long detailedMeasurements(int searcherIndex, boolean calledAsInvoking) {
+        long now = getNow();
+        if (searcherTracking == null) {
+            initBreakdown();
+        }
+        SearcherTimer timeSpentIn = getPreviouslyRunSearcher(searcherIndex, calledAsInvoking);
+        long spent = now - last;
+        if (timeSpentIn != null && last != 0L) {
+            if (invoking) {
+                timeSpentIn.addInvoking(getActivity(), spent);
+            } else {
+                timeSpentIn.addReturning(getActivity(), spent);
+            }
+        }
+        last = now;
+        if (searcherIndex >= searcherTracking.length) {
+            // We are now outside the search chain and will go back up with the
+            // default result.
+            invoking = false;
+        } else {
+            invoking = calledAsInvoking;
+        }
+        return now;
+    }
+
+    private void enteringState(int searcherIndex, boolean detailed, final Activity activity) {
+        long now = 0L;
+        if (detailed) {
+            now = detailedMeasurements(searcherIndex, true);
+        }
+        if (isNewState(activity)) {
+            if (now == 0L) {
+                now = getNow();
+            }
+            concludeState(now);
+            initNewState(now, activity);
+        } else {
+            return;
+        }
+    }
+
+    private long fetchTime(Activity filter, Tag container) {
+        if (filter == container.activity) {
+            return container.end - container.start;
+        } else {
+            return 0L;
+        }
+    }
+
+    public long fillTime() {
+        return typedSum(Activity.FILL);
+    }
+
+    public long first() {
+        if (tags.isEmpty()) {
+            return 0L;
+        } else {
+            return tags.get(0).start;
+        }
+    }
+
+    public long firstFill() {
+        for (Tag t : tags) {
+            if (t.activity == Activity.FILL) {
+                return t.start;
+            }
+        }
+        return 0L;
+    }
+
+    private Activity getActivity() {
+        if (state == null) {
+            throw new IllegalStateException("Trying to measure an interval having only one point.");
+        }
+        return state.activity;
+    }
+
+    private long getNow() {
+        return timeSource.now();
+    }
+
+    private SearcherTimer getPreviouslyRunSearcher(int searcherIndex, boolean calledAsInvoking) {
+        if (calledAsInvoking) {
+            searcherIndex -= 1;
+            if (searcherIndex < entryIndex) {
+                return null;
+            } else {
+                return searcherTracking[searcherIndex];
+            }
+        } else {
+            return searcherTracking[searcherIndex];
+        }
+    }
+
+    private void initBreakdown() {
+        if (searcherTracking != null) {
+            throw new IllegalStateException("initBreakdown invoked"
+                    + " when measurement structures are already initialized.");
+        }
+        List<? extends Processor> searchers = searchChain.components();
+        searcherTracking = new SearcherTimer[searchers.size()];
+        for (int i = 0; i < searcherTracking.length; ++i) {
+            searcherTracking[i] = new SearcherTimer(searchers.get(i).getId().stringValue());
+        }
+    }
+
+    private void initNewState(long now, Activity activity) {
+        state = new State(now, activity);
+    }
+
+    void injectTimeSource(TimeSource source) {
+        this.timeSource = source;
+    }
+
+    private boolean isNewState(Activity callPath) {
+        if (state == null) {
+            return true;
+        } else if (callPath == state.activity) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    public long last() {
+        if (tags.isEmpty()) {
+            return 0L;
+        } else {
+            return tags.get(tags.size() - 1).end;
+        }
+    }
+
+    public long pingTime() {
+        return typedSum(Activity.PING);
+    }
+
+    private long returnfromState(int searcherIndex, boolean detailed) {
+        if (detailed) {
+            return detailedMeasurements(searcherIndex, false);
+        } else {
+            return 0L;
+        }
+    }
+
+    public void sampleFill(int searcherIndex, boolean detailed) {
+        enteringState(searcherIndex, detailed, Activity.FILL);
+    }
+
+    public void sampleFillReturn(int searcherIndex, boolean detailed, Result annotationReference) {
+        ElapsedTime elapsed = getElapsedTime(annotationReference);
+        sampleReturn(searcherIndex, detailed, elapsed);
+    }
+
+    public void samplePing(int searcherIndex, boolean detailed) {
+        enteringState(searcherIndex, detailed, Activity.PING);
+    }
+
+    public void samplePingReturn(int searcherIndex, boolean detailed, Pong annotationReference) {
+        ElapsedTime elapsed = getElapsedTime(annotationReference);
+        sampleReturn(searcherIndex, detailed, elapsed);
+    }
+
+    public void sampleSearch(int searcherIndex, boolean detailed) {
+        enteringState(searcherIndex, detailed, Activity.SEARCH);
+    }
+
+    public void sampleSearchReturn(int searcherIndex, boolean detailed, Result annotationReference) {
+        ElapsedTime elapsed = getElapsedTime(annotationReference);
+        sampleReturn(searcherIndex, detailed, elapsed);
+    }
+
+    private void sampleReturn(int searcherIndex, boolean detailed, ElapsedTime elapsed) {
+        long now = returnfromState(searcherIndex, detailed);
+        if (searcherIndex == entryIndex) {
+            concludeStateOnExit(now);
+            if (elapsed != null) {
+                elapsed.add(this);
+            }
+        }
+    }
+
+    private ElapsedTime getElapsedTime(Result r) {
+        return r == null ? null : r.getElapsedTime();
+    }
+
+    private ElapsedTime getElapsedTime(Pong p) {
+        return p == null ? null : p.getElapsedTime();
+    }
+
+    SearcherTimer[] searcherTracking() {
+        return searcherTracking;
+    }
+
+    public long searchTime() {
+        return typedSum(Activity.SEARCH);
+    }
+
+    public long totalTime() {
+        return last() - first();
+    }
+
+    private long typedSum(Activity activity) {
+        long sum = 0L;
+        for (Tag tag : tags) {
+            sum += fetchTime(activity, tag);
+        }
+        return sum;
+    }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java
new file mode 100644
index 00000000000..0b16c87df07
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java
@@ -0,0 +1,144 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.statistics;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.chain.dependencies.Before;
+import com.yahoo.search.statistics.TimingSearcherConfig.Timer;
+import com.yahoo.prelude.Ping;
+import com.yahoo.prelude.Pong;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.cluster.PingableSearcher;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.statistics.TimeTracker.Activity;
+import com.yahoo.statistics.Statistics;
+import com.yahoo.statistics.Value;
+
+
+/**
+ * A searcher which is intended to be useful as a general probe for
+ * measuring time consumption a search chain.
+ *
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@Before("rawQuery")
+public class TimingSearcher extends PingableSearcher {
+    private Value measurements;
+    private final boolean measurePing;
+    private final boolean measureSearch;
+    private final boolean measureFill;
+    private static final Parameters defaultParameters = new Parameters(null, Activity.SEARCH);
+
+    public static class Parameters {
+        final String eventName;
+        final Activity pathToSample;
+
+        public Parameters(String eventName, Activity pathToSample) {
+            super();
+            this.eventName = eventName;
+            this.pathToSample = pathToSample;
+        }
+    }
+
+    TimingSearcher(ComponentId id, Parameters setUp, Statistics manager) {
+        super(id);
+        if (setUp == null) {
+            setUp = defaultParameters;
+        }
+        String eventName = setUp.eventName;
+        if (eventName == null || "".equals(eventName)) {
+            eventName = id.getName();
+        }
+        measurements = new Value(eventName, manager, new Value.Parameters()
+                .setNameExtension(true).setLogMax(true).setLogMin(true)
+                .setLogMean(true).setLogSum(true).setLogInsertions(true)
+                .setAppendChar('_'));
+
+        measurePing = setUp.pathToSample == Activity.PING;
+        measureSearch = setUp.pathToSample == Activity.SEARCH;
+        measureFill = setUp.pathToSample == Activity.FILL;
+    }
+
+    public TimingSearcher(ComponentId id, TimingSearcherConfig config, Statistics manager) {
+        this(id, buildParameters(config, id.getName()), manager);
+    }
+
+    private static Parameters buildParameters(
+            TimingSearcherConfig config, String searcherName) {
+        for (int i = 0; i < config.timer().size(); ++i) {
+            Timer t = config.timer(i);
+            if (t.name().equals(searcherName)) {
+                return buildParameters(t);
+            }
+        }
+        return null;
+    }
+
+    private static Parameters buildParameters(Timer t) {
+        Activity m;
+        Timer.Measure.Enum toSample = t.measure();
+        if (toSample == Timer.Measure.FILL) {
+            m = Activity.FILL;
+        } else if (toSample == Timer.Measure.PING) {
+            m = Activity.PING;
+        } else {
+            m = Activity.SEARCH;
+        }
+        return new Parameters(t.eventname(), m);
+    }
+
+    private long preMeasure(boolean doIt) {
+        if (doIt) {
+            return System.currentTimeMillis();
+        } else {
+            return 0L;
+        }
+    }
+
+    private void postMeasure(boolean doIt, long start) {
+        if (doIt) {
+            long elapsed = System.currentTimeMillis() - start;
+            measurements.put(elapsed);
+        }
+    }
+
+    @Override
+    public void fill(Result result, String summaryClass, Execution execution) {
+        long start = preMeasure(measureFill);
+        super.fill(result, summaryClass, execution);
+        postMeasure(measureFill, start);
+    }
+
+    @Override
+    public Pong ping(Ping ping, Execution execution) {
+        long start = preMeasure(measurePing);
+        Pong pong = execution.ping(ping);
+        postMeasure(measurePing, start);
+        return pong;
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        long start = preMeasure(measureSearch);
+        Result result = execution.search(query);
+        postMeasure(measureSearch, start);
+        return result;
+    }
+
+    /**
+     * This method is only included for testing.
+     */
+    public void setMeasurements(Value measurements) {
+        this.measurements = measurements;
+    }
+
+    @Override
+    public void deconstruct() {
+        // avoid dangling, duplicate loggers
+        measurements.cancel();
+        super.deconstruct();
+    }
+
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/statistics/package-info.java b/container-search/src/main/java/com/yahoo/search/statistics/package-info.java
new file mode 100644
index 00000000000..04626fa913e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/statistics/package-info.java
@@ -0,0 +1,7 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+@PublicApi
+package com.yahoo.search.statistics;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/template/.gitignore b/container-search/src/main/java/com/yahoo/search/template/.gitignore
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/template/.gitignore
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ArgumentsTypeChecker.java b/container-search/src/main/java/com/yahoo/search/yql/ArgumentsTypeChecker.java
new file mode 100644
index 00000000000..c297bf80cac
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/ArgumentsTypeChecker.java
@@ -0,0 +1,30 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Preconditions;
+
+import java.util.List;
+
+final class ArgumentsTypeChecker {
+
+    private final Operator target;
+    private final List<OperatorTypeChecker> checkers;
+
+    public ArgumentsTypeChecker(Operator target, List<OperatorTypeChecker> checkers) {
+        this.target = target;
+        this.checkers = checkers;
+    }
+
+    public void check(Object... args) {
+        if (args == null) {
+            Preconditions.checkArgument(checkers.size() == 0, "Operator %s argument count mismatch: expected %s got 0", target, checkers.size());
+            return;
+        } else {
+            Preconditions.checkArgument(args.length == checkers.size(), "Operator %s argument count mismatch: expected: %s got %s", target, checkers.size(), args.length);
+        }
+        for (int i = 0; i < checkers.size(); ++i) {
+            checkers.get(i).check(args[i]);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java
new file mode 100644
index 00000000000..33e684357af
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveFileStream.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import org.antlr.v4.runtime.ANTLRFileStream;
+import org.antlr.v4.runtime.CharStream;
+
+import java.io.IOException;
+
+/**
+ * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token.
+ */
+
+class CaseInsensitiveFileStream extends ANTLRFileStream {
+
+    public CaseInsensitiveFileStream(String fileName) throws IOException {
+        super(fileName);
+    }
+
+    public CaseInsensitiveFileStream(String fileName, String encoding) throws IOException {
+        super(fileName, encoding);
+    }
+
+    @Override
+    public int LA(int i) {
+        if (i == 0) {
+            return 0;
+        }
+        if (i < 0) {
+            i++; // e.g., translate LA(-1) to use offset 0
+        }
+
+        if ((p + i - 1) >= n) {
+            return CharStream.EOF;
+        }
+        return Character.toLowerCase(data[p + i - 1]);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java
new file mode 100644
index 00000000000..e15fe04bb39
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/CaseInsensitiveInputStream.java
@@ -0,0 +1,50 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.CharStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Enable ANTLR to do case insensitive comparisons when reading from files without throwing away the case in the token.
+ */
+class CaseInsensitiveInputStream extends ANTLRInputStream {
+
+    public CaseInsensitiveInputStream() {
+        super();
+    }
+
+    public CaseInsensitiveInputStream(InputStream input) throws IOException {
+        super(input);
+    }
+
+    public CaseInsensitiveInputStream(InputStream input, int size) throws IOException {
+        super(input, size);
+    }
+    
+    public CaseInsensitiveInputStream(char[] data, int numberOfActualCharsInArray) throws IOException {
+        super(data, numberOfActualCharsInArray);
+    }
+    
+    public CaseInsensitiveInputStream(String input) throws IOException {
+        super(input);
+    }
+
+    @Override
+    public int LA(int i) {
+        if (i == 0) {
+            return 0;
+        }
+        if (i < 0) {
+            i++; // e.g., translate LA(-1) to use offset 0
+        }
+
+        if ((p + i - 1) >= n) {
+            return CharStream.EOF;
+        }
+        return Character.toLowerCase(data[p + i - 1]);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ExpressionOperator.java b/container-search/src/main/java/com/yahoo/search/yql/ExpressionOperator.java
new file mode 100644
index 00000000000..e9fe52d33e7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/ExpressionOperator.java
@@ -0,0 +1,84 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Predicate;
+
+/**
+ * Operators on expressions.
+ */
+enum ExpressionOperator implements Operator {
+
+    AND(TypeCheckers.EXPRS),
+    OR(TypeCheckers.EXPRS),
+    EQ(ExpressionOperator.class, ExpressionOperator.class),
+    NEQ(ExpressionOperator.class, ExpressionOperator.class),
+    LT(ExpressionOperator.class, ExpressionOperator.class),
+    GT(ExpressionOperator.class, ExpressionOperator.class),
+    LTEQ(ExpressionOperator.class, ExpressionOperator.class),
+    GTEQ(ExpressionOperator.class, ExpressionOperator.class),
+
+    IN(ExpressionOperator.class, ExpressionOperator.class),
+    IN_QUERY(ExpressionOperator.class, SequenceOperator.class),
+    NOT_IN(ExpressionOperator.class, ExpressionOperator.class),
+    NOT_IN_QUERY(ExpressionOperator.class, SequenceOperator.class),
+
+    LIKE(ExpressionOperator.class, ExpressionOperator.class),
+    NOT_LIKE(ExpressionOperator.class, ExpressionOperator.class),
+
+    IS_NULL(ExpressionOperator.class),
+    IS_NOT_NULL(ExpressionOperator.class),
+    MATCHES(ExpressionOperator.class, ExpressionOperator.class),
+    NOT_MATCHES(ExpressionOperator.class, ExpressionOperator.class),
+    CONTAINS(ExpressionOperator.class, ExpressionOperator.class),
+
+    ADD(ExpressionOperator.class, ExpressionOperator.class),
+    SUB(ExpressionOperator.class, ExpressionOperator.class),
+    MULT(ExpressionOperator.class, ExpressionOperator.class),
+    DIV(ExpressionOperator.class, ExpressionOperator.class),
+    MOD(ExpressionOperator.class, ExpressionOperator.class),
+
+    NEGATE(ExpressionOperator.class),
+    NOT(ExpressionOperator.class),
+
+    MAP(TypeCheckers.LIST_OF_STRING, TypeCheckers.EXPRS),
+
+    ARRAY(TypeCheckers.EXPRS),
+
+    INDEX(ExpressionOperator.class, ExpressionOperator.class),
+    PROPREF(ExpressionOperator.class, String.class),
+
+    CALL(TypeCheckers.LIST_OF_STRING, TypeCheckers.EXPRS),
+
+    VARREF(String.class),
+
+    LITERAL(TypeCheckers.LITERAL_TYPES),
+
+    READ_RECORD(String.class),
+    READ_FIELD(String.class, String.class),
+    READ_MODULE(TypeCheckers.LIST_OF_STRING),
+
+    VESPA_GROUPING(String.class),
+
+    NULL();
+
+    private final ArgumentsTypeChecker checker;
+
+
+    private ExpressionOperator(Object... types) {
+        checker = TypeCheckers.make(this, types);
+    }
+
+
+    @Override
+    public void checkArguments(Object... args) {
+        checker.check(args);
+    }
+
+    public static Predicate<OperatorNode<? extends Operator>> IS = new Predicate<OperatorNode<? extends Operator>>() {
+        @Override
+        public boolean apply(OperatorNode<? extends Operator> input) {
+            return input.getOperator() instanceof ExpressionOperator;
+        }
+    };
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java
new file mode 100644
index 00000000000..f6e8ee1f27a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java
@@ -0,0 +1,156 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass;
+import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.query.Presentation;
+import com.yahoo.search.searchchain.Execution;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * Ensure the fields specified in {@link Presentation#getSummaryFields()} are
+ * available after filling phase.
+ *
+ * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a>
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@Beta
+@After(MinimalQueryInserter.EXTERNAL_YQL)
+public class FieldFiller extends Searcher {
+
+    private final Set<String> intersectionOfAttributes;
+    private final SummaryIntersections summaryDb = new SummaryIntersections();
+    public static final CompoundName FIELD_FILLER_DISABLE = new CompoundName(
+            "FieldFiller.disable");
+
+    private static class SummaryIntersections {
+        private final Map<String, Map<String, Set<String>>> db = new HashMap<>();
+
+        void add(String dbName, Summaryclass summary) {
+            Map<String, Set<String>> docType = getOrCreateDocType(dbName);
+            Set<String> fields = new HashSet<>(summary.fields().size());
+            for (Fields f : summary.fields()) {
+                fields.add(f.name());
+            }
+            docType.put(summary.name(), fields);
+        }
+
+        @NonNull
+        private Map<String, Set<String>> getOrCreateDocType(String dbName) {
+            Map<String, Set<String>> docType = db.get(dbName);
+            if (docType == null) {
+                docType = new HashMap<>();
+                db.put(dbName, docType);
+            }
+            return docType;
+        }
+
+        boolean hasAll(Set<String> requested, String summaryName, Set<String> restrict) {
+            Set<String> explicitRestriction;
+            Set<String> intersection = null;
+
+            if (restrict.isEmpty()) {
+                explicitRestriction = db.keySet();
+            } else {
+                explicitRestriction = restrict;
+            }
+
+            for (String docType : explicitRestriction) {
+                Map<String, Set<String>> summaries = db.get(docType);
+                Set<String> summary;
+
+                if (summaries == null) {
+                    continue;
+                }
+                summary = summaries.get(summaryName);
+                if (summary == null) {
+                    intersection = null;
+                    break;
+                }
+                if (intersection == null) {
+                    intersection = new HashSet<>(summary.size());
+                    intersection.addAll(summary);
+                } else {
+                    intersection.retainAll(summary);
+                }
+            }
+            return intersection == null ? false : intersection
+                    .containsAll(requested);
+        }
+    }
+
+    public FieldFiller(DocumentdbInfoConfig config) {
+        intersectionOfAttributes = new HashSet<>();
+        boolean first = true;
+
+        for (Documentdb db : config.documentdb()) {
+            for (Summaryclass summary : db.summaryclass()) {
+                Set<String> attributes = null;
+                if (Execution.ATTRIBUTEPREFETCH.equals(summary.name())) {
+                    attributes = new HashSet<>(summary.fields().size());
+                    for (Fields f : summary.fields()) {
+                        attributes.add(f.name());
+                    }
+                    if (first) {
+                        first = false;
+                        intersectionOfAttributes.addAll(attributes);
+                    } else {
+                        intersectionOfAttributes.retainAll(attributes);
+                    }
+                }
+                // yes, we store attribute prefetch here as well, this is in
+                // case we get a query where we have a restrict parameter which
+                // makes filling with attribute prefetch possible even though it
+                // wouldn't have been possible without restricting the set of
+                // doctypes
+                summaryDb.add(db.name(), summary);
+            }
+        }
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        return execution.search(query);
+    }
+
+    @Override
+    public void fill(Result result, String summaryClass, Execution execution) {
+        execution.fill(result, summaryClass);
+
+        final Set<String> summaryFields = result.getQuery().getPresentation()
+                .getSummaryFields();
+
+        if (summaryFields.isEmpty()
+                || summaryClass == null
+                || result.getQuery().properties()
+                        .getBoolean(FIELD_FILLER_DISABLE)) {
+            return;
+        }
+
+        if (intersectionOfAttributes.containsAll(summaryFields)) {
+            if (!Execution.ATTRIBUTEPREFETCH.equals(summaryClass)) {
+                execution.fill(result, Execution.ATTRIBUTEPREFETCH);
+            }
+        } else {
+            // Yes, summaryClass may be Execution.ATTRIBUTEPREFETCH here
+            if (!summaryDb.hasAll(summaryFields, summaryClass, result
+                    .getQuery().getModel().getRestrict())) {
+                execution.fill(result, null);
+            }
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java
new file mode 100644
index 00000000000..b44fdadd17b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.component.chain.dependencies.Before;
+import com.yahoo.prelude.fastsearch.FastHit;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+
+/**
+ * Remove fields which are not explicitly requested, if any field is explicitly
+ * requested. Disable using FieldFilter.disable=true in request.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@Beta
+@After(MinimalQueryInserter.EXTERNAL_YQL)
+@Before("com.yahoo.search.yql.FieldFiller")
+public class FieldFilter extends Searcher {
+
+    public static final CompoundName FIELD_FILTER_DISABLE = new CompoundName("FieldFilter.disable");
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        Result result = execution.search(query);
+        filter(result);
+        return result;
+    }
+
+    @Override
+    public void fill(Result result, String summaryClass, Execution execution) {
+        execution.fill(result, summaryClass);
+        filter(result);
+    }
+
+    private void filter(Result result) {
+        Set<String> requestedFields;
+
+        if (result.getQuery().properties().getBoolean(FIELD_FILTER_DISABLE)) return;
+        if (result.getQuery().getPresentation().getSummaryFields().isEmpty()) return;
+
+        requestedFields = result.getQuery().getPresentation().getSummaryFields();
+        for (Iterator<Hit> i = result.hits().unorderedDeepIterator(); i.hasNext();) {
+            Hit h = i.next();
+            if (h.isMeta()) continue;
+            for (Iterator<Entry<String, Object>> fields = h.fieldIterator(); fields.hasNext();) {
+                Entry<String, Object> field = fields.next();
+                if ( ! requestedFields.contains(field.getKey()))
+                    fields.remove();
+            }
+
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/JavaListTypeChecker.java b/container-search/src/main/java/com/yahoo/search/yql/JavaListTypeChecker.java
new file mode 100644
index 00000000000..86e2cbf01ff
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/JavaListTypeChecker.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Preconditions;
+
+import java.util.List;
+
+class JavaListTypeChecker extends OperatorTypeChecker {
+
+    private final Class<?> elementType;
+
+    public JavaListTypeChecker(Operator parent, int idx, Class<?> elementType) {
+        super(parent, idx);
+        this.elementType = elementType;
+    }
+
+    @Override
+    public void check(Object argument) {
+        Preconditions.checkNotNull(argument, "Argument %s of %s must not be null", idx, parent);
+        Preconditions.checkArgument(argument instanceof List, "Argument %s of %s must be a List<%s>", idx, parent, elementType.getName(), argument.getClass().getName());
+        List<?> lst = (List<?>) argument;
+        for (Object elt : lst) {
+            Preconditions.checkNotNull(elt, "Argument %s of %s List elements may not be null", idx, parent);
+            Preconditions.checkArgument(elementType.isInstance(elt), "Argument %s of %s List elements must be %s (is %s)", idx, parent, elementType.getName(), elt.getClass().getName());
+        }
+    }
+
+}
+
diff --git a/container-search/src/main/java/com/yahoo/search/yql/JavaTypeChecker.java b/container-search/src/main/java/com/yahoo/search/yql/JavaTypeChecker.java
new file mode 100644
index 00000000000..bf91474c19b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/JavaTypeChecker.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Preconditions;
+
+class JavaTypeChecker extends OperatorTypeChecker {
+
+    private final Class<?> type;
+
+    public JavaTypeChecker(Operator parent, int idx, Class<?> type) {
+        super(parent, idx);
+        this.type = type;
+    }
+
+    @Override
+    public void check(Object argument) {
+        Preconditions.checkNotNull(argument, "Argument %s of %s must not be null", idx, parent);
+        Preconditions.checkArgument(type.isInstance(argument), "Argument %s of %s must be %s (is: %s).", idx, parent, type.getName(), argument.getClass().getName());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/JavaUnionTypeChecker.java b/container-search/src/main/java/com/yahoo/search/yql/JavaUnionTypeChecker.java
new file mode 100644
index 00000000000..a94027a9bd2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/JavaUnionTypeChecker.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Set;
+
+public class JavaUnionTypeChecker extends OperatorTypeChecker {
+
+    private final Set<Class<?>> types;
+
+    public JavaUnionTypeChecker(Operator parent, int idx, Set<Class<?>> types) {
+        super(parent, idx);
+        this.types = types;
+    }
+
+    public JavaUnionTypeChecker(Operator parent, int idx, Class<?>... types) {
+        super(parent, idx);
+        this.types = ImmutableSet.copyOf(types);
+    }
+
+    @Override
+    public void check(Object argument) {
+        Preconditions.checkNotNull(argument, "Argument %s of %s must not be null", idx, parent);
+        for (Class<?> candidate : types) {
+            if (candidate.isInstance(argument)) {
+                return;
+            }
+        }
+        Preconditions.checkArgument(false, "Argument %s of %s must be %s (is: %s).", idx, parent, Joiner.on("|").join(types), argument.getClass());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/Location.java b/container-search/src/main/java/com/yahoo/search/yql/Location.java
new file mode 100644
index 00000000000..a304ed75536
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/Location.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+/**
+ * A pointer to a location in a YQL source program.
+ */
+final class Location {
+
+    private final String programName;
+    private final int lineNumber;
+    private final int characterOffset;
+
+    public Location(String programName, int lineNumber, int characterOffset) {
+        this.programName = programName;
+        this.lineNumber = lineNumber;
+        this.characterOffset = characterOffset;
+    }
+
+
+    public int getLineNumber() {
+        return lineNumber;
+    }
+
+    public int getCharacterOffset() {
+        return characterOffset;
+    }
+
+    @Override
+    public String toString() {
+        if (programName != null) {
+            return programName + ":L" + lineNumber + ":" + characterOffset;
+        } else {
+            return "L" + lineNumber + ":" + characterOffset;
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
new file mode 100644
index 00000000000..d710754e887
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
@@ -0,0 +1,98 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.search.query.parser.Parsable;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.searchchain.PhaseNames;
+import com.yahoo.yolean.chain.After;
+import com.yahoo.yolean.chain.Before;
+import com.yahoo.yolean.chain.Provides;
+
+/**
+ * Minimal combinator for YQL+ syntax and heuristically parsed user queries.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @since 5.1.28
+ */
+@Beta
+@Provides(MinimalQueryInserter.EXTERNAL_YQL)
+@Before(PhaseNames.TRANSFORMED_QUERY)
+@After("com.yahoo.prelude.statistics.StatisticsSearcher")
+public class MinimalQueryInserter extends Searcher {
+    public static final String EXTERNAL_YQL = "ExternalYql";
+
+    public static final CompoundName YQL = new CompoundName("yql");
+
+    private static final CompoundName MAX_HITS = new CompoundName("maxHits");
+    private static final CompoundName MAX_OFFSET = new CompoundName("maxOffset");
+
+    public MinimalQueryInserter() {
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        if (query.properties().get(YQL) == null) {
+            return execution.search(query);
+        }
+        ParserEnvironment env = ParserEnvironment.fromExecutionContext(execution.context());
+        YqlParser parser = (YqlParser) ParserFactory.newInstance(Query.Type.YQL, env);
+        parser.setQueryParser(false);
+        parser.setUserQuery(query);
+        QueryTree newTree;
+        try {
+            newTree = parser.parse(Parsable.fromQueryModel(query.getModel())
+                    .setQuery(query.properties().getString(YQL)));
+        } catch (RuntimeException e) {
+            return new Result(query, ErrorMessage.createInvalidQueryParameter(
+                    "Could not instantiate query from YQL+", e));
+        }
+        if (parser.getOffset() != null) {
+            final int maxHits = query.properties().getInteger(MAX_HITS);
+            final int maxOffset = query.properties().getInteger(MAX_OFFSET);
+            if (parser.getOffset() > maxOffset) {
+                return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested offset " + parser.getOffset()
+                        + ", but the max offset allowed is " + maxOffset + "."));
+            }
+            if (parser.getHits() > maxHits) {
+                return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested " + parser.getHits()
+                        + " hits returned, but max hits allowed is " + maxHits + "."));
+
+            }
+        }
+        query.getModel().getQueryTree().setRoot(newTree.getRoot());
+        query.getPresentation().getSummaryFields().addAll(parser.getYqlSummaryFields());
+        for (VespaGroupingStep step : parser.getGroupingSteps()) {
+            GroupingRequest.newInstance(query)
+                           .setRootOperation(step.getOperation())
+                           .continuations().addAll(step.continuations());
+        }
+        if (parser.getYqlSources().size() == 0) {
+            query.getModel().getSources().clear();
+        } else {
+            query.getModel().getSources().addAll(parser.getYqlSources());
+        }
+        if (parser.getOffset() != null) {
+            query.setOffset(parser.getOffset());
+            query.setHits(parser.getHits());
+        }
+        if (parser.getTimeout() != null) {
+            query.setTimeout(parser.getTimeout().longValue());
+        }
+        if (parser.getSorting() != null) {
+            query.getRanking().setSorting(parser.getSorting());
+        }
+        query.trace("YQL+ query parsed", true, 2);
+        return execution.search(query);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/NodeTypeChecker.java b/container-search/src/main/java/com/yahoo/search/yql/NodeTypeChecker.java
new file mode 100644
index 00000000000..c407689e107
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/NodeTypeChecker.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import java.util.Set;
+
+/**
+ * Check that an argument is an OperatorNode of a particular operator set.
+ */
+class NodeTypeChecker extends OperatorTypeChecker {
+
+    private final Class<? extends Operator> operatorType;
+    private final Set<? extends Operator> operators;
+
+    public NodeTypeChecker(Operator parent, int idx, Class<? extends Operator> operatorType, Set<? extends Operator> operators) {
+        super(parent, idx);
+        this.operatorType = operatorType;
+        this.operators = operators;
+    }
+
+    @Override
+    public void check(Object argument) {
+        Preconditions.checkNotNull(argument, "Argument %s of %s must not be null", idx, parent);
+        Preconditions.checkArgument(argument instanceof OperatorNode, "Argument %s of %s must be an OperatorNode<%s> (is %s).", idx, parent, operatorType.getName(), argument.getClass());
+        OperatorNode<?> node = (OperatorNode<?>) argument;
+        Operator op = node.getOperator();
+        Preconditions.checkArgument(operatorType.isInstance(op), "Argument %s of %s must be an OperatorNode<%s> (is: %s).", idx, parent, operatorType.getName(), op.getClass());
+        if (!operators.isEmpty()) {
+            Preconditions.checkArgument(operators.contains(op), "Argument %s of %s must be %s (is %s).", idx, parent, Joiner.on("|").join(operators), op);
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/NullItemException.java b/container-search/src/main/java/com/yahoo/search/yql/NullItemException.java
new file mode 100644
index 00000000000..c50f22ff711
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/NullItemException.java
@@ -0,0 +1,14 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+/**
+ * Used to communicate a NullItem has been encountered in the query tree.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+@SuppressWarnings("serial")
+public class NullItemException extends RuntimeException {
+    public NullItemException(String message) {
+        super(message);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/Operator.java b/container-search/src/main/java/com/yahoo/search/yql/Operator.java
new file mode 100644
index 00000000000..f5c0f9fb56d
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/Operator.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+interface Operator {
+
+    String name();
+
+    void checkArguments(Object... args);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java b/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java
new file mode 100644
index 00000000000..d1b65ee258b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/OperatorNode.java
@@ -0,0 +1,261 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import javax.annotation.Nullable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents a use of an operator against concrete arguments. The types of arguments depend on the operator.
+ * <p>
+ * The extension point of this scheme is the Operator rather than new types of Nodes.
+ * <p>
+ * Operators SHOULD take a fixed number of arguments -- wrap variable argument counts in Lists.
+ */
+final class OperatorNode<T extends Operator> {
+
+    public static <T extends Operator> OperatorNode<T> create(T operator, Object... args) {
+        operator.checkArguments(args == null ? EMPTY_ARGS : args);
+        return new OperatorNode<T>(operator, args);
+    }
+
+    public static <T extends Operator> OperatorNode<T> create(Location loc, T operator, Object... args) {
+        operator.checkArguments(args == null ? EMPTY_ARGS : args);
+        return new OperatorNode<T>(loc, operator, args);
+    }
+
+    public static <T extends Operator> OperatorNode<T> create(Location loc, Map<String, Object> annotations, T operator, Object... args) {
+        operator.checkArguments(args == null ? EMPTY_ARGS : args);
+        return new OperatorNode<T>(loc, annotations, operator, args);
+    }
+
+    private static final Object[] EMPTY_ARGS = new Object[0];
+
+    private final Location location;
+    private final T operator;
+    private Map<String, Object> annotations = ImmutableMap.of();
+    private final Object[] args;
+
+    private OperatorNode(T operator, Object... args) {
+        this.location = null;
+        this.operator = operator;
+        if (args == null) {
+            this.args = EMPTY_ARGS;
+        } else {
+            this.args = args;
+        }
+    }
+
+    private OperatorNode(Location loc, T operator, Object... args) {
+        this.location = loc;
+        this.operator = operator;
+        if (args == null) {
+            this.args = EMPTY_ARGS;
+        } else {
+            this.args = args;
+        }
+    }
+
+    private OperatorNode(Location loc, Map<String, Object> annotations, T operator, Object... args) {
+        this.location = loc;
+        this.operator = operator;
+        this.annotations = ImmutableMap.copyOf(annotations);
+        if (args == null) {
+            this.args = EMPTY_ARGS;
+        } else {
+            this.args = args;
+        }
+    }
+
+    public T getOperator() {
+        return operator;
+    }
+
+    public Object[] getArguments() {
+        // this is only called by a test right now, but ImmutableList.copyOf won't tolerate null elements
+        if (args.length == 0) {
+            return args;
+        }
+        Object[] copy = new Object[args.length];
+        System.arraycopy(args, 0, copy, 0, args.length);
+        return copy;
+    }
+
+    public <T> T getArgument(int i) {
+        return (T) args[i];
+    }
+
+    public <T> T getArgument(int i, Class<T> clazz) {
+        return clazz.cast(getArgument(i));
+    }
+
+    public Location getLocation() {
+        return location;
+    }
+
+    public Object getAnnotation(String name) {
+        return annotations.get(name);
+    }
+
+    public OperatorNode<T> putAnnotation(String name, Object value) {
+        if (annotations.isEmpty()) {
+            annotations = Maps.newLinkedHashMap();
+        } else if (annotations instanceof ImmutableMap) {
+            annotations = Maps.newLinkedHashMap(annotations);
+        }
+        annotations.put(name, value);
+        return this;
+    }
+
+    public Map<String, Object> getAnnotations() {
+        // TODO: this should be a read-only view?
+        return ImmutableMap.copyOf(annotations);
+    }
+
+    public OperatorNode<T> transform(Function<Object, Object> argumentTransform) {
+        if (args.length == 0) {
+            // nothing to transform, so no change is possible
+            return this;
+        }
+        Object[] newArgs = new Object[args.length];
+        boolean changed = false;
+        for (int i = 0; i < args.length; ++i) {
+            Object target = args[i];
+            if (target instanceof List) {
+                List<Object> newList = Lists.newArrayListWithExpectedSize(((List) target).size());
+                for (Object val : (List) target) {
+                    newList.add(argumentTransform.apply(val));
+                }
+                newArgs[i] = newList;
+                // this will always 'change' the tree, maybe fix later
+            } else {
+                newArgs[i] = argumentTransform.apply(args[i]);
+            }
+            changed = changed || newArgs[i] != args[i];
+        }
+        if (changed) {
+            return new OperatorNode<>(location, annotations, operator, newArgs);
+        }
+        return this;
+    }
+
+    public void visit(OperatorVisitor visitor) {
+        if (visitor.enter(this)) {
+            for (Object target : args) {
+                if (target instanceof List) {
+                    for (Object val : (List) target) {
+                        if (val instanceof OperatorNode) {
+                            ((OperatorNode) val).visit(visitor);
+                        }
+                    }
+                } else if (target instanceof OperatorNode) {
+                    ((OperatorNode) target).visit(visitor);
+
+                }
+            }
+        }
+        visitor.exit(this);
+    }
+
+    // we are aware only of types used in our logical operator trees -- OperatorNode, List, and constant values
+    private static final Function<Object, Object> COPY = new Function<Object, Object>() {
+        @Nullable
+        @Override
+        public Object apply(@Nullable Object input) {
+            if (input instanceof List) {
+                List<Object> newList = Lists.newArrayListWithExpectedSize(((List) input).size());
+                for (Object val : (List) input) {
+                    newList.add(COPY.apply(val));
+                }
+                return newList;
+            } else if (input instanceof OperatorNode) {
+                return ((OperatorNode) input).copy();
+            } else if (input instanceof String || input instanceof Number || input instanceof Boolean) {
+                return input;
+            } else {
+                // this may be annoying but COPY not understanding how to COPY and quietly reusing
+                // when it may not be immutable could be dangerous
+                throw new IllegalArgumentException("Unexpected value type in OperatorNode tree: " + input);
+            }
+        }
+    };
+
+    public OperatorNode<T> copy() {
+        Object[] newArgs = new Object[args.length];
+        for (int i = 0; i < args.length; ++i) {
+            newArgs[i] = COPY.apply(args[i]);
+        }
+        return new OperatorNode<>(location, ImmutableMap.copyOf(annotations), operator, newArgs);
+    }
+
+    public void toString(StringBuilder output) {
+        output.append("(")
+              .append(operator.name());
+        if(location != null) {
+            output.append(" L")
+                  .append(location.getCharacterOffset())
+                  .append(":")
+                  .append(location.getLineNumber());
+        }
+        if(annotations != null && !annotations.isEmpty()) {
+            output.append(" {");
+            Joiner.on(", ").withKeyValueSeparator("=")
+                    .appendTo(output, annotations);
+            output.append("}");
+        }
+        boolean first = true;
+        for(Object arg : args) {
+            if(!first) {
+                output.append(",");
+            }
+            first = false;
+            output.append(" ");
+            if(arg instanceof OperatorNode) {
+                ((OperatorNode) arg).toString(output);
+            } else if(arg instanceof Iterable) {
+                output.append("[");
+                Joiner.on(", ").appendTo(output, (Iterable)arg);
+                output.append("]");
+            } else {
+                output.append(arg.toString());
+            }
+        }
+        output.append(")");
+    }
+
+    public String toString() {
+        StringBuilder output = new StringBuilder();
+        toString(output);
+        return output.toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        OperatorNode that = (OperatorNode) o;
+
+        if (!annotations.equals(that.annotations)) return false;
+        // Probably incorrect - comparing Object[] arrays with Arrays.equals
+        if (!Arrays.equals(args, that.args)) return false;
+        if (!operator.equals(that.operator)) return false;
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = operator.hashCode();
+        result = 31 * result + annotations.hashCode();
+        result = 31 * result + Arrays.hashCode(args);
+        return result;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/OperatorNodeListTypeChecker.java b/container-search/src/main/java/com/yahoo/search/yql/OperatorNodeListTypeChecker.java
new file mode 100644
index 00000000000..d0c98fb3d11
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/OperatorNodeListTypeChecker.java
@@ -0,0 +1,35 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+
+import java.util.List;
+import java.util.Set;
+
+class OperatorNodeListTypeChecker extends OperatorTypeChecker {
+
+    private final Class<? extends Operator> operatorType;
+    private final Set<? extends Operator> operators;
+
+    public OperatorNodeListTypeChecker(Operator parent, int idx, Class<? extends Operator> operatorType, Set<? extends Operator> operators) {
+        super(parent, idx);
+        this.operatorType = operatorType;
+        this.operators = operators;
+    }
+
+    @Override
+    public void check(Object argument) {
+        Preconditions.checkNotNull(argument, "Argument %s of %s must not be null", idx, parent);
+        Preconditions.checkArgument(argument instanceof List, "Argument %s of %s must be a List<OperatorNode<%s>>", idx, parent, operatorType.getName(), argument.getClass());
+        List<OperatorNode<?>> lst = (List<OperatorNode<?>>) argument;
+        for (OperatorNode<?> node : lst) {
+            Operator op = node.getOperator();
+            Preconditions.checkArgument(operatorType.isInstance(op), "Argument %s of %s must contain only OperatorNode<%s> (is: %s).", idx, parent, operatorType.getName(), op.getClass());
+            if (!operators.isEmpty()) {
+                Preconditions.checkArgument(operators.contains(op), "Argument %s of %s must contain only %s (is %s).", idx, parent, Joiner.on("|").join(operators), op);
+            }
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/OperatorTypeChecker.java b/container-search/src/main/java/com/yahoo/search/yql/OperatorTypeChecker.java
new file mode 100644
index 00000000000..8266f414fa7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/OperatorTypeChecker.java
@@ -0,0 +1,19 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+/**
+ * Check the type of a single argument.
+ */
+abstract class OperatorTypeChecker {
+
+    protected final Operator parent;
+    protected final int idx;
+
+    protected OperatorTypeChecker(Operator parent, int idx) {
+        this.parent = parent;
+        this.idx = idx;
+    }
+
+    public abstract void check(Object argument);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/OperatorVisitor.java b/container-search/src/main/java/com/yahoo/search/yql/OperatorVisitor.java
new file mode 100644
index 00000000000..73c3612c1c9
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/OperatorVisitor.java
@@ -0,0 +1,10 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+interface OperatorVisitor {
+
+    <T extends Operator> boolean enter(OperatorNode<T> node);
+
+    <T extends Operator> void exit(OperatorNode<T> node);
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ParserBase.java b/container-search/src/main/java/com/yahoo/search/yql/ParserBase.java
new file mode 100644
index 00000000000..af3418919e8
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/ParserBase.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.collect.Sets;
+
+import org.antlr.v4.runtime.Parser;
+import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.tree.ParseTree;
+
+import java.util.Set;
+
+/**
+ * Provides semantic helper functions to Parser.
+ */
+abstract class ParserBase extends Parser {
+
+    private static String arrayRuleName = "array";
+    public ParserBase(TokenStream input) {
+        super(input);
+    }
+    
+    private Set<String> arrayParameters = Sets.newHashSet();
+
+    public void registerParameter(String name, String typeName) {
+        if (typeName.equals(arrayRuleName)) {
+            arrayParameters.add(name);
+       }
+    }
+
+    public boolean isArrayParameter(ParseTree nameNode) {
+        String name = nameNode.getText();
+        if (name.startsWith("@")) {
+            name = name.substring(1);
+        }
+        return name != null && arrayParameters.contains(name);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java
new file mode 100644
index 00000000000..592bd690d56
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java
@@ -0,0 +1,38 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+class ProgramCompileException extends RuntimeException {
+
+    private Location sourceLocation;
+
+    public ProgramCompileException(String message) {
+        super(message);
+    }
+
+    public ProgramCompileException(String message, Object... args) {
+        super(formatMessage(message, args));
+    }
+
+    private static String formatMessage(String message, Object... args) {
+        return args == null ? message : String.format(message, args);
+    }
+
+    public ProgramCompileException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ProgramCompileException(Throwable cause) {
+        super(cause);
+    }
+
+    public ProgramCompileException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+
+    public ProgramCompileException(Location sourceLocation, String message, Object... args) {
+        super(String.format("%s %s", sourceLocation != null ? sourceLocation : "", args == null ? message : String.format(message, args)));
+        this.sourceLocation = sourceLocation;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
new file mode 100644
index 00000000000..a8d1bc43a4c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
@@ -0,0 +1,1549 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.yahoo.search.yql.yqlplusParser.AnnotationContext;
+import com.yahoo.search.yql.yqlplusParser.AnnotateExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.ArgumentContext;
+import com.yahoo.search.yql.yqlplusParser.ArgumentsContext;
+import com.yahoo.search.yql.yqlplusParser.ArrayLiteralContext;
+import com.yahoo.search.yql.yqlplusParser.ArrayTypeContext;
+import com.yahoo.search.yql.yqlplusParser.Call_sourceContext;
+import com.yahoo.search.yql.yqlplusParser.ConstantArrayContext;
+import com.yahoo.search.yql.yqlplusParser.ConstantExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.ConstantMapExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.ConstantPropertyNameAndValueContext;
+import com.yahoo.search.yql.yqlplusParser.Delete_statementContext;
+import com.yahoo.search.yql.yqlplusParser.DereferencedExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.EqualityExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.ExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.FallbackContext;
+import com.yahoo.search.yql.yqlplusParser.Field_defContext;
+import com.yahoo.search.yql.yqlplusParser.Field_names_specContext;
+import com.yahoo.search.yql.yqlplusParser.Field_values_group_specContext;
+import com.yahoo.search.yql.yqlplusParser.Field_values_specContext;
+import com.yahoo.search.yql.yqlplusParser.IdentContext;
+import com.yahoo.search.yql.yqlplusParser.Import_listContext;
+import com.yahoo.search.yql.yqlplusParser.Import_statementContext;
+import com.yahoo.search.yql.yqlplusParser.InNotInTargetContext;
+import com.yahoo.search.yql.yqlplusParser.Insert_sourceContext;
+import com.yahoo.search.yql.yqlplusParser.Insert_statementContext;
+import com.yahoo.search.yql.yqlplusParser.Insert_valuesContext;
+import com.yahoo.search.yql.yqlplusParser.JoinExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.Join_exprContext;
+import com.yahoo.search.yql.yqlplusParser.LimitContext;
+import com.yahoo.search.yql.yqlplusParser.Literal_elementContext;
+import com.yahoo.search.yql.yqlplusParser.Literal_listContext;
+import com.yahoo.search.yql.yqlplusParser.LogicalANDExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.LogicalORExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.MapExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.MapTypeContext;
+import com.yahoo.search.yql.yqlplusParser.Merge_componentContext;
+import com.yahoo.search.yql.yqlplusParser.Merge_statementContext;
+import com.yahoo.search.yql.yqlplusParser.ModuleIdContext;
+import com.yahoo.search.yql.yqlplusParser.ModuleNameContext;
+import com.yahoo.search.yql.yqlplusParser.MultiplicativeExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.Namespaced_nameContext;
+import com.yahoo.search.yql.yqlplusParser.Next_statementContext;
+import com.yahoo.search.yql.yqlplusParser.OffsetContext;
+import com.yahoo.search.yql.yqlplusParser.OrderbyContext;
+import com.yahoo.search.yql.yqlplusParser.Orderby_fieldContext;
+import com.yahoo.search.yql.yqlplusParser.Output_specContext;
+import com.yahoo.search.yql.yqlplusParser.Paged_clauseContext;
+import com.yahoo.search.yql.yqlplusParser.ParamsContext;
+import com.yahoo.search.yql.yqlplusParser.Pipeline_stepContext;
+import com.yahoo.search.yql.yqlplusParser.Procedure_argumentContext;
+import com.yahoo.search.yql.yqlplusParser.Program_arglistContext;
+import com.yahoo.search.yql.yqlplusParser.Project_specContext;
+import com.yahoo.search.yql.yqlplusParser.ProgramContext;
+import com.yahoo.search.yql.yqlplusParser.PropertyNameAndValueContext;
+import com.yahoo.search.yql.yqlplusParser.Query_statementContext;
+import com.yahoo.search.yql.yqlplusParser.RelationalExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.RelationalOpContext;
+import com.yahoo.search.yql.yqlplusParser.Returning_specContext;
+import com.yahoo.search.yql.yqlplusParser.Scalar_literalContext;
+import com.yahoo.search.yql.yqlplusParser.Select_source_joinContext;
+import com.yahoo.search.yql.yqlplusParser.Select_source_multiContext;
+import com.yahoo.search.yql.yqlplusParser.Select_statementContext;
+import com.yahoo.search.yql.yqlplusParser.Selectvar_statementContext;
+import com.yahoo.search.yql.yqlplusParser.Sequence_sourceContext;
+import com.yahoo.search.yql.yqlplusParser.Source_listContext;
+import com.yahoo.search.yql.yqlplusParser.Source_specContext;
+import com.yahoo.search.yql.yqlplusParser.Source_statementContext;
+import com.yahoo.search.yql.yqlplusParser.StatementContext;
+import com.yahoo.search.yql.yqlplusParser.TimeoutContext;
+import com.yahoo.search.yql.yqlplusParser.TypenameContext;
+import com.yahoo.search.yql.yqlplusParser.UnaryExpressionContext;
+import com.yahoo.search.yql.yqlplusParser.Update_statementContext;
+import com.yahoo.search.yql.yqlplusParser.Update_valuesContext;
+import com.yahoo.search.yql.yqlplusParser.ViewContext;
+import com.yahoo.search.yql.yqlplusParser.WhereContext;
+
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.atn.PredictionMode;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.misc.Nullable;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.RuleNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Translate the ANTLR grammar into the logical representation.
+ */
+final class ProgramParser {
+
+    public yqlplusParser prepareParser(String programName, InputStream input) throws IOException {
+        return prepareParser(programName, new CaseInsensitiveInputStream(input));
+    }
+
+    public yqlplusParser prepareParser(String programName, String input) throws IOException {
+        return prepareParser(programName, new CaseInsensitiveInputStream(input));
+    }
+
+    public yqlplusParser prepareParser(File file) throws IOException {
+        return prepareParser(file.getAbsoluteFile().toString(), new CaseInsensitiveFileStream(file.getAbsolutePath()));
+    }
+
+
+    private yqlplusParser prepareParser(final String programName, CharStream input) {
+        yqlplusLexer lex = new yqlplusLexer(input);
+        lex.addErrorListener(new BaseErrorListener() {
+          @Override
+          public void syntaxError(@NotNull Recognizer<?, ?> recognizer,
+                      @Nullable Object offendingSymbol,
+                      int line,
+                      int charPositionInLine,
+                      @NotNull String msg,
+                      @Nullable RecognitionException e)
+          {
+            throw new ProgramCompileException(new Location(programName, line, charPositionInLine), msg);
+          }
+
+        });
+        TokenStream tokens = new CommonTokenStream(lex);
+        yqlplusParser parser = new yqlplusParser(tokens);
+        parser.addErrorListener(new BaseErrorListener() {
+          @Override
+          public void syntaxError(@NotNull Recognizer<?, ?> recognizer,
+                      @Nullable Object offendingSymbol,
+                      int line,
+                      int charPositionInLine,
+                      @NotNull String msg,
+                      @Nullable RecognitionException e)
+          {
+            throw new ProgramCompileException(new Location(programName, line, charPositionInLine), msg);
+          }
+
+        });
+        parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
+        return parser;
+    }
+
+    private ProgramContext parseProgram(yqlplusParser parser) throws  RecognitionException {
+        try {
+            return parser.program();
+        } catch (RecognitionException e) {
+            //Retry parsing using full LL mode
+            parser.reset();
+            parser.getInterpreter().setPredictionMode(PredictionMode.LL);
+            return parser.program();
+        }
+    }
+
+    public OperatorNode<StatementOperator> parse(String programName, InputStream program) throws IOException, RecognitionException {
+        yqlplusParser parser = prepareParser(programName, program);
+        return convertProgram(parseProgram(parser), parser, programName);
+    }
+
+    public OperatorNode<StatementOperator> parse(String programName, String program) throws IOException, RecognitionException {
+        yqlplusParser parser = prepareParser(programName, program);
+        return convertProgram(parseProgram(parser), parser, programName);
+    }
+
+    public OperatorNode<StatementOperator> parse(File input) throws IOException, RecognitionException {
+        yqlplusParser parser = prepareParser(input);
+        return convertProgram(parseProgram(parser), parser, input.getAbsoluteFile().toString());
+    }
+
+    public OperatorNode<ExpressionOperator> parseExpression(String input) throws IOException, RecognitionException {
+        return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), new Scope());
+    }
+
+    public OperatorNode<ExpressionOperator> parseExpression(String input, Set<String> visibleAliases) throws IOException, RecognitionException {
+        Scope scope = new Scope();
+        final Location loc = new Location("<expression>", -1, -1);
+        for (String alias : visibleAliases) {
+            scope.defineDataSource(loc, alias);
+        }
+        return convertExpr(prepareParser("<expression>", input).expression(false).getRuleContext(), scope);
+    }
+
+    private Location toLocation(Scope scope, ParseTree node) {
+        Token start;
+        if (node instanceof ParserRuleContext) {
+            start = ((ParserRuleContext)node).start;
+        } else if (node instanceof TerminalNode) {
+            start = ((TerminalNode)node).getSymbol();
+        } else {
+        	throw new ProgramCompileException("Location is not available for type " + node.getClass());
+        }
+        Location location = new Location(scope != null? scope.programName: "<string>", start.getLine(), start.getCharPositionInLine());
+        return location;
+    }
+
+    private List<String> readName(Namespaced_nameContext node) {
+        List<String> path = Lists.newArrayList();
+        for (ParseTree elt:node.children) {
+            if (!(getParseTreeIndex(elt) == yqlplusParser.DOT)) {
+                path.add(elt.getText());
+            }
+         }
+        return path;
+    }
+
+    static class Binding {
+        private final List<String> binding;
+
+        Binding(String moduleName, String exportName) {
+            this.binding = ImmutableList.of(moduleName, exportName);
+        }
+
+        Binding(String moduleName) {
+            this.binding = ImmutableList.of(moduleName);
+        }
+
+        Binding(List<String> binding) {
+            this.binding = binding;
+        }
+
+        public List<String> toPath() {
+            return binding;
+        }
+
+        public List<String> toPathWith(List<String> rest) {
+            return ImmutableList.copyOf(Iterables.concat(toPath(), rest));
+        }
+    }
+
+    static class Scope {
+        final Scope root;
+        final Scope parent;
+        Set<String> cursors = ImmutableSet.of();
+        Set<String> variables = ImmutableSet.of();
+        Set<String> views = Sets.newHashSet();
+        Map<String, Binding> bindings = Maps.newHashMap();
+        final yqlplusParser parser;
+        final String programName;
+
+        Scope() {
+            this.parser = null;
+            this.programName = null;
+            this.root = this;
+            this.parent = null;
+        }
+
+        Scope(yqlplusParser parser, String programName) {
+            this.parser = parser;
+            this.programName = programName;
+            this.root = this;
+            this.parent = null;
+        }
+
+        Scope(Scope root, Scope parent) {
+            this.root = root;
+            this.parent = parent;
+            this.parser = parent.parser;
+            this.programName = parent.programName;
+        }
+
+        public yqlplusParser getParser() {
+            return parser;
+        }
+
+        public String getProgramName() {
+            return programName;
+        }
+
+        public Set<String> getCursors() {
+            return cursors;
+        }
+
+
+        boolean isBound(String name) {
+            // bindings live only in the 'root' node
+            return root.bindings.containsKey(name);
+        }
+
+        public Binding getBinding(String name) {
+            return root.bindings.get(name);
+        }
+
+        public List<String> resolvePath(List<String> path) {
+            if (path.size() < 1 || !isBound(path.get(0))) {
+                return path;
+            } else {
+                return getBinding(path.get(0)).toPathWith(path.subList(1, path.size()));
+            }
+        }
+
+        boolean isCursor(String name) {
+            return cursors.contains(name) || (parent != null && parent.isCursor(name));
+        }
+
+        boolean isVariable(String name) {
+            return variables.contains(name) || (parent != null && parent.isVariable(name));
+        }
+
+        public void bindModule(Location loc, List<String> binding, String symbolName) {
+            if (isBound(symbolName)) {
+                throw new ProgramCompileException(loc, "Name '%s' is already used.", symbolName);
+            }
+            root.bindings.put(symbolName, new Binding(binding));
+        }
+
+        public void bindModuleSymbol(Location loc, List<String> moduleName, String exportName, String symbolName) {
+            ImmutableList.Builder<String> builder = ImmutableList.builder();
+            builder.addAll(moduleName);
+            builder.add(exportName);
+            bindModule(loc, builder.build(), symbolName);
+        }
+
+        public void defineDataSource(Location loc, String name) {
+            if (isCursor(name)) {
+                throw new ProgramCompileException(loc, "Alias '%s' is already used.", name);
+            }
+            if (cursors.isEmpty()) {
+                cursors = Sets.newHashSet();
+            }
+            cursors.add(name);
+        }
+
+        public void defineVariable(Location loc, String name) {
+            if (isVariable(name)) {
+                throw new ProgramCompileException(loc, "Variable/argument '%s' is already used.", name);
+            }
+            if (variables.isEmpty()) {
+                variables = Sets.newHashSet();
+            }
+            variables.add(name);
+
+        }
+
+        public void defineView(Location loc, String text) {
+            if (this != root) {
+                throw new IllegalStateException("Views MUST be defined in 'root' scope only");
+            }
+            if (views.contains(text)) {
+                throw new ProgramCompileException(loc, "View '%s' already defined", text);
+            }
+            views.add(text);
+        }
+
+        Scope child() {
+            return new Scope(root, this);
+        }
+
+        Scope getRoot() {
+            return root;
+        }
+    }
+
+    private OperatorNode<SequenceOperator> convertSelectOrInsertOrUpdateOrDelete(ParseTree node, Scope scopeParent) {
+
+        Preconditions.checkArgument(node instanceof Select_statementContext || node instanceof Insert_statementContext ||
+              node instanceof Update_statementContext || node instanceof Delete_statementContext);
+
+      // 	SELECT^ select_field_spec select_source where? orderby? limit? offset? timeout? fallback?
+        // select is the only place to define where/orderby/limit/offset and joins
+        Scope scope = scopeParent.child();
+        ProjectionBuilder proj = null;
+        OperatorNode<SequenceOperator> source = null;
+        OperatorNode<ExpressionOperator> filter = null;
+        List<OperatorNode<SortOperator>> orderby = null;
+        OperatorNode<ExpressionOperator> offset = null;
+        OperatorNode<ExpressionOperator> limit = null;
+        OperatorNode<ExpressionOperator> timeout = null;
+        OperatorNode<SequenceOperator> fallback = null;
+        OperatorNode<SequenceOperator> insertValues = null;
+        OperatorNode<ExpressionOperator> updateValues = null;
+
+        ParseTree sourceNode;
+
+        if (node instanceof Select_statementContext ) {
+            sourceNode = node.getChild(2) != null ?  node.getChild(2).getChild(0):null;
+        } else {
+            sourceNode = node.getChild(1);
+        }
+
+        if (sourceNode != null) {
+            switch (getParseTreeIndex(sourceNode)) {
+                // ALL_SOURCE and MULTI_SOURCE are how FROM SOURCES
+                // *|source_name,... are parsed
+                // They can't be used directly with the JOIN syntax at this time
+                case yqlplusParser.RULE_select_source_all: {
+                	Location location = toLocation(scope, sourceNode.getChild(2));
+                    source = OperatorNode.create(location, SequenceOperator.ALL);
+                    source.putAnnotation("alias", "row");
+                    scope.defineDataSource(location, "row");
+                }
+                    break;
+                case yqlplusParser.RULE_select_source_multi:
+                	Source_listContext multiSourceContext = ((Select_source_multiContext) sourceNode).source_list();
+                    source = readMultiSource(scope, multiSourceContext);
+                    source.putAnnotation("alias", "row");
+                    scope.defineDataSource(toLocation(scope, multiSourceContext), "row");
+                    break;
+                case yqlplusParser.RULE_select_source_join:
+                    source = convertSource((ParserRuleContext) sourceNode.getChild(1), scope);
+                    List<Join_exprContext> joinContexts = ((Select_source_joinContext)sourceNode).join_expr();
+                    for (Join_exprContext joinContext:joinContexts) {
+                        source = convertJoin(joinContext, source, scope);
+                    }
+                    break;
+                case yqlplusParser.RULE_insert_source:
+                    Insert_sourceContext insertSourceContext = (Insert_sourceContext) sourceNode;
+                    source = convertSource((ParserRuleContext)insertSourceContext.getChild(1), scope);
+                    break;
+                case yqlplusParser.RULE_delete_source:
+                    source = convertSource((ParserRuleContext)sourceNode.getChild(1), scope);
+                    break;
+                case yqlplusParser.RULE_update_source:
+                    source = convertSource((ParserRuleContext)sourceNode.getChild(0), scope);
+                    break;
+                }
+            } else {
+                source = OperatorNode.create(SequenceOperator.EMPTY);
+            }
+
+            for (int i = 1; i < node.getChildCount(); ++i) {
+                ParseTree child = node.getChild(i);
+                switch (getParseTreeIndex(child)) {
+                case yqlplusParser.RULE_select_field_spec:
+                    if (getParseTreeIndex(child.getChild(0)) == yqlplusParser.RULE_project_spec) {
+                        proj = readProjection(((Project_specContext) child.getChild(0)).field_def(), scope);
+                    }
+                    break;
+                case yqlplusParser.RULE_returning_spec:
+                    proj = readProjection(((Returning_specContext) child).select_field_spec().project_spec().field_def(), scope);
+                    break;
+                case yqlplusParser.RULE_where:
+                    filter = convertExpr(((WhereContext) child).expression(), scope);
+                    break;
+                case yqlplusParser.RULE_orderby:
+                    // OrderbyContext orderby()
+                    List<Orderby_fieldContext> orderFieds = ((OrderbyContext) child)
+                            .orderby_fields().orderby_field();
+                    orderby = Lists.newArrayListWithExpectedSize(orderFieds.size());
+                    for (int j = 0; j < orderFieds.size(); ++j) {
+                        orderby.add(convertSortKey(orderFieds.get(j), scope));
+                    }
+                    break;
+                case yqlplusParser.RULE_limit:
+                    limit = convertExpr(((LimitContext) child).fixed_or_parameter(), scope);
+                    break;
+                case yqlplusParser.RULE_offset:
+                    offset = convertExpr(((OffsetContext) child).fixed_or_parameter(), scope);
+                    break;
+                case yqlplusParser.RULE_timeout:
+                    timeout = convertExpr(((TimeoutContext) child).fixed_or_parameter(), scope);
+                    break;
+                case yqlplusParser.RULE_fallback:
+                    fallback = convertQuery(((FallbackContext) child).select_statement(), scope);
+                    break;
+                case yqlplusParser.RULE_insert_values:
+                    if (child.getChild(0) instanceof yqlplusParser.Query_statementContext) {
+                		insertValues = convertQuery(child.getChild(0).getChild(0), scope);
+                	} else {
+                			insertValues = readBatchValues(((Insert_valuesContext) child).field_names_spec(), ((Insert_valuesContext)child).field_values_group_spec(), scope);
+                	}
+                    break;
+                case yqlplusParser.RULE_update_values:
+                    if (getParseTreeIndex(child.getChild(0)) == yqlplusParser.RULE_field_def) {
+                        updateValues = readValues(((Update_valuesContext)child).field_def(), scope);
+                    } else {
+                        updateValues = readValues((Field_names_specContext)child.getChild(0), (Field_values_specContext)child.getChild(2), scope);
+                    }
+                    break;
+                }
+            }
+        // now assemble the logical plan
+        OperatorNode<SequenceOperator> result = source;
+        // filter
+        if (filter != null) {
+            result = OperatorNode.create(SequenceOperator.FILTER, result, filter);
+        }
+        // insert values
+        if (insertValues != null) {
+            result = OperatorNode.create(SequenceOperator.INSERT, result, insertValues);
+        }
+        // update
+        if (updateValues != null) {
+            if (filter != null) {
+                result = OperatorNode.create(SequenceOperator.UPDATE, source, updateValues, filter);
+            } else {
+                result = OperatorNode.create(SequenceOperator.UPDATE_ALL, source, updateValues);
+            }
+        }
+        // delete
+        if (getParseTreeIndex(node) == yqlplusParser.RULE_delete_statement) {
+            if (filter != null) {
+                result = OperatorNode.create(SequenceOperator.DELETE, source, filter);
+            } else {
+                result = OperatorNode.create(SequenceOperator.DELETE_ALL, source);
+            }
+        }
+        // then sort (or project and sort)
+        boolean projectBeforeSort = false;
+        if (orderby != null) {
+            if (proj != null) {
+                for (OperatorNode<SortOperator> sortKey : orderby) {
+                    OperatorNode<ExpressionOperator> sortExpression = sortKey.getArgument(0);
+                    List<OperatorNode<ExpressionOperator>> sortReadFields = getReadFieldExpressions(sortExpression);
+                    for (OperatorNode<ExpressionOperator> sortReadField : sortReadFields) {
+                        String sortKeyField = sortReadField.getArgument(1);
+                        if (proj.isAlias(sortKeyField)) {
+                            // TODO: Add support for "mixed" case
+                            projectBeforeSort = true;
+                            break;
+                        }
+                    }
+                }
+            }
+            if (projectBeforeSort) {
+                result = OperatorNode.create(SequenceOperator.SORT, proj.make(result), orderby);
+            } else {
+                result = OperatorNode.create(SequenceOperator.SORT, result, orderby);
+            }
+        }
+        // then offset/limit (must be done after sorting!)
+        if (offset != null && limit != null) {
+            result = OperatorNode.create(SequenceOperator.SLICE, result, offset, limit);
+        } else if (offset != null) {
+            result = OperatorNode.create(SequenceOperator.OFFSET, result, offset);
+        } else if (limit != null) {
+            result = OperatorNode.create(SequenceOperator.LIMIT, result, limit);
+        }
+        // finally, project (if not already)
+        if (proj != null && !projectBeforeSort) {
+            result = proj.make(result);
+        }
+        if (timeout != null) {
+            result = OperatorNode.create(SequenceOperator.TIMEOUT, result, timeout);
+        }
+        // if there's a fallback, emit a fallback node
+        if (fallback != null) {
+            result = OperatorNode.create(SequenceOperator.FALLBACK, result, fallback);
+        }
+        return result;
+    }
+
+    private OperatorNode<ExpressionOperator> readValues(List<Field_defContext> fieldDefs, Scope scope) {
+        List<String> fieldNames;
+        List<OperatorNode<ExpressionOperator>> fieldValues;
+        int numPairs = fieldDefs.size();
+        fieldNames = Lists.newArrayListWithExpectedSize(numPairs);
+        fieldValues = Lists.newArrayListWithExpectedSize(numPairs);
+        for (int j = 0; j < numPairs; j++) {
+            ParseTree startNode = fieldDefs.get(j);
+            while(startNode.getChildCount() < 3) {
+                startNode = startNode.getChild(0);
+            }
+            fieldNames.add((String) convertExpr(startNode.getChild(0), scope).getArgument(1));
+            fieldValues.add(convertExpr(startNode.getChild(2), scope));
+        }
+        return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues);
+    }
+
+    private OperatorNode<SequenceOperator> readMultiSource(Scope scope, Source_listContext multiSource) {
+        List<List<String>> sourceNameList = Lists.newArrayList();
+        List<Namespaced_nameContext> nameSpaces = multiSource.namespaced_name();
+        for(Namespaced_nameContext node : nameSpaces) {
+            List<String> name = readName(node);
+            sourceNameList.add(name);
+        }
+        return OperatorNode.create(toLocation(scope, multiSource), SequenceOperator.MULTISOURCE, sourceNameList);
+    }
+//    pipeline_step
+//    : namespaced_name arguments[false]?
+//    ;
+    private OperatorNode<SequenceOperator> convertPipe(Query_statementContext queryStatementContext, List<Pipeline_stepContext> nodes, Scope scope) {
+        OperatorNode<SequenceOperator> result = convertQuery(queryStatementContext.getChild(0), scope.getRoot());
+        for (Pipeline_stepContext step:nodes) {
+            if (getParseTreeIndex(step.getChild(0)) == yqlplusParser.RULE_vespa_grouping) {
+                result = OperatorNode.create(SequenceOperator.PIPE, result, ImmutableList.<String>of(),
+                                             ImmutableList.of(convertExpr(step.getChild(0), scope)));
+            } else {
+                List<String> name = readName(step.namespaced_name());
+                List<OperatorNode<ExpressionOperator>> args = ImmutableList.of();
+                //LPAREN (argument[$in_select] (COMMA argument[$in_select])*) RPAREN
+                if (step.getChildCount() > 1) {
+                    ArgumentsContext arguments = step.arguments();
+                    if (arguments.getChildCount() > 2) {
+                        List<ArgumentContext> argumentContextList = arguments.argument();
+                        args = Lists.newArrayListWithExpectedSize(argumentContextList.size());
+                        for (ArgumentContext argumentContext: argumentContextList) {
+                            args.add(convertExpr(argumentContext.expression(), scope.getRoot()));
+
+                        }
+                    }
+                }
+                result = OperatorNode.create(SequenceOperator.PIPE, result, scope.resolvePath(name), args);
+            }
+        }
+        return result;
+    }
+
+    private OperatorNode<SequenceOperator> convertMerge(List<Merge_componentContext> mergeComponentList, Scope scope) {
+          Preconditions.checkArgument(mergeComponentList != null);
+          List<OperatorNode<SequenceOperator>> sources = Lists.newArrayListWithExpectedSize(mergeComponentList.size());
+          for (Merge_componentContext mergeComponent:mergeComponentList) {
+              Select_statementContext selectContext = mergeComponent.select_statement();
+              Source_statementContext sourceContext = mergeComponent.source_statement();
+              if (selectContext != null) {
+                  sources.add(convertQuery(selectContext, scope.getRoot()));
+              } else {
+                  sources.add(convertQuery(sourceContext, scope.getRoot()));
+              }
+          }
+          return OperatorNode.create(SequenceOperator.MERGE, sources);
+      }
+
+    private OperatorNode<SequenceOperator> convertQuery(ParseTree node, Scope scope) {
+        if (node instanceof Select_statementContext
+           || node instanceof Insert_statementContext
+           || node instanceof Update_statementContext
+           || node instanceof Delete_statementContext) {
+            return convertSelectOrInsertOrUpdateOrDelete(node, scope.getRoot());
+        } else if (node instanceof Source_statementContext) { //for pipe
+            Source_statementContext sourceStatementContext = (Source_statementContext)node;
+            return convertPipe(sourceStatementContext.query_statement(), sourceStatementContext.pipeline_step(), scope);
+        } else if (node instanceof Merge_statementContext) {
+            return convertMerge(((Merge_statementContext)node).merge_component(), scope);
+        } else {
+            throw new IllegalArgumentException("Unexpected argument type to convertQueryStatement: " + node.toStringTree());
+        }
+
+    }
+
+    private OperatorNode<SequenceOperator> convertJoin(Join_exprContext node, OperatorNode<SequenceOperator> left, Scope scope) {
+        Source_specContext sourceSpec = node.source_spec();
+        OperatorNode<SequenceOperator> right = convertSource(sourceSpec, scope);
+        JoinExpressionContext joinContext = node.joinExpression();
+        OperatorNode<ExpressionOperator> joinExpression = readBinOp(ExpressionOperator.valueOf("EQ"), joinContext.getChild(0), joinContext.getChild(2), scope);
+        if (joinExpression.getOperator() != ExpressionOperator.EQ) {
+            throw new ProgramCompileException(joinExpression.getLocation(), "Unexpected join expression type: %s (expected EQ)", joinExpression.getOperator());
+        }
+        return OperatorNode.create(toLocation(scope, sourceSpec), node.join_spec().LEFT() != null ? SequenceOperator.LEFT_JOIN : SequenceOperator.JOIN, left, right, joinExpression);
+    }
+
+    private String assignAlias(String alias, ParserRuleContext node, Scope scope) {
+        if (alias == null) {
+            alias = "source";
+        }
+        
+        if (node != null && node instanceof yqlplusParser.Alias_defContext) {
+            //alias_def :   (AS? ID);
+            ParseTree idChild = node;
+            if (node.getChildCount() > 1) {
+                idChild = node.getChild(1);
+            }
+            alias = idChild.getText();
+            if (scope.isCursor(alias)) {
+                throw new ProgramCompileException(toLocation(scope, idChild), "Source alias '%s' is already used", alias);
+            }
+            scope.defineDataSource(toLocation(scope, idChild), alias);
+            return alias;
+        } else {
+            String candidate = alias;
+            int c = 0;
+            while (scope.isCursor(candidate)) {
+                candidate = alias + (++c);
+            }
+            scope.defineDataSource(null, candidate);
+            return alias;
+        }
+   }
+
+   private OperatorNode<SequenceOperator> convertSource(ParserRuleContext sourceSpecNode, Scope scope) {
+
+        // DataSources
+       String alias;
+       OperatorNode<SequenceOperator> result;
+       ParserRuleContext dataSourceNode = sourceSpecNode;
+       ParserRuleContext aliasContext = null;
+       //data_source
+       //:   call_source
+       //|   LPAREN source_statement RPAREN
+       //|   sequence_source
+       //;
+       if (sourceSpecNode instanceof Source_specContext) {
+           dataSourceNode = (ParserRuleContext)sourceSpecNode.getChild(0);
+           if (sourceSpecNode.getChildCount() == 2) {
+               aliasContext = (ParserRuleContext)sourceSpecNode.getChild(1);
+           }
+           if (dataSourceNode.getChild(0) instanceof Call_sourceContext ||
+               dataSourceNode.getChild(0) instanceof  Sequence_sourceContext) {
+               dataSourceNode = (ParserRuleContext)dataSourceNode.getChild(0);              
+           } else { //source_statement
+               dataSourceNode = (ParserRuleContext)dataSourceNode.getChild(1); 
+           }
+       }
+       switch (getParseTreeIndex(dataSourceNode)) {
+           case yqlplusParser.RULE_write_data_source:
+           case yqlplusParser.RULE_call_source: {
+               List<String> names = readName((Namespaced_nameContext)dataSourceNode.getChild(Namespaced_nameContext.class, 0));
+               alias = assignAlias(names.get(names.size() - 1), aliasContext, scope);
+               List<OperatorNode<ExpressionOperator>> arguments = ImmutableList.of();
+               ArgumentsContext argumentsContext = dataSourceNode.getRuleContext(ArgumentsContext.class,0);
+               if ( argumentsContext != null) {
+                   List<ArgumentContext> argumentContexts = argumentsContext.argument();
+                   arguments = Lists.newArrayListWithExpectedSize(argumentContexts.size());
+                   for (ArgumentContext argumentContext:argumentContexts) {
+                       arguments.add(convertExpr(argumentContext, scope));
+                   }
+               }
+               if (names.size() == 1 && scope.isVariable(names.get(0))) {
+                   String ident = names.get(0);
+                   if (arguments.size() > 0) {
+                       throw new ProgramCompileException(toLocation(scope, argumentsContext), "Invalid call-with-arguments on local source '%s'", ident);
+                   }
+                   result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.EVALUATE, OperatorNode.create(toLocation(scope, dataSourceNode), ExpressionOperator.VARREF, ident));
+                } else {
+                   result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.SCAN, scope.resolvePath(names), arguments);
+                }
+                break;
+            }
+            case yqlplusParser.RULE_sequence_source: {
+                IdentContext identContext = dataSourceNode.getRuleContext(IdentContext.class,0);
+                String ident = identContext.getText();
+                if (!scope.isVariable(ident)) {
+                    throw new ProgramCompileException(toLocation(scope, identContext), "Unknown variable reference '%s'", ident);
+                }
+                alias = assignAlias(ident, aliasContext, scope);
+                result = OperatorNode.create(toLocation(scope, dataSourceNode), SequenceOperator.EVALUATE, OperatorNode.create(toLocation(scope, dataSourceNode), ExpressionOperator.VARREF, ident));
+                break;
+            }
+            case yqlplusParser.RULE_source_statement: {
+                alias = assignAlias(null, dataSourceNode, scope);
+                result = convertQuery(dataSourceNode, scope);
+                break;
+            }
+            default:
+                throw new IllegalArgumentException("Unexpected argument type to convertSource: " + dataSourceNode.getText());
+        }
+        result.putAnnotation("alias", alias);
+        return result;
+    }
+
+    private OperatorNode<TypeOperator> decodeType(Scope scope, TypenameContext type) {
+
+        TypeOperator op;
+        ParseTree typeNode = type.getChild(0);
+        switch (getParseTreeIndex(typeNode)) {
+            case yqlplusParser.TYPE_BOOLEAN:
+                op = TypeOperator.BOOLEAN;
+                break;
+            case yqlplusParser.TYPE_BYTE:
+                op = TypeOperator.BYTE;
+                break;
+            case yqlplusParser.TYPE_DOUBLE:
+                op = TypeOperator.DOUBLE;
+                break;
+            case yqlplusParser.TYPE_INT16:
+                op = TypeOperator.INT16;
+                break;
+            case yqlplusParser.TYPE_INT32:
+                op = TypeOperator.INT32;
+                break;
+            case yqlplusParser.TYPE_INT64:
+                op = TypeOperator.INT64;
+                break;
+            case yqlplusParser.TYPE_STRING:
+                op = TypeOperator.STRING;
+                break;
+            case yqlplusParser.TYPE_TIMESTAMP:
+                op = TypeOperator.TIMESTAMP;
+                break;
+            case yqlplusParser.RULE_arrayType:
+                return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.ARRAY, decodeType(scope, ((ArrayTypeContext)typeNode).getChild(TypenameContext.class, 0)));
+            case yqlplusParser.RULE_mapType:
+                return OperatorNode.create(toLocation(scope, typeNode), TypeOperator.MAP, decodeType(scope, ((MapTypeContext)typeNode).getChild(TypenameContext.class, 0)));
+            default:
+                throw new ProgramCompileException("Unknown type " + typeNode.getText());
+        }
+        return OperatorNode.create(toLocation(scope, typeNode), op);
+    }
+
+    private List<String> createBindingName(ParseTree node) {
+        if (node instanceof ModuleNameContext) {
+            if (((ModuleNameContext)node).namespaced_name() != null) {
+                return readName(((ModuleNameContext)node).namespaced_name());
+            } else if (((ModuleNameContext)node).literalString() != null) {
+                return ImmutableList.of(((ModuleNameContext)node).literalString().STRING().getText());
+            }
+        } else if (node instanceof ModuleIdContext) {
+            return ImmutableList.of(node.getText());
+        }
+        throw new ProgramCompileException("Wrong context");
+    }
+
+    private OperatorNode<StatementOperator> convertProgram(
+            ParserRuleContext program, yqlplusParser parser, String programName) {
+        Scope scope = new Scope(parser, programName);
+        List<OperatorNode<StatementOperator>> stmts = Lists.newArrayList();
+        int output = 0;
+        for (ParseTree node : program.children) {
+            if (!(node instanceof ParserRuleContext)) {
+                continue;
+            }
+            ParserRuleContext ruleContext = (ParserRuleContext) node;
+            switch (ruleContext.getRuleIndex()) {
+            case yqlplusParser.RULE_params: {
+                // ^(ARGUMENT ident typeref expression?)
+                ParamsContext paramsContext = (ParamsContext) ruleContext;
+                Program_arglistContext program_arglistContext = paramsContext.program_arglist();
+                if (program_arglistContext != null) {
+                    List<Procedure_argumentContext> argList = program_arglistContext.procedure_argument();
+                    for (Procedure_argumentContext procedureArgumentContext : argList) {
+                        String name = procedureArgumentContext.ident().getText();
+                        OperatorNode<TypeOperator> type = decodeType(scope, procedureArgumentContext.getChild(TypenameContext.class, 0));
+                        OperatorNode<ExpressionOperator> defaultValue = OperatorNode.create(ExpressionOperator.NULL);
+                        if (procedureArgumentContext.expression() != null) {
+                            defaultValue = convertExpr(procedureArgumentContext.expression(), scope);
+                        }
+                        scope.defineVariable(toLocation(scope, procedureArgumentContext), name);
+                        stmts.add(OperatorNode.create(StatementOperator.ARGUMENT, name, type, defaultValue));
+                    }
+                }
+                break;
+            }
+            case yqlplusParser.RULE_import_statement: {
+                Import_statementContext importContext = (Import_statementContext) ruleContext;
+                if (null == importContext.import_list()) {
+                     List<String> name = createBindingName(node.getChild(1));
+                    String target;
+                    Location location = toLocation(scope, node.getChild(1));
+                    if (node.getChildCount() == 2) {
+                        target = name.get(0);
+                    } else if (node.getChildCount() == 4) {
+                        target = node.getChild(3).getText();
+                    } else {
+                        throw new ProgramCompileException("Unknown node count for IMPORT: " + node.toStringTree());
+                    }
+                    scope.bindModule(location, name, target);
+                } else {
+                    // | FROM moduleName IMPORT import_list -> ^(IMPORT_FROM
+                    // moduleName import_list+)
+                    Import_listContext importListContext = importContext.import_list();
+                    List<String> name = createBindingName(importContext.moduleName());
+                    Location location = toLocation(scope, importContext.moduleName());
+                    List<ModuleIdContext> moduleIds = importListContext.moduleId();
+                    List<String> symbols = Lists.newArrayListWithExpectedSize(moduleIds.size());
+                    for (ModuleIdContext cnode : moduleIds) {
+                        symbols.add(cnode.ID().getText());
+                    }
+                    for (String sym : symbols) {
+                        scope.bindModuleSymbol(location, name, sym, sym);
+                    }
+                }
+                break;
+            }
+
+            // DDL
+            case yqlplusParser.RULE_ddl:
+                ruleContext = (ParserRuleContext)ruleContext.getChild(0);
+            case yqlplusParser.RULE_view: {
+                // view and projection expansion now has to be done by the
+                // execution engine
+                // since views/projections, in order to be useful, have to
+                // support being used from outside the same program
+                ViewContext viewContext = (ViewContext) ruleContext;
+                Location loc = toLocation(scope, viewContext);
+                scope.getRoot().defineView(loc, viewContext.ID().getText());
+                stmts.add(OperatorNode.create(loc, StatementOperator.DEFINE_VIEW, viewContext.ID().getText(), convertQuery(viewContext.source_statement(), scope.getRoot())));
+                break;
+            }
+            case yqlplusParser.RULE_statement: {
+                // ^(STATEMENT_QUERY source_statement paged_clause?
+                // output_spec?)
+                StatementContext statementContext = (StatementContext) ruleContext;
+                switch (getParseTreeIndex(ruleContext.getChild(0))) {
+                case yqlplusParser.RULE_selectvar_statement: {
+                    // ^(STATEMENT_SELECTVAR ident source_statement)
+                    Selectvar_statementContext selectVarContext = (Selectvar_statementContext) ruleContext.getChild(0);
+                    String variable = selectVarContext.ident().getText();
+                    OperatorNode<SequenceOperator> query = convertQuery(selectVarContext.source_statement(), scope);
+                    Location location = toLocation(scope, selectVarContext.ident());
+                    scope.defineVariable(location, variable);
+                    stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable));
+                    break;
+                }
+                case yqlplusParser.RULE_next_statement: {
+                    // NEXT^ literalString OUTPUT! AS! ident
+                    Next_statementContext nextStateContext = (Next_statementContext) ruleContext.getChild(0);
+                    String continuationValue = StringUnescaper.unquote(nextStateContext.literalString().getText());
+                    String variable = nextStateContext.ident().getText();
+                    Location location = toLocation(scope, node);
+                    OperatorNode<SequenceOperator> next = OperatorNode.create(location, SequenceOperator.NEXT, continuationValue);
+                    stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, next, variable));
+                    stmts.add(OperatorNode.create(location, StatementOperator.OUTPUT, variable));
+                    scope.defineVariable(location, variable);
+                    break;
+                }
+                case yqlplusParser.RULE_output_statement:
+                    Source_statementContext source_statement = statementContext.output_statement().source_statement();
+                    OperatorNode<SequenceOperator> query;
+                    if (source_statement.getChildCount() == 1) {
+                        query = convertQuery( source_statement.query_statement().getChild(0), scope);
+                    } else {
+                        query = convertQuery(source_statement, scope);
+                    }
+                    String variable = "result" + (++output);
+                    boolean isCountVariable = false;
+                    OperatorNode<ExpressionOperator> pageSize = null;
+                    ParseTree outputStatement = node.getChild(0);
+                    Location location = toLocation(scope, outputStatement);
+                    for (int i = 1; i < outputStatement.getChildCount(); ++i) {
+                        ParseTree child = outputStatement.getChild(i);
+                        switch (getParseTreeIndex(child)) {
+                        case yqlplusParser.RULE_paged_clause:
+                            Paged_clauseContext pagedContext = (Paged_clauseContext) child;
+                            pageSize = convertExpr(pagedContext.fixed_or_parameter(), scope);
+                            break;
+                        case yqlplusParser.RULE_output_spec:
+                            Output_specContext outputSpecContext = (Output_specContext) child;
+                            variable = outputSpecContext.ident().getText();
+                            if (outputSpecContext.COUNT() != null) {
+                                isCountVariable = true;
+                            }
+                            break;
+                        default:
+                            throw new ProgramCompileException( "Unknown statement attribute: " + child.toStringTree());
+                        }
+                    }
+                    scope.defineVariable(location, variable);
+                    if (pageSize != null) {
+                        query = OperatorNode.create(SequenceOperator.PAGE, query, pageSize);
+                    }
+                    stmts.add(OperatorNode.create(location, StatementOperator.EXECUTE, query, variable));
+                    stmts.add(OperatorNode.create(location, isCountVariable ? StatementOperator.COUNT:StatementOperator.OUTPUT, variable));
+                }
+                break;
+            }
+            default:
+                throw new ProgramCompileException("Unknown program element: " + node.getText());
+            }
+        }
+        // traverse the tree, find all of the namespaced calls not covered by
+        // imports so we can
+        // define "implicit" import statements for them (to make engine
+        // implementation easier)
+        return OperatorNode.create(StatementOperator.PROGRAM, stmts);
+    }
+
+    private OperatorNode<SortOperator> convertSortKey(Orderby_fieldContext node, Scope scope) {
+        TerminalNode descDef = node.DESC();
+        OperatorNode<ExpressionOperator> exprNode = convertExpr(node.expression(), scope);
+        if (descDef != null ) {
+            return OperatorNode.create(toLocation(scope, descDef), SortOperator.DESC, exprNode);
+        } else {
+            return OperatorNode.create(toLocation(scope, node), SortOperator.ASC, exprNode);
+        }
+    }
+
+    private ProjectionBuilder readProjection(List<Field_defContext> fieldDefs, Scope scope) {
+        if (null == fieldDefs)
+                throw new ProgramCompileException("Null fieldDefs");
+        ProjectionBuilder proj = new ProjectionBuilder();
+        for (Field_defContext rulenode : fieldDefs) {
+            // FIELD
+                // expression alias_def?
+                OperatorNode<ExpressionOperator> expr = convertExpr((ExpressionContext)rulenode.getChild(0), scope);
+
+                String aliasName = null;
+                if (rulenode.getChildCount() > 1) {
+                   // ^(ALIAS ID)
+                    aliasName = rulenode.alias_def().ID().getText();
+                }
+                proj.addField(aliasName, expr);
+            // no grammar for the other rule types at this time
+        }
+        return proj;
+    }
+
+    public static int getParseTreeIndex(ParseTree parseTree) {
+        if (parseTree instanceof TerminalNode) {
+            return ((TerminalNode)parseTree).getSymbol().getType();
+        } else {
+            return ((RuleNode)parseTree).getRuleContext().getRuleIndex();
+        }
+    }
+
+	public OperatorNode<ExpressionOperator> convertExpr(ParseTree parseTree,
+			Scope scope) {
+	  switch (getParseTreeIndex(parseTree)) {
+	        case yqlplusParser.RULE_vespa_grouping: {
+	                ParseTree firstChild = parseTree.getChild(0);
+	                if (getParseTreeIndex(firstChild) == yqlplusParser.RULE_annotation) {
+	                    ParseTree secondChild = parseTree.getChild(1);
+	                    OperatorNode<ExpressionOperator> annotation = convertExpr(((AnnotationContext) firstChild)
+	                            .constantMapExpression(), scope);
+	                    OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild),
+	                            ExpressionOperator.VESPA_GROUPING, secondChild.getText());
+	                    List<String> names = (List<String>) annotation.getArgument(0);
+	                    List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation
+	                            .getArgument(1);
+	                    for (int i = 0; i < names.size(); ++i) {
+	                        expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i)));
+	                    }
+	                    return expr;
+	                } else {
+	                    return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VESPA_GROUPING,
+	                            firstChild.getText());
+	                }
+	        }
+		case yqlplusParser.RULE_nullOperator:
+			return OperatorNode.create(ExpressionOperator.NULL);
+		case yqlplusParser.RULE_argument:
+			return convertExpr(parseTree.getChild(0), scope);
+		case yqlplusParser.RULE_fixed_or_parameter: {
+			ParseTree firstChild = parseTree.getChild(0);
+			if (getParseTreeIndex(firstChild) == yqlplusParser.INT) {
+				return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.LITERAL, new Integer(firstChild.getText()));
+			} else {
+				return convertExpr(firstChild, scope);
+			}
+		}
+        case yqlplusParser.RULE_constantMapExpression: {
+            List<ConstantPropertyNameAndValueContext> propertyList = ((ConstantMapExpressionContext) parseTree).constantPropertyNameAndValue();
+            List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size());
+            List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithExpectedSize(propertyList.size());
+            for (ConstantPropertyNameAndValueContext child : propertyList) {
+                // : propertyName ':' expression[$expression::namespace] ->
+                // ^(PROPERTY propertyName expression)
+                names.add(StringUnescaper.unquote(child.getChild(0).getText()));
+                exprs.add(convertExpr(child.getChild(2), scope));
+            }
+            return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs);
+        }
+		case yqlplusParser.RULE_mapExpression: {
+			List<PropertyNameAndValueContext> propertyList = ((MapExpressionContext)parseTree).propertyNameAndValue();
+			List<String> names = Lists.newArrayListWithExpectedSize(propertyList.size());
+			List<OperatorNode<ExpressionOperator>> exprs = Lists.newArrayListWithCapacity(propertyList.size());
+			for (PropertyNameAndValueContext child : propertyList) {
+				// : propertyName ':' expression[$expression::namespace] ->
+				// ^(PROPERTY propertyName expression)
+				names.add(StringUnescaper.unquote(child.getChild(0).getText()));
+				exprs.add(convertExpr(child.getChild(2), scope));
+			}
+			return OperatorNode.create(toLocation(scope, parseTree),ExpressionOperator.MAP, names, exprs);
+		}
+		case yqlplusParser.RULE_constantArray: {
+            List<ConstantExpressionContext> expressionList = ((ConstantArrayContext)parseTree).constantExpression();
+            List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size());
+            for (ConstantExpressionContext expr : expressionList) {
+                values.add(convertExpr(expr, scope));
+            }
+            return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values);
+        }
+		case yqlplusParser.RULE_arrayLiteral: {
+			List<ExpressionContext> expressionList = ((ArrayLiteralContext) parseTree).expression();
+			List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(expressionList.size());
+			for (ExpressionContext expr : expressionList) {
+				values.add(convertExpr(expr, scope));
+			}
+			return OperatorNode.create(toLocation(scope, expressionList.isEmpty()? parseTree:expressionList.get(0)), ExpressionOperator.ARRAY, values);
+		}
+		//dereferencedExpression: primaryExpression(indexref[in_select]| propertyref)*
+		case yqlplusParser.RULE_dereferencedExpression: {
+			DereferencedExpressionContext dereferencedExpression = (DereferencedExpressionContext) parseTree;
+			Iterator<ParseTree> it = dereferencedExpression.children.iterator();
+			OperatorNode<ExpressionOperator> result = convertExpr(it.next(), scope);
+			while (it.hasNext()) {
+				ParseTree defTree = it.next();
+				if (getParseTreeIndex(defTree) == yqlplusParser.RULE_propertyref) {
+				    //DOT nm=ID
+					result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.PROPREF, result, defTree.getChild(1).getText());
+				} else {
+				    //indexref
+					result = OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.INDEX, result, convertExpr(defTree.getChild(1), scope));
+				}
+			}
+			return result;
+		}
+		case yqlplusParser.RULE_primaryExpression: {
+			// ^(CALL namespaced_name arguments)
+		    ParseTree firstChild = parseTree.getChild(0);
+			switch (getParseTreeIndex(firstChild)) {
+			    case yqlplusParser.RULE_fieldref: {
+			         return convertExpr(firstChild, scope);
+			    }
+			    case yqlplusParser.RULE_callExpresion: {
+					List<ArgumentContext> args = ((ArgumentsContext) firstChild.getChild(1)).argument();
+					List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(args.size());
+					for (ArgumentContext argContext : args) {
+						arguments.add(convertExpr(argContext.expression(),scope));
+					}
+					return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.CALL, scope.resolvePath(readName((Namespaced_nameContext) firstChild.getChild(0))), arguments);
+				}
+				// TODO add processing this is not implemented in V3
+				// case yqlplusParser.APPLY:
+
+			    case yqlplusParser.RULE_parameter:
+			        // external variable reference
+			        return OperatorNode.create(toLocation(scope, firstChild), ExpressionOperator.VARREF, firstChild.getChild(1).getText());
+			    case yqlplusParser.RULE_scalar_literal:
+			    case yqlplusParser.RULE_arrayLiteral:
+			    case yqlplusParser.RULE_mapExpression:
+			        return convertExpr(firstChild, scope);
+			    case yqlplusParser.LPAREN:
+			        return convertExpr(parseTree.getChild(1), scope);
+			}
+		}
+		// TODO: Temporarily disable CAST - think through how types are named
+		// case yqlplusParser.CAST: {
+		//
+		// return new Cast()
+		// }
+		// return new CastExpression(payload);
+		case yqlplusParser.RULE_parameter: {
+			// external variable reference
+			ParserRuleContext parameterContext = (ParserRuleContext) parseTree;
+			IdentContext identContext = parameterContext.getRuleContext(IdentContext.class, 0);
+			return OperatorNode.create(toLocation(scope, identContext), ExpressionOperator.VARREF, identContext.getText());
+		}
+		case yqlplusParser.RULE_annotateExpression: {
+		    //annotation logicalORExpression
+			AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation();
+			OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope);
+			OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope);
+			List<String> names = (List<String>) annotation.getArgument(0);
+			List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation.getArgument(1);
+			for (int i = 0; i < names.size(); ++i) {
+				expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i)));
+			}
+			return expr;
+		}
+		case yqlplusParser.RULE_expression: {
+		    return convertExpr(parseTree.getChild(0), scope);
+		}
+		case yqlplusParser.RULE_logicalANDExpression:
+			LogicalANDExpressionContext andExpressionContext = (LogicalANDExpressionContext) parseTree;
+			return readConjOp(ExpressionOperator.AND, andExpressionContext.equalityExpression(), scope);
+		case yqlplusParser.RULE_logicalORExpression: {
+			int childCount = parseTree.getChildCount();
+			LogicalORExpressionContext logicalORExpressionContext = (LogicalORExpressionContext) parseTree;
+			if (childCount > 1) {
+				return readConjOrOp(ExpressionOperator.OR, logicalORExpressionContext, scope);
+			} else {
+				List<EqualityExpressionContext> equalityExpressionList = ((LogicalANDExpressionContext) parseTree.getChild(0)).equalityExpression();
+				if (equalityExpressionList.size() > 1) {
+					return readConjOp(ExpressionOperator.AND, equalityExpressionList, scope);
+				} else {
+					return convertExpr(equalityExpressionList.get(0), scope);
+				}
+			}
+		}
+		case yqlplusParser.RULE_equalityExpression: {
+			EqualityExpressionContext equalityExpression = (EqualityExpressionContext) parseTree;
+			RelationalExpressionContext relationalExpressionContext = equalityExpression.relationalExpression(0);
+			OperatorNode<ExpressionOperator> expr = convertExpr(relationalExpressionContext, scope);
+			InNotInTargetContext inNotInTarget = equalityExpression.inNotInTarget();
+			int childCount = equalityExpression.getChildCount();
+			if (childCount == 1) {
+				return expr;
+			}
+			if (inNotInTarget != null) {
+				Literal_listContext literalListContext = inNotInTarget.literal_list();
+				boolean isIN = equalityExpression.IN() != null;
+				if (literalListContext == null) {
+					Select_statementContext selectStatementContext = inNotInTarget.select_statement();
+					OperatorNode<SequenceOperator> query = convertQuery(selectStatementContext, scope);
+					return OperatorNode.create(expr.getLocation(),isIN ? ExpressionOperator.IN_QUERY: ExpressionOperator.NOT_IN_QUERY, expr, query);
+				} else {
+					// we need to identify the type of the target; if it's a
+					// scalar we need to wrap it in a CREATE_ARRAY
+					// if it's already a CREATE ARRAY then it's fine, otherwise
+					// we need to know the variable type
+					// return readBinOp(node.getType() == yqlplusParser.IN ?
+					// ExpressionOperator.IN : ExpressionOperator.NOT_IN, node,
+					// scope);
+					return readBinOp(isIN ? ExpressionOperator.IN: ExpressionOperator.NOT_IN, equalityExpression.getChild(0), literalListContext, scope);
+				}
+
+			} else {
+				ParseTree firstChild = equalityExpression.getChild(1);
+				if (equalityExpression.getChildCount() == 2) {
+					switch (getParseTreeIndex(firstChild)) {
+					    case yqlplusParser.IS_NULL:
+					        return readUnOp(ExpressionOperator.IS_NULL, relationalExpressionContext, scope);
+					    case yqlplusParser.IS_NOT_NULL:
+					        return readUnOp(ExpressionOperator.IS_NOT_NULL, relationalExpressionContext, scope);
+					}
+				} else {
+					switch (getParseTreeIndex(firstChild.getChild(0))) {
+					    case yqlplusParser.EQ:
+					        return readBinOp(ExpressionOperator.EQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope);
+					    case yqlplusParser.NEQ:
+					        return readBinOp(ExpressionOperator.NEQ, equalityExpression.getChild(0), equalityExpression.getChild(2), scope);
+					    case yqlplusParser.LIKE:
+					        return readBinOp(ExpressionOperator.LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope);
+					    case yqlplusParser.NOTLIKE:
+					        return readBinOp(ExpressionOperator.NOT_LIKE, equalityExpression.getChild(0), equalityExpression.getChild(2), scope);
+					    case yqlplusParser.MATCHES:
+					        return readBinOp(ExpressionOperator.MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope);
+					    case yqlplusParser.NOTMATCHES:
+					        return readBinOp(ExpressionOperator.NOT_MATCHES, equalityExpression.getChild(0), equalityExpression.getChild(2), scope);
+					    case yqlplusParser.CONTAINS:
+					        return readBinOp(ExpressionOperator.CONTAINS, equalityExpression.getChild(0), equalityExpression.getChild(2), scope);
+					}
+				}
+
+			}
+			break;
+		}
+		case yqlplusParser.RULE_relationalExpression: {
+			RelationalExpressionContext relationalExpressionContext = (RelationalExpressionContext) parseTree;
+			RelationalOpContext opContext = relationalExpressionContext.relationalOp();
+			if (opContext != null) {
+				switch (getParseTreeIndex(relationalExpressionContext.relationalOp().getChild(0))) {
+					case yqlplusParser.LT:
+					    return readBinOp(ExpressionOperator.LT, parseTree, scope);
+					case yqlplusParser.LTEQ:
+					    return readBinOp(ExpressionOperator.LTEQ, parseTree, scope);
+					case yqlplusParser.GT:
+					    return readBinOp(ExpressionOperator.GT, parseTree, scope);
+					case yqlplusParser.GTEQ:
+					    return readBinOp(ExpressionOperator.GTEQ, parseTree, scope);
+				}
+			} else {
+				return convertExpr(relationalExpressionContext.additiveExpression(0), scope);
+			}
+		    }
+			break;
+		case yqlplusParser.RULE_additiveExpression:
+		case yqlplusParser.RULE_multiplicativeExpression: {
+			if (parseTree.getChildCount() > 1) {
+				String opStr = parseTree.getChild(1).getText();
+				switch (opStr) {
+				    case "+":
+				        return readBinOp(ExpressionOperator.ADD, parseTree, scope);
+				    case "-":
+				        return readBinOp(ExpressionOperator.SUB, parseTree, scope);
+				    case "/":
+				        return readBinOp(ExpressionOperator.DIV, parseTree, scope);
+				    case "*":
+				        return readBinOp(ExpressionOperator.MULT, parseTree, scope);
+				    case "%":
+				        return readBinOp(ExpressionOperator.MOD, parseTree, scope);
+				    default:
+				        if (parseTree.getChild(0) instanceof UnaryExpressionContext) {
+				            return convertExpr(parseTree.getChild(0), scope);
+				        } else {
+				            throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.toStringTree());
+				        }
+				}
+			} else {
+				if (parseTree.getChild(0) instanceof UnaryExpressionContext) {
+					return convertExpr(parseTree.getChild(0), scope);
+				} else if (parseTree.getChild(0) instanceof MultiplicativeExpressionContext) {
+					return convertExpr(parseTree.getChild(0), scope);
+				} else {
+					throw new ProgramCompileException(toLocation(scope, parseTree), "Unknown expression type: " + parseTree.getText());
+				}
+			}
+		}
+		case yqlplusParser.RULE_unaryExpression: {
+			if (1 == parseTree.getChildCount()) {
+				return convertExpr(parseTree.getChild(0), scope);
+			} else if (2 == parseTree.getChildCount()) {
+				if ("-".equals(parseTree.getChild(0).getText())) {
+					return readUnOp(ExpressionOperator.NEGATE, parseTree, scope);
+				} else if ("!".equals(parseTree.getChild(0).getText())) {
+					return readUnOp(ExpressionOperator.NOT, parseTree, scope);
+				}
+				throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown unary operator " + parseTree.getText());
+			} else {
+				throw new ProgramCompileException(toLocation(scope, parseTree),"Unknown child count " + parseTree.getChildCount() + " of " + parseTree.getText());
+			}
+		}
+		case yqlplusParser.RULE_fieldref:
+		case yqlplusParser.RULE_joinDereferencedExpression: {
+			// all in-scope data sources should be defined in scope
+			// the 'first' field in a namespaced reference must be:
+			// - a field name if (and only if) there is exactly one data source
+			// in scope OR
+			// - an alias name, which will be followed by a field name
+			// ^(FIELDREF<FieldReference>[$expression::namespace]
+			// namespaced_name)
+			List<String> path = readName((Namespaced_nameContext) parseTree.getChild(0));
+			Location loc = toLocation(scope, parseTree.getChild(0));
+			String alias = path.get(0);
+			OperatorNode<ExpressionOperator> result = null;
+			int start = 0;
+			if (scope.isCursor(alias)) {
+				if (path.size() > 1) {
+					result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(1));
+					start = 2;
+				} else {
+					result = OperatorNode.create(loc, ExpressionOperator.READ_RECORD, alias);
+					start = 1;
+				}
+			} else if (scope.isBound(alias)) {
+				return OperatorNode.create(loc, ExpressionOperator.READ_MODULE, scope.getBinding(alias).toPathWith(path.subList(1, path.size())));
+			} else if (scope.getCursors().size() == 1) {
+				alias = scope.getCursors().iterator().next();
+				result = OperatorNode.create(loc, ExpressionOperator.READ_FIELD, alias, path.get(0));
+				start = 1;
+			} else {
+				// ah ha, we can't end up with a 'loose' UDF call because it
+				// won't be a module or known alias
+				// so we need not support implicit imports for constants used in
+				// UDFs
+				throw new ProgramCompileException(loc, "Unknown field or alias '%s'", alias);
+			}
+			for (int idx = start; idx < path.size(); ++idx) {
+				result = OperatorNode.create(loc, ExpressionOperator.PROPREF, result, path.get(idx));
+			}
+			return result;
+		}
+		case yqlplusParser.RULE_scalar_literal:
+			return OperatorNode.create(toLocation(scope, parseTree), ExpressionOperator.LITERAL, convertLiteral((Scalar_literalContext) parseTree));
+		case yqlplusParser.RULE_insert_values:
+			return readValues((Insert_valuesContext) parseTree, scope);
+		case yqlplusParser.RULE_constantExpression:
+			return convertExpr(parseTree.getChild(0), scope);
+		case yqlplusParser.RULE_literal_list:
+			if (getParseTreeIndex(parseTree.getChild(1)) == yqlplusParser.RULE_array_parameter) {
+				return convertExpr(parseTree.getChild(1), scope);
+			} else {
+				List<Literal_elementContext> elements = ((Literal_listContext) parseTree).literal_element();
+				ParseTree firldElement = elements.get(0).getChild(0);
+				if (elements.size() == 1 && scope.getParser().isArrayParameter(firldElement)) {
+					return convertExpr(firldElement, scope);
+				} else {
+					List<OperatorNode<ExpressionOperator>> values = Lists.newArrayListWithExpectedSize(elements.size());
+					for (Literal_elementContext child : elements) {
+						values.add(convertExpr(child.getChild(0), scope));
+					}
+					return OperatorNode.create(toLocation(scope, elements.get(0)),ExpressionOperator.ARRAY, values);
+				}
+			}
+		}
+		throw new ProgramCompileException(toLocation(scope, parseTree),
+				"Unknown expression type: " + parseTree.getText());
+	}
+
+    public Object convertLiteral(Scalar_literalContext literal) {
+        int parseTreeIndex = getParseTreeIndex(literal.getChild(0));
+        String text = literal.getChild(0).getText();
+        switch(parseTreeIndex) {
+            case yqlplusParser.INT:
+                return new Integer(text);
+            case yqlplusParser.FLOAT:
+                return new Double(text);
+            case yqlplusParser.STRING:
+                return StringUnescaper.unquote(text);
+            case yqlplusParser.TRUE:
+            case yqlplusParser.FALSE:
+                return new Boolean(text);
+            case yqlplusParser.LONG_INT:
+                return Long.parseLong(text.substring(0, text.length()-1));
+            default:
+                throw new ProgramCompileException("Unknow literal type " + text);
+        }
+    }
+
+    private Object readConstantExpression(OperatorNode<ExpressionOperator> node) {
+        switch (node.getOperator()) {
+            case LITERAL:
+                return node.getArgument(0);
+            case MAP: {
+                ImmutableMap.Builder<String, Object> map = ImmutableMap.builder();
+                List<String> names = (List<String>) node.getArgument(0);
+                List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(1);
+                for (int i = 0; i < names.size(); ++i) {
+                    map.put(names.get(i), readConstantExpression(exprs.get(i)));
+                }
+                return map.build();
+            }
+            case ARRAY: {
+                List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(0);
+                ImmutableList.Builder<Object> lst = ImmutableList.builder();
+                for (OperatorNode<ExpressionOperator> expr : exprs) {
+                    lst.add(readConstantExpression(expr));
+                }
+                return lst.build();
+            }
+            default:
+                throw new ProgramCompileException(node.getLocation(), "Internal error: Unknown constant expression type: " + node.getOperator());
+        }
+    }
+
+    private OperatorNode<ExpressionOperator> readBinOp(ExpressionOperator op, ParseTree node, Scope scope) {
+        assert node.getChildCount() == 3;
+        return OperatorNode.create(op, convertExpr(node.getChild(0), scope), convertExpr(node.getChild(2), scope));
+    }
+
+    private OperatorNode<ExpressionOperator> readBinOp(ExpressionOperator op, ParseTree operand1, ParseTree operand2, Scope scope) {
+        return OperatorNode.create(op, convertExpr(operand1, scope), convertExpr(operand2, scope));
+    }
+
+    private OperatorNode<ExpressionOperator> readConjOp(ExpressionOperator op, List<EqualityExpressionContext> nodes, Scope scope) {
+        List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(nodes.size());
+        for (ParseTree child : nodes) {
+            arguments.add(convertExpr(child, scope));
+        }
+        return OperatorNode.create(op, arguments);
+    }
+
+    private OperatorNode<ExpressionOperator> readConjOrOp(ExpressionOperator op, LogicalORExpressionContext node, Scope scope) {
+        List<LogicalANDExpressionContext> andExpressionList = node.logicalANDExpression();
+        List<OperatorNode<ExpressionOperator>> arguments = Lists.newArrayListWithExpectedSize(andExpressionList.size());
+        for (LogicalANDExpressionContext child : andExpressionList) {
+         	List<EqualityExpressionContext> equalities = child.equalityExpression();
+         	if (equalities.size() == 1) {
+         		arguments.add(convertExpr(equalities.get(0), scope));
+         	} else {
+         		List<OperatorNode<ExpressionOperator>> andArguments = Lists.newArrayListWithExpectedSize(equalities.size());
+         		for (EqualityExpressionContext subTreeChild:equalities) {
+         				andArguments.add(convertExpr(subTreeChild, scope));
+         		}
+         		arguments.add(OperatorNode.create(ExpressionOperator.AND, andArguments));
+         	}
+
+        }
+        return OperatorNode.create(op, arguments);
+    }
+
+    // (IS_NULL | IS_NOT_NULL)
+    // unaryExpression
+    private OperatorNode<ExpressionOperator> readUnOp(ExpressionOperator op, ParseTree node, Scope scope) {
+        assert (node instanceof TerminalNode) || (node.getChildCount() == 1) || (node instanceof UnaryExpressionContext);
+        if (node instanceof TerminalNode) {
+            return OperatorNode.create(op, convertExpr(node, scope));
+        } else if (node.getChildCount() == 1) {
+            return OperatorNode.create(op, convertExpr(node.getChild(0), scope));
+        } else {
+            return OperatorNode.create(op, convertExpr(node.getChild(1), scope));
+        }
+    }
+
+    private OperatorNode<ExpressionOperator> readValues(Field_names_specContext nameDefs, Field_values_specContext values, Scope scope) {
+    	List<Field_defContext> fieldDefs = nameDefs.field_def();
+        List<ExpressionContext> valueDefs = values.expression();
+        assert fieldDefs.size() == valueDefs.size();
+        List<String> fieldNames;
+        List<OperatorNode<ExpressionOperator>> fieldValues;
+        int numPairs = fieldDefs.size();
+            fieldNames = Lists.newArrayListWithExpectedSize(numPairs);
+            fieldValues = Lists.newArrayListWithExpectedSize(numPairs);
+            for (int i = 0; i < numPairs; i++) {
+                fieldNames.add((String) convertExpr(fieldDefs.get(i).expression(), scope).getArgument(1));
+                fieldValues.add(convertExpr(valueDefs.get(i), scope));
+            }
+        return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues);
+    }
+
+    private OperatorNode<ExpressionOperator> readValues(ParserRuleContext node, Scope scope) {
+        List<String> fieldNames;
+        List<OperatorNode<ExpressionOperator>> fieldValues;
+        if (node.getRuleIndex() == yqlplusParser.RULE_field_def) {
+            Field_defContext fieldDefContext = (Field_defContext)node;
+            //TODO double check
+            fieldNames = Lists.newArrayListWithExpectedSize(node.getChildCount());
+            fieldValues = Lists.newArrayListWithExpectedSize(node.getChildCount());
+            for (int i = 0; i < node.getChildCount(); i++) {
+                fieldNames.add((String) convertExpr(node.getChild(i).getChild(0).getChild(0), scope).getArgument(1));
+                fieldValues.add(convertExpr(node.getChild(i).getChild(0).getChild(1), scope));
+            }
+        } else {
+            assert node.getChildCount() % 2 == 0;
+            int numPairs = node.getChildCount() / 2;
+            fieldNames = Lists.newArrayListWithExpectedSize(numPairs);
+            fieldValues = Lists.newArrayListWithExpectedSize(numPairs);
+            for (int i = 0; i < numPairs; i++) {
+                fieldNames.add((String) convertExpr(node.getChild(i).getChild(0), scope).getArgument(1));
+                fieldValues.add(convertExpr(node.getChild(numPairs + i), scope));
+            }
+        }
+        return OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues);
+    }
+
+    /*
+     * Converts node list
+     *
+     *   a_name, b_name, c_name, a_value_1, b_value_1, c_value_1, a_value_2, b_value_2, c_value2, a_value_3, b_value_3, c_value_3
+     *
+     * into corresponding constant sequence:
+     *
+     *   [ { a_name : a_value_1, b_name : b_value_1, c_name : c_value_1 }, ... ]
+     *
+     */
+    private OperatorNode<SequenceOperator> readBatchValues(Field_names_specContext nameDefs, List<Field_values_group_specContext> valueGroups, Scope scope) {
+    	List<Field_defContext> nameContexts = nameDefs.field_def();
+        List<String> fieldNames = Lists.newArrayList();
+        for (Field_defContext nameContext:nameContexts) {
+        	fieldNames.add((String) convertExpr(nameContext.getChild(0), scope).getArgument(1));
+        }
+        List<OperatorNode> records = Lists.newArrayList();
+        for (Field_values_group_specContext valueGorup:valueGroups) {
+        	List<ExpressionContext> expressionList = valueGorup.expression();
+            List<OperatorNode<ExpressionOperator>> fieldValues = Lists.newArrayListWithExpectedSize(expressionList.size());
+            for (ExpressionContext expressionContext:expressionList) {
+                fieldValues.add(convertExpr(expressionContext, scope));
+            }
+            records.add(OperatorNode.create(ExpressionOperator.MAP, fieldNames, fieldValues));
+        }
+        // Return constant sequence of records with the given name/values
+        return OperatorNode.create(SequenceOperator.EVALUATE, OperatorNode.create(ExpressionOperator.ARRAY, records));
+    }
+
+    /*
+     * Scans the given node for READ_FIELD expressions.
+     *
+     * TODO: Search recursively and consider additional operators
+     *
+     * @param in the node to scan
+     * @return list of READ_FIELD expressions
+     */
+    private List<OperatorNode<ExpressionOperator>> getReadFieldExpressions(OperatorNode<ExpressionOperator> in) {
+        List<OperatorNode<ExpressionOperator>> readFieldList = Lists.newArrayList();
+        switch (in.getOperator()) {
+            case READ_FIELD:
+                readFieldList.add(in);
+                break;
+            case CALL:
+                List<OperatorNode<ExpressionOperator>> callArgs = in.getArgument(1);
+                for (OperatorNode<ExpressionOperator> callArg : callArgs) {
+                    if (callArg.getOperator() == ExpressionOperator.READ_FIELD) {
+                        readFieldList.add(callArg);
+                    }
+                }
+                break;
+        }
+        return readFieldList;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProjectOperator.java b/container-search/src/main/java/com/yahoo/search/yql/ProjectOperator.java
new file mode 100644
index 00000000000..16ecc4c4077
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/ProjectOperator.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Predicate;
+
+/**
+ * Represents a projection command which affects the output record.
+ */
+enum ProjectOperator implements Operator {
+
+    FIELD(ExpressionOperator.class, String.class),  // FIELD expr name
+    RECORD(ExpressionOperator.class, String.class), // RECORD expr name
+    MERGE_RECORD(String.class);                     // MERGE_RECORD name (alias of record to merge)
+
+    private final ArgumentsTypeChecker checker;
+
+    public static Predicate<OperatorNode<? extends Operator>> IS = new Predicate<OperatorNode<? extends Operator>>() {
+        @Override
+        public boolean apply(OperatorNode<? extends Operator> input) {
+            return input.getOperator() instanceof ProjectOperator;
+        }
+    };
+
+    private ProjectOperator(Object... types) {
+        checker = TypeCheckers.make(this, types);
+    }
+
+
+    @Override
+    public void checkArguments(Object... args) {
+        checker.check(args);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProjectionBuilder.java b/container-search/src/main/java/com/yahoo/search/yql/ProjectionBuilder.java
new file mode 100644
index 00000000000..109d1cd654b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/ProjectionBuilder.java
@@ -0,0 +1,73 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import java.util.Map;
+import java.util.Set;
+
+class ProjectionBuilder {
+
+    private Map<String, OperatorNode<ExpressionOperator>> fields = Maps.newLinkedHashMap();
+    private Set<String> aliasNames = Sets.newHashSet();
+
+    public void addField(String name, OperatorNode<ExpressionOperator> expr) {
+        String aliasName = name;
+        if (name == null) {
+            name = assignName(expr);
+        }
+        if (fields.containsKey(name)) {
+            throw new ProgramCompileException(expr.getLocation(), "Field alias '%s' already defined", name);
+        }
+        fields.put(name, expr);
+        if (aliasName != null) {
+            // Store use
+            aliasNames.add(aliasName);
+        }
+    }
+
+    public boolean isAlias(String name) {
+        return aliasNames.contains(name);
+    }
+
+    private String assignName(OperatorNode<ExpressionOperator> expr) {
+        String baseName = "expr";
+        switch (expr.getOperator()) {
+            case PROPREF:
+                baseName = (String) expr.getArgument(1);
+                break;
+            case READ_RECORD:
+                baseName = (String) expr.getArgument(0);
+                break;
+            case READ_FIELD:
+                baseName = (String) expr.getArgument(1);
+                break;
+            case VARREF:
+                baseName = (String) expr.getArgument(0);
+                break;
+            // fall through, leaving baseName alone
+        }
+        int c = 0;
+        String candidate = baseName;
+        while (fields.containsKey(candidate)) {
+            candidate = baseName + (++c);
+        }
+        return candidate;
+    }
+
+    public OperatorNode<SequenceOperator> make(OperatorNode<SequenceOperator> target) {
+        ImmutableList.Builder<OperatorNode<ProjectOperator>> lst = ImmutableList.builder();
+        for (Map.Entry<String, OperatorNode<ExpressionOperator>> e : fields.entrySet()) {
+            if (e.getKey().startsWith("*")) {
+                lst.add(OperatorNode.create(ProjectOperator.MERGE_RECORD, e.getValue().getArgument(0)));
+            } else if (e.getValue().getOperator() == ExpressionOperator.READ_RECORD) {
+                lst.add(OperatorNode.create(ProjectOperator.RECORD, e.getValue(), e.getKey()));
+            } else {
+                lst.add(OperatorNode.create(ProjectOperator.FIELD, e.getValue(), e.getKey()));
+            }
+        }
+        return OperatorNode.create(SequenceOperator.PROJECT, target, lst.build());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/SequenceOperator.java b/container-search/src/main/java/com/yahoo/search/yql/SequenceOperator.java
new file mode 100644
index 00000000000..65d1e039e10
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/SequenceOperator.java
@@ -0,0 +1,68 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Predicate;
+import com.google.inject.TypeLiteral;
+
+import java.util.List;
+
+/**
+ * Logical sequence operators represent a logical description of a "source" (query against data stores + pipes), representing
+ * a source_expression in the grammar.
+ */
+enum SequenceOperator implements Operator {
+
+    SCAN(TypeCheckers.LIST_OF_STRING, TypeCheckers.EXPRS),    // scan a named data source (with optional arguments)
+    /**
+     * INSERT(target-sequence, input-records)
+     */
+    INSERT(SequenceOperator.class, SequenceOperator.class),
+    UPDATE(SequenceOperator.class, ExpressionOperator.MAP, ExpressionOperator.class),
+    UPDATE_ALL(SequenceOperator.class, ExpressionOperator.MAP),
+    DELETE(SequenceOperator.class, ExpressionOperator.class),
+    DELETE_ALL(SequenceOperator.class),
+    EMPTY(),    // emits a single, empty row
+    // evaluate the given expression and use the result as a sequence
+    EVALUATE(ExpressionOperator.class),
+    NEXT(String.class),
+
+    PROJECT(SequenceOperator.class, new TypeLiteral<List<OperatorNode<ProjectOperator>>>() {
+    }), // transform a sequence into a new schema
+    FILTER(SequenceOperator.class, ExpressionOperator.class),  // filter a sequence by an expression
+    SORT(SequenceOperator.class, new TypeLiteral<List<OperatorNode<SortOperator>>>() {
+    }),    // sort a sequence
+    PIPE(SequenceOperator.class, TypeCheckers.LIST_OF_STRING, TypeCheckers.EXPRS),    // pipe from one source through a named transformation
+    LIMIT(SequenceOperator.class, ExpressionOperator.class),
+    OFFSET(SequenceOperator.class, ExpressionOperator.class),
+    SLICE(SequenceOperator.class, ExpressionOperator.class, ExpressionOperator.class),
+    MERGE(TypeCheckers.SEQUENCES),
+    JOIN(SequenceOperator.class, SequenceOperator.class, ExpressionOperator.class),     // combine two (or more, in the case of MERGE) sequences to produce a new sequence
+    LEFT_JOIN(SequenceOperator.class, SequenceOperator.class, ExpressionOperator.class),
+
+    FALLBACK(SequenceOperator.class, SequenceOperator.class),
+
+    TIMEOUT(SequenceOperator.class, ExpressionOperator.class),
+    PAGE(SequenceOperator.class, ExpressionOperator.class),
+    ALL(),
+    MULTISOURCE(TypeCheckers.LIST_OF_LIST_OF_STRING);
+
+    private final ArgumentsTypeChecker checker;
+
+    public static Predicate<OperatorNode<? extends Operator>> IS = new Predicate<OperatorNode<? extends Operator>>() {
+        @Override
+        public boolean apply(OperatorNode<? extends Operator> input) {
+            return input.getOperator() instanceof SequenceOperator;
+        }
+    };
+
+    private SequenceOperator(Object... types) {
+        checker = TypeCheckers.make(this, types);
+    }
+
+
+    @Override
+    public void checkArguments(Object... args) {
+        checker.check(args);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/SortOperator.java b/container-search/src/main/java/com/yahoo/search/yql/SortOperator.java
new file mode 100644
index 00000000000..db03f787524
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/SortOperator.java
@@ -0,0 +1,33 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Predicate;
+
+/**
+ * Represents a sort argument. ORDER BY foo; → (ASC foo)
+ */
+enum SortOperator implements Operator {
+
+    ASC(ExpressionOperator.class),
+    DESC(ExpressionOperator.class);
+
+    private final ArgumentsTypeChecker checker;
+
+    public static Predicate<OperatorNode<? extends Operator>> IS = new Predicate<OperatorNode<? extends Operator>>() {
+        @Override
+        public boolean apply(OperatorNode<? extends Operator> input) {
+            return input.getOperator() instanceof SortOperator;
+        }
+    };
+
+    private SortOperator(Object... types) {
+        checker = TypeCheckers.make(this, types);
+    }
+
+
+    @Override
+    public void checkArguments(Object... args) {
+        checker.check(args);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/StatementOperator.java b/container-search/src/main/java/com/yahoo/search/yql/StatementOperator.java
new file mode 100644
index 00000000000..f25212e1098
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/StatementOperator.java
@@ -0,0 +1,41 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Predicate;
+import com.google.inject.TypeLiteral;
+
+import java.util.List;
+
+/**
+ * Represents program statements.
+ */
+enum StatementOperator implements Operator {
+
+    PROGRAM(new TypeLiteral<List<OperatorNode<StatementOperator>>>() {
+    }),
+    ARGUMENT(String.class, TypeOperator.class, ExpressionOperator.class),
+    DEFINE_VIEW(String.class, SequenceOperator.class),
+    EXECUTE(SequenceOperator.class, String.class),
+    OUTPUT(String.class),
+    COUNT(String.class);
+
+    private final ArgumentsTypeChecker checker;
+
+    public static Predicate<OperatorNode<? extends Operator>> IS = new Predicate<OperatorNode<? extends Operator>>() {
+        @Override
+        public boolean apply(OperatorNode<? extends Operator> input) {
+            return input.getOperator() instanceof StatementOperator;
+        }
+    };
+
+    private StatementOperator(Object... types) {
+        checker = TypeCheckers.make(this, types);
+    }
+
+
+    @Override
+    public void checkArguments(Object... args) {
+        checker.check(args);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/StringUnescaper.java b/container-search/src/main/java/com/yahoo/search/yql/StringUnescaper.java
new file mode 100644
index 00000000000..76d81429ab3
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/StringUnescaper.java
@@ -0,0 +1,123 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+class StringUnescaper {
+
+    private static boolean lookaheadOctal(String v, int point) {
+        return point < v.length() && "01234567".indexOf(v.charAt(point)) != -1;
+    }
+
+    public static String unquote(String token) {
+        if (null == token || !(token.startsWith("'") && token.endsWith("'") || token.startsWith("\"") && token.endsWith("\""))) {
+            return token;
+        }
+        // remove quotes from around string and unescape it
+        String value = token.substring(1, token.length() - 1);
+        // first quickly check to see if \ is present -- if not then there's no escaping and we're done
+        int idx = value.indexOf('\\');
+        if (idx == -1) {
+            return value;
+        }
+        // the output string will be no bigger than the input string, since escapes add characters
+        StringBuilder result = new StringBuilder(value.length());
+        int start = 0;
+        while (idx != -1) {
+            result.append(value.subSequence(start, idx));
+            start = idx + 1;
+            switch (value.charAt(start)) {
+                case 'b':
+                    result.append('\b');
+                    ++start;
+                    break;
+                case 't':
+                    result.append('\t');
+                    ++start;
+                    break;
+                case 'n':
+                    result.append('\n');
+                    ++start;
+                    break;
+                case 'f':
+                    result.append('\f');
+                    ++start;
+                    break;
+                case 'r':
+                    result.append('\r');
+                    ++start;
+                    break;
+                case '"':
+                    result.append('"');
+                    ++start;
+                    break;
+                case '\'':
+                    result.append('\'');
+                    ++start;
+                    break;
+                case '\\':
+                    result.append('\\');
+                    ++start;
+                    break;
+                case '/':
+                    result.append('/');
+                    ++start;
+                    break;
+                case 'u':
+                    // hex hex hex hex
+                    ++start;
+                    result.append((char) Integer.parseInt(value.substring(start, start + 4), 16));
+                    start += 4;
+                    break;
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                    // octal escape
+                    // 1, 2, or 3 bytes
+                    // peek ahead
+                    if (lookaheadOctal(value, start + 1) && lookaheadOctal(value, start + 2)) {
+                        result.append((char) Integer.parseInt(value.substring(start, start + 3), 8));
+                        start += 3;
+                    } else if (lookaheadOctal(value, start + 1)) {
+                        result.append((char) Integer.parseInt(value.substring(start, start + 2), 8));
+                        start += 2;
+                    } else {
+                        result.append((char) Integer.parseInt(value.substring(start, start + 1), 8));
+                        start += 1;
+                    }
+                    break;
+                default:
+                    // the lexer should be ensuring there are no malformed escapes here, so we'll just blow up
+                    throw new IllegalArgumentException("Unknown escape sequence in token: " + token);
+            }
+            idx = value.indexOf('\\', start);
+        }
+        result.append(value.subSequence(start, value.length()));
+        return result.toString();
+    }
+
+    public static String escape(String value) {
+        int idx = value.indexOf('\'');
+        if (idx == -1) {
+            return "\'" + value + "\'";
+
+        }
+        StringBuilder result = new StringBuilder(value.length() + 5);
+        result.append("'");
+        // right now we only escape ' on output
+        int start = 0;
+        while (idx != -1) {
+            result.append(value.subSequence(start, idx));
+            start = idx + 1;
+            result.append("\\'");
+            idx = value.indexOf('\\', start);
+        }
+        result.append(value.subSequence(start, value.length()));
+        result.append("'");
+        return result.toString();
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/TypeCheckers.java b/container-search/src/main/java/com/yahoo/search/yql/TypeCheckers.java
new file mode 100644
index 00000000000..32aca6d5708
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/TypeCheckers.java
@@ -0,0 +1,108 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.inject.TypeLiteral;
+
+import java.lang.reflect.ParameterizedType;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+final class TypeCheckers {
+
+    public static final TypeLiteral<List<String>> LIST_OF_STRING = new TypeLiteral<List<String>>() {
+    };
+    public static final TypeLiteral<List<List<String>>> LIST_OF_LIST_OF_STRING = new TypeLiteral<List<List<String>>>() {
+    };
+    public static final TypeLiteral<List<OperatorNode<SequenceOperator>>> SEQUENCES = new TypeLiteral<List<OperatorNode<SequenceOperator>>>() {
+    };
+    public static final TypeLiteral<List<OperatorNode<ExpressionOperator>>> EXPRS = new TypeLiteral<List<OperatorNode<ExpressionOperator>>>() {
+    };
+    public static final TypeLiteral<List<List<OperatorNode<ExpressionOperator>>>> LIST_OF_EXPRS = new TypeLiteral<List<List<OperatorNode<ExpressionOperator>>>>() {
+    };
+    public static final ImmutableSet<Class<?>> LITERAL_TYPES = ImmutableSet.<Class<?>>builder()
+            .add(String.class)
+            .add(Integer.class)
+            .add(Double.class)
+            .add(Boolean.class)
+            .add(Float.class)
+            .add(Byte.class)
+            .add(Long.class)
+            .add(List.class)
+            .add(Map.class)
+            .build();
+
+    private TypeCheckers() {
+    }
+
+    public static ArgumentsTypeChecker make(Operator target, Object... types) {
+        // Class<?> extends Operator -> NodeTypeChecker
+        if (types == null) {
+            types = new Object[0];
+        }
+        List<OperatorTypeChecker> checkers = Lists.newArrayListWithCapacity(types.length);
+        for (int i = 0; i < types.length; ++i) {
+            checkers.add(createChecker(target, i, types[i]));
+        }
+        return new ArgumentsTypeChecker(target, checkers);
+    }
+
+    // this is festooned with instance checkes before all the casting
+    @SuppressWarnings("unchecked")
+    private static OperatorTypeChecker createChecker(Operator parent, int idx, Object value) {
+        if (value instanceof TypeLiteral) {
+            TypeLiteral<?> lit = (TypeLiteral<?>) value;
+            Class<?> raw = lit.getRawType();
+            if (List.class.isAssignableFrom(raw)) {
+                Preconditions.checkArgument(lit.getType() instanceof ParameterizedType, "TypeLiteral without a ParameterizedType for List");
+                ParameterizedType type = (ParameterizedType) lit.getType();
+                TypeLiteral<?> arg = TypeLiteral.get(type.getActualTypeArguments()[0]);
+                if (OperatorNode.class.isAssignableFrom(arg.getRawType())) {
+                    Preconditions.checkArgument(arg.getType() instanceof ParameterizedType, "Type spec must be List<OperatorNode<?>>");
+                    Class<? extends Operator> optype = (Class<? extends Operator>) TypeLiteral.get(((ParameterizedType) arg.getType()).getActualTypeArguments()[0]).getRawType();
+                    return new OperatorNodeListTypeChecker(parent, idx, optype, ImmutableSet.<Operator>of());
+                } else {
+                    return new JavaListTypeChecker(parent, idx, arg.getRawType());
+                }
+            }
+            throw new IllegalArgumentException("don't know how to handle TypeLiteral " + value);
+        }
+        if (value instanceof Class) {
+            Class<?> clazz = (Class<?>) value;
+            if (Operator.class.isAssignableFrom(clazz)) {
+                return new NodeTypeChecker(parent, idx, (Class<? extends Operator>) clazz, ImmutableSet.<Operator>of());
+            } else {
+                return new JavaTypeChecker(parent, idx, clazz);
+            }
+        } else if (value instanceof Operator) {
+            Operator operator = (Operator) value;
+            Class<? extends Operator> clazz = operator.getClass();
+            Set<? extends Operator> allowed;
+            if (Enum.class.isInstance(value)) {
+                Class<? extends Enum> enumClazz = (Class<? extends Enum>) clazz;
+                allowed = (Set<? extends Operator>) EnumSet.of(enumClazz.cast(value));
+            } else {
+                allowed = ImmutableSet.of(operator);
+            }
+            return new NodeTypeChecker(parent, idx, clazz, allowed);
+        } else if (value instanceof EnumSet) {
+            EnumSet<?> v = (EnumSet<?>) value;
+            Enum elt = Iterables.get(v, 0);
+            if (elt instanceof Operator) {
+                Class<? extends Operator> opclass = (Class<? extends Operator>) elt.getClass();
+                Set<? extends Operator> allowed = (Set<? extends Operator>) v;
+                return new NodeTypeChecker(parent, idx, opclass, allowed);
+            }
+        } else if (value instanceof Set) {
+            // Set<Class<?>>
+            return new JavaUnionTypeChecker(parent, idx, (Set<Class<?>>) value);
+        }
+        throw new IllegalArgumentException("I don't know how to create a checker from " + value);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/TypeOperator.java b/container-search/src/main/java/com/yahoo/search/yql/TypeOperator.java
new file mode 100644
index 00000000000..01b1f88cc5e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/TypeOperator.java
@@ -0,0 +1,37 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.google.common.base.Predicate;
+
+enum TypeOperator implements Operator {
+
+    BYTE,
+    INT16,
+    INT32,
+    INT64,
+    STRING,
+    DOUBLE,
+    TIMESTAMP,
+    BOOLEAN,
+    ARRAY(TypeOperator.class),
+    MAP(TypeOperator.class);
+
+    private final ArgumentsTypeChecker checker;
+
+    public static Predicate<OperatorNode<? extends Operator>> IS = new Predicate<OperatorNode<? extends Operator>>() {
+        @Override
+        public boolean apply(OperatorNode<? extends Operator> input) {
+            return input.getOperator() instanceof TypeOperator;
+        }
+    };
+
+    TypeOperator(Object... types) {
+        checker = TypeCheckers.make(this, types);
+    }
+
+    @Override
+    public void checkArguments(Object... args) {
+        checker.check(args);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaGroupingStep.java b/container-search/src/main/java/com/yahoo/search/yql/VespaGroupingStep.java
new file mode 100644
index 00000000000..520728dc231
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/VespaGroupingStep.java
@@ -0,0 +1,29 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.grouping.request.GroupingOperation;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
+ */
+public class VespaGroupingStep {
+
+    private final GroupingOperation operation;
+    private final List<Continuation> continuations = new ArrayList<>();
+
+    public VespaGroupingStep(GroupingOperation operation) {
+        this.operation = operation;
+    }
+
+    public GroupingOperation getOperation() {
+        return operation;
+    }
+
+    public List<Continuation> continuations() {
+        return continuations;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
new file mode 100644
index 00000000000..397225a087c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
@@ -0,0 +1,1381 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import static com.yahoo.search.yql.YqlParser.ACCENT_DROP;
+import static com.yahoo.search.yql.YqlParser.ALTERNATIVES;
+import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING;
+import static com.yahoo.search.yql.YqlParser.BOUNDS;
+import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN;
+import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN;
+import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN;
+import static com.yahoo.search.yql.YqlParser.CONNECTION_ID;
+import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT;
+import static com.yahoo.search.yql.YqlParser.CONNECTIVITY;
+import static com.yahoo.search.yql.YqlParser.DISTANCE;
+import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT;
+import static com.yahoo.search.yql.YqlParser.EQUIV;
+import static com.yahoo.search.yql.YqlParser.FILTER;
+import static com.yahoo.search.yql.YqlParser.HIT_LIMIT;
+import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS;
+import static com.yahoo.search.yql.YqlParser.LABEL;
+import static com.yahoo.search.yql.YqlParser.NEAR;
+import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE;
+import static com.yahoo.search.yql.YqlParser.ONEAR;
+import static com.yahoo.search.yql.YqlParser.ORIGIN;
+import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH;
+import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET;
+import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL;
+import static com.yahoo.search.yql.YqlParser.PHRASE;
+import static com.yahoo.search.yql.YqlParser.PREFIX;
+import static com.yahoo.search.yql.YqlParser.RANGE;
+import static com.yahoo.search.yql.YqlParser.RANK;
+import static com.yahoo.search.yql.YqlParser.RANKED;
+import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD;
+import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE;
+import static com.yahoo.search.yql.YqlParser.STEM;
+import static com.yahoo.search.yql.YqlParser.SUBSTRING;
+import static com.yahoo.search.yql.YqlParser.SUFFIX;
+import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS;
+import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR;
+import static com.yahoo.search.yql.YqlParser.UNIQUE_ID;
+import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA;
+import static com.yahoo.search.yql.YqlParser.WAND;
+import static com.yahoo.search.yql.YqlParser.WEAK_AND;
+import static com.yahoo.search.yql.YqlParser.WEIGHT;
+import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.AndSegmentItem;
+import com.yahoo.prelude.query.DotProductItem;
+import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.IndexedItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.MarkerWordItem;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.prelude.query.NotItem;
+import com.yahoo.prelude.query.NullItem;
+import com.yahoo.prelude.query.ONearItem;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PhraseSegmentItem;
+import com.yahoo.prelude.query.PredicateQueryItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.RangeItem;
+import com.yahoo.prelude.query.RankItem;
+import com.yahoo.prelude.query.RegExpItem;
+import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.SubstringItem;
+import com.yahoo.prelude.query.SuffixItem;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.prelude.query.ToolBox;
+import com.yahoo.prelude.query.ToolBox.QueryVisitor;
+import com.yahoo.prelude.query.WandItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WeightedSetItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.Query;
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.grouping.GroupingRequest;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * Serialize Vespa query trees to YQL+ strings.
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ */
+public class VespaSerializer {
+    // TODO refactor, too much copy/paste
+
+    private static class AndSegmentSerializer extends Serializer {
+        private static void serializeWords(StringBuilder destination,
+                AndSegmentItem segment) {
+            for (int i = 0; i < segment.getItemCount(); ++i) {
+                if (i > 0) {
+                    destination.append(", ");
+                }
+                Item current = segment.getItem(i);
+                if (current instanceof WordItem) {
+                    destination.append('"');
+                    escape(((WordItem) current).getIndexedString(), destination)
+                            .append('"');
+                } else {
+                    throw new IllegalArgumentException(
+                            "Serializing of "
+                                    + current.getClass().getSimpleName()
+                                    + " in segment AND expressions not implemented, please report this as a bug.");
+                }
+            }
+        }
+
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            return serialize(destination, item, true);
+        }
+
+        static boolean serialize(StringBuilder destination, Item item,
+                boolean includeField) {
+            AndSegmentItem phrase = (AndSegmentItem) item;
+            Substring origin = phrase.getOrigin();
+            String image;
+            int offset;
+            int length;
+
+            if (origin == null) {
+                image = phrase.getRawWord();
+                offset = 0;
+                length = image.length();
+            } else {
+                image = origin.getSuperstring();
+                offset = origin.start;
+                length = origin.end - origin.start;
+            }
+
+            if (includeField) {
+                destination.append(normalizeIndexName(phrase.getIndexName()))
+                        .append(" contains ");
+            }
+            destination.append("([{");
+            serializeOrigin(destination, image, offset, length);
+            destination.append(", \"").append(AND_SEGMENTING)
+                    .append("\": true");
+            destination.append("}]");
+            destination.append(PHRASE).append('(');
+            serializeWords(destination, phrase);
+            destination.append("))");
+            return false;
+        }
+    }
+
+    private static class AndSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+            destination.append(')');
+        }
+
+        @Override
+        String separator(Deque<SerializerWrapper> state) {
+            return " AND ";
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            destination.append("(");
+            return true;
+        }
+    }
+
+    private static class DotProductSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            serializeWeightedSetContents(destination, DOT_PRODUCT,
+                    (WeightedSetItem) item);
+            return false;
+        }
+
+    }
+
+    private static class EquivSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            EquivItem e = (EquivItem) item;
+            String annotations = leafAnnotations(e);
+            destination.append(getIndexName(e.getItem(0))).append(" contains ");
+            if (annotations.length() > 0) {
+                destination.append("([{").append(annotations).append("}]");
+            }
+            destination.append(EQUIV).append('(');
+            int initLen = destination.length();
+            for (Iterator<Item> i = e.getItemIterator(); i.hasNext();) {
+                Item x = i.next();
+                if (destination.length() > initLen) {
+                    destination.append(", ");
+                }
+                if (x instanceof PhraseItem) {
+                    PhraseSerializer.serialize(destination, x, false);
+                } else {
+                    destination.append('"');
+                    escape(((IndexedItem) x).getIndexedString(), destination);
+                    destination.append('"');
+                }
+            }
+            if (annotations.length() > 0) {
+                destination.append(')');
+            }
+            destination.append(')');
+            return false;
+        }
+
+    }
+
+    private static class NearSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            NearItem n = (NearItem) item;
+            String annotations = nearAnnotations(n);
+
+            destination.append(getIndexName(n.getItem(0))).append(" contains ");
+            if (annotations.length() > 0) {
+                destination.append('(').append(annotations);
+            }
+            destination.append(NEAR).append('(');
+            int initLen = destination.length();
+            for (ListIterator<Item> i = n.getItemIterator(); i.hasNext();) {
+                WordItem close = (WordItem) i.next();
+                if (destination.length() > initLen) {
+                    destination.append(", ");
+                }
+                destination.append('"');
+                escape(close.getIndexedString(), destination).append('"');
+            }
+            destination.append(')');
+            if (annotations.length() > 0) {
+                destination.append(')');
+            }
+            return false;
+        }
+
+        static String nearAnnotations(NearItem n) {
+            if (n.getDistance() != NearItem.defaultDistance) {
+                return "[{\"" + DISTANCE + "\": " + n.getDistance() + "}]";
+            } else {
+                return "";
+            }
+        }
+
+    }
+
+    private static class NotSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+            destination.append(')');
+        }
+
+        @Override
+        String separator(Deque<SerializerWrapper> state) {
+            if (state.peekFirst().subItems == 1) {
+                return ") AND !(";
+            } else {
+                return " OR ";
+            }
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            destination.append("(");
+            return true;
+        }
+    }
+
+    private static class NullSerializer extends Serializer {
+
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            throw new NullItemException(
+                    "NullItem encountered in query tree."
+                            + " This is usually a symptom of an invalid query or an error"
+                            + " in a query transformer.");
+        }
+    }
+
+    private static class NumberSerializer extends Serializer {
+
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            IntItem intItem = (IntItem) item;
+            if (intItem.getFromLimit().number()
+                    .equals(intItem.getToLimit().number())) {
+                destination.append(normalizeIndexName(intItem.getIndexName()))
+                        .append(" = ");
+                annotatedNumberImage(intItem, intItem.getFromLimit().number()
+                        .toString(), destination);
+            } else if (intItem.getFromLimit().isInfinite()) {
+                destination.append(normalizeIndexName(intItem.getIndexName()));
+                destination.append(intItem.getToLimit().isInclusive() ? " <= "
+                        : " < ");
+                annotatedNumberImage(intItem, intItem.getToLimit().number()
+                        .toString(), destination);
+            } else if (intItem.getToLimit().isInfinite()) {
+                destination.append(normalizeIndexName(intItem.getIndexName()));
+                destination
+                        .append(intItem.getFromLimit().isInclusive() ? " >= "
+                                : " > ");
+                annotatedNumberImage(intItem, intItem.getFromLimit().number()
+                        .toString(), destination);
+            } else {
+                serializeAsRange(destination, intItem);
+            }
+            return false;
+        }
+
+        private void serializeAsRange(StringBuilder destination, IntItem intItem) {
+            String annotations = leafAnnotations(intItem);
+            boolean leftOpen = !intItem.getFromLimit().isInclusive();
+            boolean rightOpen = !intItem.getToLimit().isInclusive();
+            String boundsAnnotation = "";
+            int initLen;
+
+            if (leftOpen && rightOpen) {
+                boundsAnnotation = "\"" + BOUNDS + "\": " + "\"" + BOUNDS_OPEN
+                        + "\"";
+            } else if (leftOpen) {
+                boundsAnnotation = "\"" + BOUNDS + "\": " + "\""
+                        + BOUNDS_LEFT_OPEN + "\"";
+            } else if (rightOpen) {
+                boundsAnnotation = "\"" + BOUNDS + "\": " + "\""
+                        + BOUNDS_RIGHT_OPEN + "\"";
+            }
+            if (annotations.length() > 0 || boundsAnnotation.length() > 0) {
+                destination.append("[{");
+            }
+            initLen = destination.length();
+            if (annotations.length() > 0) {
+
+                destination.append(annotations);
+            }
+            comma(destination, initLen);
+            if (boundsAnnotation.length() > 0) {
+                destination.append(boundsAnnotation);
+            }
+            if (initLen != annotations.length()) {
+                destination.append("}]");
+            }
+            destination.append(RANGE).append('(')
+                    .append(normalizeIndexName(intItem.getIndexName()))
+                    .append(", ").append(intItem.getFromLimit().number())
+                    .append(", ").append(intItem.getToLimit().number())
+                    .append(")");
+        }
+
+        private void annotatedNumberImage(IntItem item, String rawNumber,
+                StringBuilder image) {
+            String annotations = leafAnnotations(item);
+
+            if (annotations.length() > 0) {
+                image.append("([{").append(annotations).append("}]");
+            }
+            if ('-' == rawNumber.charAt(0)) {
+                image.append('(');
+            }
+            image.append(rawNumber);
+            appendLongIfNecessary(rawNumber, image);
+            if ('-' == rawNumber.charAt(0)) {
+                image.append(')');
+            }
+            if (annotations.length() > 0) {
+                image.append(')');
+            }
+        }
+
+        private void appendLongIfNecessary(String rawNumber, StringBuilder image) {
+            // floating point
+            if (rawNumber.indexOf('.') >= 0) {
+                return;
+            }
+            try {
+                long l = Long.parseLong(rawNumber);
+                if (l < Integer.MIN_VALUE || l > Integer.MAX_VALUE) {
+                    image.append('L');
+                }
+            } catch (NumberFormatException e) {
+                // somebody has managed to init an IntItem containing noise,
+                // just give up
+                return;
+            }
+        }
+    }
+
+    private static class RegExpSerializer extends Serializer {
+
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            RegExpItem regexp = (RegExpItem) item;
+
+            String annotations = leafAnnotations(regexp);
+            destination.append(normalizeIndexName(regexp.getIndexName())).append(
+                    " matches ");
+            annotatedTerm(destination, regexp, annotations);
+            return false;
+        }
+    }
+
+    private static class ONearSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            NearItem n = (NearItem) item;
+            String annotations = NearSerializer.nearAnnotations(n);
+
+            destination.append(getIndexName(n.getItem(0))).append(" contains ");
+            if (annotations.length() > 0) {
+                destination.append('(').append(annotations);
+            }
+            destination.append(ONEAR).append('(');
+            int initLen = destination.length();
+            for (ListIterator<Item> i = n.getItemIterator(); i.hasNext();) {
+                WordItem close = (WordItem) i.next();
+                if (destination.length() > initLen) {
+                    destination.append(", ");
+                }
+                destination.append('"');
+                escape(close.getIndexedString(), destination).append('"');
+            }
+            destination.append(')');
+            if (annotations.length() > 0) {
+                destination.append(')');
+            }
+            return false;
+        }
+
+    }
+
+    private static class OrSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+            destination.append(')');
+        }
+
+        @Override
+        String separator(Deque<SerializerWrapper> state) {
+            return " OR ";
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            destination.append("(");
+            return true;
+        }
+    }
+
+    private static class PhraseSegmentSerializer extends Serializer {
+
+        private static void serializeWords(StringBuilder destination,
+                PhraseSegmentItem segment) {
+            for (int i = 0; i < segment.getItemCount(); ++i) {
+                if (i > 0) {
+                    destination.append(", ");
+                }
+                Item current = segment.getItem(i);
+                if (current instanceof WordItem) {
+                    destination.append('"');
+                    escape(((WordItem) current).getIndexedString(), destination)
+                            .append('"');
+                } else {
+                    throw new IllegalArgumentException(
+                            "Serializing of "
+                                    + current.getClass().getSimpleName()
+                                    + " in phrases not implemented, please report this as a bug.");
+                }
+            }
+        }
+
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            return serialize(destination, item, true);
+        }
+
+        static boolean serialize(StringBuilder destination, Item item,
+                boolean includeField) {
+            PhraseSegmentItem phrase = (PhraseSegmentItem) item;
+            Substring origin = phrase.getOrigin();
+            String image;
+            int offset;
+            int length;
+
+            if (includeField) {
+                destination.append(normalizeIndexName(phrase.getIndexName()))
+                        .append(" contains ");
+            }
+            if (origin == null) {
+                image = phrase.getRawWord();
+                offset = 0;
+                length = image.length();
+            } else {
+                image = origin.getSuperstring();
+                offset = origin.start;
+                length = origin.end - origin.start;
+            }
+
+            destination.append("([{");
+            serializeOrigin(destination, image, offset, length);
+            String annotations = leafAnnotations(phrase);
+            if (annotations.length() > 0) {
+                destination.append(", ").append(annotations);
+            }
+            if (phrase.getSegmentingRule() == SegmentingRule.BOOLEAN_AND) {
+                destination.append(", ").append('"').append(AND_SEGMENTING)
+                        .append("\": true");
+            }
+            destination.append("}]");
+            destination.append(PHRASE).append('(');
+            serializeWords(destination, phrase);
+            destination.append("))");
+            return false;
+        }
+    }
+
+    private static class PhraseSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            return serialize(destination, item, true);
+        }
+
+        static boolean serialize(StringBuilder destination, Item item,
+                boolean includeField) {
+
+            PhraseItem phrase = (PhraseItem) item;
+            String annotations = leafAnnotations(phrase);
+
+            if (includeField) {
+                destination.append(normalizeIndexName(phrase.getIndexName()))
+                        .append(" contains ");
+
+            }
+            if (annotations.length() > 0) {
+                destination.append("([{").append(annotations).append("}]");
+            }
+
+            destination.append(PHRASE).append('(');
+            for (int i = 0; i < phrase.getItemCount(); ++i) {
+                if (i > 0) {
+                    destination.append(", ");
+                }
+                Item current = phrase.getItem(i);
+                if (current instanceof WordItem) {
+                    WordSerializer.serializeWordWithoutIndex(destination,
+                            current);
+                } else if (current instanceof PhraseSegmentItem) {
+                    PhraseSegmentSerializer.serialize(destination, current,
+                            false);
+                } else if (current instanceof WordAlternativesItem) {
+                    WordAlternativesSerializer.serialize(destination, (WordAlternativesItem) current, false);
+                } else {
+                    throw new IllegalArgumentException(
+                            "Serializing of "
+                                    + current.getClass().getSimpleName()
+                                    + " in phrases not implemented, please report this as a bug.");
+                }
+            }
+            destination.append(')');
+            if (annotations.length() > 0) {
+                destination.append(')');
+            }
+            return false;
+        }
+
+    }
+
+    private static class PredicateQuerySerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            PredicateQueryItem pItem = (PredicateQueryItem) item;
+            destination.append("predicate(").append(pItem.getIndexName())
+                    .append(',');
+            appendFeatures(destination, pItem.getFeatures());
+            destination.append(',');
+            appendFeatures(destination, pItem.getRangeFeatures());
+            destination.append(')');
+            return false;
+        }
+
+        private void appendFeatures(StringBuilder destination,
+                Collection<? extends PredicateQueryItem.EntryBase> features) {
+            if (features.isEmpty()) {
+                destination.append('0'); // Workaround for empty maps.
+                return;
+            }
+            destination.append('{');
+            boolean first = true;
+            for (PredicateQueryItem.EntryBase entry : features) {
+                if (!first) {
+                    destination.append(',');
+                }
+                if (entry.getSubQueryBitmap() != PredicateQueryItem.ALL_SUB_QUERIES) {
+                    destination.append("\"0x").append(
+                            Long.toHexString(entry.getSubQueryBitmap()));
+                    destination.append("\":{");
+                    appendKeyValue(destination, entry);
+                    destination.append('}');
+                } else {
+                    appendKeyValue(destination, entry);
+                }
+                first = false;
+            }
+            destination.append('}');
+        }
+
+        private void appendKeyValue(StringBuilder destination,
+                PredicateQueryItem.EntryBase entry) {
+            destination.append('"');
+            escape(entry.getKey(), destination);
+            destination.append("\":");
+            if (entry instanceof PredicateQueryItem.Entry) {
+                destination.append('"');
+                escape(((PredicateQueryItem.Entry) entry).getValue(),
+                        destination);
+                destination.append('"');
+            } else {
+                destination.append(((PredicateQueryItem.RangeEntry) entry)
+                        .getValue());
+                destination.append('L');
+            }
+        }
+
+    }
+
+    private static class RangeSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            RangeItem range = (RangeItem) item;
+            String annotations = leafAnnotations(range);
+            if (annotations.length() > 0) {
+                destination.append("[{").append(annotations).append("}]");
+            }
+            destination.append(RANGE).append('(')
+                    .append(normalizeIndexName(range.getIndexName()))
+                    .append(", ");
+            appendNumberImage(destination, range.getFrom()); // TODO: Serialize
+                                                             // inclusive/exclusive
+            destination.append(", ");
+            appendNumberImage(destination, range.getTo());
+            destination.append(')');
+            return false;
+        }
+
+        private void appendNumberImage(StringBuilder destination, Number number) {
+            destination.append(number.toString());
+            if (number instanceof Long) {
+                destination.append('L');
+            }
+        }
+    }
+
+    private static class RankSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+            destination.append(')');
+        }
+
+        @Override
+        String separator(Deque<SerializerWrapper> state) {
+            return ", ";
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            destination.append(RANK).append('(');
+            return true;
+
+        }
+
+    }
+
+    private static class WordAlternativesSerializer extends Serializer {
+
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            return serialize(destination, (WordAlternativesItem) item, true);
+        }
+
+        static boolean serialize(StringBuilder destination, WordAlternativesItem alternatives, boolean includeField) {
+            String annotations = leafAnnotations(alternatives);
+            Substring origin = alternatives.getOrigin();
+            boolean isFromQuery = alternatives.isFromQuery();
+            boolean needsAnnotations = annotations.length() > 0 || origin != null || !isFromQuery;
+
+            if (includeField) {
+                destination.append(normalizeIndexName(alternatives.getIndexName())).append(" contains ");
+            }
+
+            if (needsAnnotations) {
+                destination.append("([{");
+                int initLen = destination.length();
+
+                if (origin != null) {
+                    String image = origin.getSuperstring();
+                    int offset = origin.start;
+                    int length = origin.end - origin.start;
+                    serializeOrigin(destination, image, offset, length);
+                }
+                if (!isFromQuery) {
+                    comma(destination, initLen);
+                    destination.append('"').append(IMPLICIT_TRANSFORMS).append("\": false");
+                }
+                if (annotations.length() > 0) {
+                    comma(destination, initLen);
+                    destination.append(annotations);
+                }
+
+                destination.append("}]");
+            }
+
+            destination.append(ALTERNATIVES).append("({");
+            int initLen = destination.length();
+            List<WordAlternativesItem.Alternative> sortedAlternatives = new ArrayList<>(alternatives.getAlternatives());
+            // ensure most precise forms first
+            Collections.sort(sortedAlternatives, (x, y) -> Double.compare(y.exactness, x.exactness));
+            for (WordAlternativesItem.Alternative alternative : sortedAlternatives) {
+                comma(destination, initLen);
+                destination.append('"');
+                escape(alternative.word, destination);
+                destination.append("\": ").append(Double.toString(alternative.exactness));
+            }
+            destination.append("})");
+            if (needsAnnotations) {
+                destination.append(')');
+            }
+            return false;
+        }
+    }
+
+    private static abstract class Serializer {
+        abstract void onExit(StringBuilder destination, Item item);
+
+        String separator(Deque<SerializerWrapper> state) {
+            throw new UnsupportedOperationException(
+                    "Having several items for this query operator serializer, "
+                            + this.getClass().getSimpleName()
+                            + ", not yet implemented.");
+        }
+
+        abstract boolean serialize(StringBuilder destination, Item item);
+    }
+
+    private static final class SerializerWrapper {
+        int subItems;
+        final Serializer type;
+        final Item item;
+
+        SerializerWrapper(Serializer type, Item item) {
+            subItems = 0;
+            this.type = type;
+            this.item = item;
+        }
+
+    }
+
+    private static final class TokenComparator implements
+            Comparator<Entry<Object, Integer>> {
+
+        @SuppressWarnings({ "rawtypes", "unchecked" })
+        @Override
+        public int compare(Entry<Object, Integer> o1, Entry<Object, Integer> o2) {
+            Comparable c1 = (Comparable) o1.getKey();
+            Comparable c2 = (Comparable) o2.getKey();
+            return c1.compareTo(c2);
+        }
+    }
+
+    private static class VespaVisitor extends QueryVisitor {
+
+        final StringBuilder destination;
+        final Deque<SerializerWrapper> state = new ArrayDeque<>();
+
+        VespaVisitor(StringBuilder destination) {
+            this.destination = destination;
+        }
+
+        @Override
+        public void onExit() {
+            SerializerWrapper w = state.removeFirst();
+            w.type.onExit(destination, w.item);
+            w = state.peekFirst();
+            if (w != null) {
+                w.subItems += 1;
+            }
+        }
+
+        @Override
+        public boolean visit(Item item) {
+            Serializer doIt = dispatch.get(item.getClass());
+
+            if (doIt == null) {
+                throw new IllegalArgumentException(item.getClass()
+                        + " not supported for YQL+ marshalling.");
+            }
+
+            if (state.peekFirst() != null && state.peekFirst().subItems > 0) {
+                destination.append(state.peekFirst().type.separator(state));
+            }
+            state.addFirst(new SerializerWrapper(doIt, item));
+            return doIt.serialize(destination, item);
+
+        }
+    }
+
+    private static class WandSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            serializeWeightedSetContents(destination, WAND,
+                    (WeightedSetItem) item,
+                    specificAnnotations((WandItem) item));
+            return false;
+        }
+
+        private String specificAnnotations(WandItem w) {
+            StringBuilder annotations = new StringBuilder();
+            int targetNumHits = w.getTargetNumHits();
+            double scoreThreshold = w.getScoreThreshold();
+            double thresholdBoostFactor = w.getThresholdBoostFactor();
+            if (targetNumHits != 10) {
+                annotations.append('"').append(TARGET_NUM_HITS).append("\": ")
+                        .append(targetNumHits);
+            }
+            if (scoreThreshold != 0) {
+                comma(annotations, 0);
+                annotations.append('"').append(SCORE_THRESHOLD).append("\": ")
+                        .append(scoreThreshold);
+            }
+            if (thresholdBoostFactor != 1) {
+                comma(annotations, 0);
+                annotations.append('"').append(THRESHOLD_BOOST_FACTOR)
+                        .append("\": ").append(thresholdBoostFactor);
+            }
+            return annotations.toString();
+        }
+
+    }
+
+    private static class WeakAndSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+            destination.append(')');
+            if (needsAnnotationBlock((WeakAndItem) item)) {
+                destination.append(')');
+            }
+        }
+
+        @Override
+        String separator(Deque<SerializerWrapper> state) {
+            return ", ";
+        }
+
+        private boolean needsAnnotationBlock(WeakAndItem item) {
+            return nonDefaultScoreThreshold(item) || nonDefaultTargetNumHits(item);
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            WeakAndItem w = (WeakAndItem) item;
+            if (needsAnnotationBlock(w)) {
+                destination.append("([{");
+            }
+            int lengthBeforeAnnotations = destination.length();
+            if (nonDefaultTargetNumHits(w)) {
+                destination.append('"').append(TARGET_NUM_HITS).append("\": ").append(w.getN());
+            }
+            if (nonDefaultScoreThreshold(w)) {
+                comma(destination, lengthBeforeAnnotations);
+                destination.append('"').append(SCORE_THRESHOLD).append("\": ").append(w.getScoreThreshold());
+            }
+            if (needsAnnotationBlock(w)) {
+                destination.append("}]");
+            }
+            destination.append(WEAK_AND).append('(');
+            return true;
+        }
+
+        private boolean nonDefaultScoreThreshold(WeakAndItem w) {
+            return w.getScoreThreshold() > 0;
+        }
+
+        private boolean nonDefaultTargetNumHits(WeakAndItem w) {
+            return w.getN() != WeakAndItem.defaultN;
+        }
+    }
+
+    private static class WeightedSetSerializer extends Serializer {
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            serializeWeightedSetContents(destination, WEIGHTED_SET,
+                    (WeightedSetItem) item);
+            return false;
+        }
+
+    }
+
+    private static class WordSerializer extends Serializer {
+
+        @Override
+        void onExit(StringBuilder destination, Item item) {
+        }
+
+        @Override
+        boolean serialize(StringBuilder destination, Item item) {
+            WordItem w = (WordItem) item;
+            StringBuilder wordAnnotations = getAllAnnotations(w);
+
+            destination.append(normalizeIndexName(w.getIndexName())).append(
+                    " contains ");
+            VespaSerializer.annotatedTerm(destination, w, wordAnnotations.toString());
+            return false;
+        }
+
+        static void serializeWordWithoutIndex(StringBuilder destination,
+                Item item) {
+            WordItem w = (WordItem) item;
+            StringBuilder wordAnnotations = getAllAnnotations(w);
+
+            VespaSerializer.annotatedTerm(destination, w, wordAnnotations.toString());
+        }
+
+        private static StringBuilder getAllAnnotations(WordItem w) {
+            StringBuilder wordAnnotations = new StringBuilder(
+                    WordSerializer.wordAnnotations(w));
+            String leafAnnotations = leafAnnotations(w);
+
+            if (leafAnnotations.length() > 0) {
+                comma(wordAnnotations, 0);
+                wordAnnotations.append(leafAnnotations(w));
+            }
+            return wordAnnotations;
+        }
+
+        private static String wordAnnotations(WordItem item) {
+            Substring origin = item.getOrigin();
+            boolean usePositionData = item.usePositionData();
+            boolean stemmed = item.isStemmed();
+            boolean lowercased = item.isLowercased();
+            boolean accentDrop = item.isNormalizable();
+            SegmentingRule andSegmenting = item.getSegmentingRule();
+            boolean isFromQuery = item.isFromQuery();
+            StringBuilder annotation = new StringBuilder();
+            boolean prefix = item instanceof PrefixItem;
+            boolean suffix = item instanceof SuffixItem;
+            boolean substring = item instanceof SubstringItem;
+            int initLen = annotation.length();
+            String image;
+            int offset;
+            int length;
+
+            if (origin == null) {
+                image = item.getRawWord();
+                offset = 0;
+                length = image.length();
+            } else {
+                image = origin.getSuperstring();
+                offset = origin.start;
+                length = origin.end - origin.start;
+            }
+
+            if (!image.substring(offset, offset + length).equals(
+                    item.getIndexedString())) {
+                VespaSerializer.serializeOrigin(annotation, image, offset,
+                        length);
+            }
+            if (usePositionData != true) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(USE_POSITION_DATA)
+                        .append("\": false");
+            }
+            if (stemmed == true) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(STEM).append("\": false");
+            }
+            if (lowercased == true) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(NORMALIZE_CASE)
+                        .append("\": false");
+            }
+            if (accentDrop == false) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(ACCENT_DROP).append("\": false");
+            }
+            if (andSegmenting == SegmentingRule.BOOLEAN_AND) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(AND_SEGMENTING)
+                        .append("\": true");
+            }
+            if (!isFromQuery) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(IMPLICIT_TRANSFORMS)
+                        .append("\": false");
+            }
+            if (prefix) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(PREFIX).append("\": true");
+            }
+            if (suffix) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(SUFFIX).append("\": true");
+            }
+            if (substring) {
+                VespaSerializer.comma(annotation, initLen);
+                annotation.append('"').append(SUBSTRING).append("\": true");
+            }
+            return annotation.toString();
+        }
+
+    }
+
+    private static final char[] DIGITS = new char[] { '0', '1', '2', '3', '4',
+            '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
+
+    private static final Map<Class<?>, Serializer> dispatch;
+
+    private static final Comparator<? super Entry<Object, Integer>> tokenComparator = new TokenComparator();
+
+    static {
+        Map<Class<?>, Serializer> dispatchBuilder = new HashMap<>();
+        dispatchBuilder.put(AndItem.class, new AndSerializer());
+        dispatchBuilder.put(AndSegmentItem.class, new AndSegmentSerializer());
+        dispatchBuilder.put(DotProductItem.class, new DotProductSerializer());
+        dispatchBuilder.put(EquivItem.class, new EquivSerializer());
+        dispatchBuilder.put(IntItem.class, new NumberSerializer());
+        dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha
+        dispatchBuilder.put(NearItem.class, new NearSerializer());
+        dispatchBuilder.put(NotItem.class, new NotSerializer());
+        dispatchBuilder.put(NullItem.class, new NullSerializer());
+        dispatchBuilder.put(ONearItem.class, new ONearSerializer());
+        dispatchBuilder.put(OrItem.class, new OrSerializer());
+        dispatchBuilder.put(PhraseItem.class, new PhraseSerializer());
+        dispatchBuilder.put(PhraseSegmentItem.class, new PhraseSegmentSerializer());
+        dispatchBuilder.put(PredicateQueryItem.class,
+                new PredicateQuerySerializer());
+        dispatchBuilder.put(PrefixItem.class, new WordSerializer()); // gotcha
+        dispatchBuilder.put(WordAlternativesItem.class, new WordAlternativesSerializer());
+        dispatchBuilder.put(RangeItem.class, new RangeSerializer());
+        dispatchBuilder.put(RankItem.class, new RankSerializer());
+        dispatchBuilder.put(SubstringItem.class, new WordSerializer()); // gotcha
+        dispatchBuilder.put(SuffixItem.class, new WordSerializer()); // gotcha
+        dispatchBuilder.put(WandItem.class, new WandSerializer());
+        dispatchBuilder.put(WeakAndItem.class, new WeakAndSerializer());
+        dispatchBuilder.put(WeightedSetItem.class, new WeightedSetSerializer());
+        dispatchBuilder.put(WordItem.class, new WordSerializer());
+        dispatchBuilder.put(RegExpItem.class, new RegExpSerializer());
+        dispatch = ImmutableMap.copyOf(dispatchBuilder);
+    }
+
+    /**
+     * Do YQL+ escaping, which is basically the same as for JSON, of the
+     * incoming string to the "quoted" buffer. The buffer returned is the same
+     * as the one given in the "quoted" parameter.
+     *
+     * @param in a string to escape
+     * @param escaped the target buffer for escaped data
+     * @return the same buffer as given in the "quoted" parameter
+     */
+    private static StringBuilder escape(String in, StringBuilder escaped) {
+        for (char c : in.toCharArray()) {
+            switch (c) {
+            case ('\b'):
+                escaped.append("\\b");
+                break;
+            case ('\t'):
+                escaped.append("\\t");
+                break;
+            case ('\n'):
+                escaped.append("\\n");
+                break;
+            case ('\f'):
+                escaped.append("\\f");
+                break;
+            case ('\r'):
+                escaped.append("\\r");
+                break;
+            case ('"'):
+                escaped.append("\\\"");
+                break;
+            case ('\''):
+                escaped.append("\\'");
+                break;
+            case ('\\'):
+                escaped.append("\\\\");
+                break;
+            case ('/'):
+                escaped.append("\\/");
+                break;
+            default:
+                if (c < 32 || c >= 127) {
+                    escaped.append("\\u").append(fourDigitHexString(c));
+                } else {
+                    escaped.append(c);
+                }
+            }
+        }
+        return escaped;
+    }
+
+    private static char[] fourDigitHexString(char c) {
+        char[] hex = new char[4];
+        int in = ((c) & 0xFFFF);
+        for (int i = 3; i >= 0; --i) {
+            hex[i] = DIGITS[in & 0xF];
+            in >>>= 4;
+        }
+        return hex;
+    }
+
+    static String getIndexName(Item item) {
+        if (!(item instanceof IndexedItem))
+            throw new IllegalArgumentException("Expected IndexedItem, got " + item.getClass());
+        return normalizeIndexName(((IndexedItem) item).getIndexName());
+    }
+
+    public static String serialize(Query query) {
+        StringBuilder out = new StringBuilder();
+        serialize(query.getModel().getQueryTree().getRoot(), out);
+        for (GroupingRequest request : GroupingRequest.getRequests(query)) {
+            out.append(" | ");
+            serialize(request, out);
+        }
+        return out.toString();
+    }
+
+    private static void serialize(GroupingRequest request, StringBuilder out) {
+        Iterator<Continuation> it = request.continuations().iterator();
+        if (it.hasNext()) {
+            out.append("[{ 'continuations':[");
+            while (it.hasNext()) {
+                out.append('\'').append(it.next()).append('\'');
+                if (it.hasNext()) {
+                    out.append(", ");
+                }
+            }
+            out.append("] }]");
+        }
+        out.append(request.getRootOperation());
+    }
+
+    private static void serialize(Item item, StringBuilder out) {
+        VespaVisitor visitor = new VespaVisitor(out);
+        ToolBox.visit(visitor, item);
+    }
+
+    static String serialize(Item item) {
+        StringBuilder out = new StringBuilder();
+        serialize(item, out);
+        return out.toString();
+    }
+
+    private static void serializeWeightedSetContents(StringBuilder destination,
+            String opName, WeightedSetItem weightedSet) {
+        serializeWeightedSetContents(destination, opName, weightedSet, "");
+    }
+
+    private static void serializeWeightedSetContents(
+            StringBuilder destination,
+            String opName, WeightedSetItem weightedSet,
+            String optionalAnnotations) {
+        addAnnotations(destination, weightedSet, optionalAnnotations);
+        destination.append(opName).append('(')
+                .append(normalizeIndexName(weightedSet.getIndexName()))
+                .append(", {");
+        int initLen = destination.length();
+        List<Entry<Object, Integer>> tokens = new ArrayList<>(
+                weightedSet.getNumTokens());
+        for (Iterator<Entry<Object, Integer>> i = weightedSet.getTokens(); i
+                .hasNext();) {
+            tokens.add(i.next());
+        }
+        Collections.sort(tokens, tokenComparator);
+        for (Entry<Object, Integer> entry : tokens) {
+            comma(destination, initLen);
+            destination.append('"');
+            escape(entry.getKey().toString(), destination);
+            destination.append("\": ").append(entry.getValue().toString());
+        }
+        destination.append("})");
+    }
+
+    private static void addAnnotations(
+            StringBuilder destination,
+            WeightedSetItem weightedSet, String optionalAnnotations) {
+        int preAnnotationValueLen;
+        int incomingLen = destination.length();
+        String annotations = leafAnnotations(weightedSet);
+
+        if (optionalAnnotations.length() > 0 || annotations.length() > 0) {
+            destination.append("[{");
+        }
+        preAnnotationValueLen = destination.length();
+        if (annotations.length() > 0) {
+            destination.append(annotations);
+        }
+        if (optionalAnnotations.length() > 0) {
+            comma(destination, preAnnotationValueLen);
+            destination.append(optionalAnnotations);
+        }
+        if (destination.length() > incomingLen) {
+            destination.append("}]");
+        }
+    }
+
+    private static void comma(StringBuilder annotation, int initLen) {
+        if (annotation.length() > initLen) {
+            annotation.append(", ");
+        }
+    }
+
+    private static String leafAnnotations(TaggableItem item) {
+        // TODO there is no usable API for the general annotations map in the
+        // Item instances
+        StringBuilder annotation = new StringBuilder();
+        int initLen = annotation.length();
+        {
+            int uniqueId = item.getUniqueID();
+            double connectivity = item.getConnectivity();
+            TaggableItem connectedTo = (TaggableItem) item.getConnectedItem();
+            double significance = item.getSignificance();
+            if (connectedTo != null && connectedTo.getUniqueID() != 0) {
+                annotation.append('"').append(CONNECTIVITY).append("\": {\"")
+                        .append(CONNECTION_ID).append("\": ")
+                        .append(connectedTo.getUniqueID()).append(", \"")
+                        .append(CONNECTION_WEIGHT).append("\": ")
+                        .append(connectivity).append("}");
+            }
+            if (item.hasExplicitSignificance()) {
+                comma(annotation, initLen);
+                annotation.append('"').append(SIGNIFICANCE).append("\": ")
+                        .append(significance);
+            }
+            if (uniqueId != 0) {
+                comma(annotation, initLen);
+                annotation.append('"').append(UNIQUE_ID).append("\": ")
+                        .append(uniqueId);
+            }
+        }
+        {
+            Item leaf = (Item) item;
+            boolean filter = leaf.isFilter();
+            boolean isRanked = leaf.isRanked();
+            String label = leaf.getLabel();
+            int weight = leaf.getWeight();
+
+            if (filter == true) {
+                comma(annotation, initLen);
+                annotation.append("\"").append(FILTER).append("\": true");
+            }
+            if (isRanked == false) {
+                comma(annotation, initLen);
+                annotation.append("\"").append(RANKED).append("\": false");
+            }
+            if (label != null) {
+                comma(annotation, initLen);
+                annotation.append("\"").append(LABEL).append("\": \"");
+                escape(label, annotation);
+                annotation.append("\"");
+            }
+            if (weight != 100) {
+                comma(annotation, initLen);
+                annotation.append('"').append(WEIGHT).append("\": ")
+                        .append(weight);
+            }
+        }
+        if (item instanceof IntItem) {
+            int hitLimit = ((IntItem) item).getHitLimit();
+            if (hitLimit != 0) {
+                comma(annotation, initLen);
+                annotation.append('"').append(HIT_LIMIT).append("\": ")
+                        .append(hitLimit);
+            }
+        }
+        return annotation.toString();
+    }
+
+    private static void serializeOrigin(StringBuilder destination,
+            String image, int offset, int length) {
+        destination.append('"').append(ORIGIN).append("\": {\"")
+                .append(ORIGIN_ORIGINAL).append("\": \"");
+        escape(image, destination);
+        destination.append("\", \"").append(ORIGIN_OFFSET).append("\": ")
+                .append(offset).append(", \"").append(ORIGIN_LENGTH)
+                .append("\": ").append(length).append("}");
+    }
+
+    private static String normalizeIndexName(@NonNull String indexName) {
+        if (indexName.length() == 0) {
+            return "default";
+        } else {
+            return indexName;
+        }
+    }
+
+    private static void annotatedTerm(StringBuilder destination, IndexedItem w, String annotations) {
+        if (annotations.length() > 0) {
+            destination.append("([{").append(annotations).append("}]");
+        }
+        destination.append('"');
+        escape(w.getIndexedString(), destination).append('"');
+        if (annotations.length() > 0) {
+            destination.append(')');
+        }
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
new file mode 100644
index 00000000000..a7cc06c95f7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
@@ -0,0 +1,1894 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+import java.math.BigInteger;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import com.google.common.annotations.Beta;
+import com.google.common.base.Preconditions;
+import com.yahoo.collections.LazyMap;
+import com.yahoo.collections.LazySet;
+import com.yahoo.collections.Tuple2;
+import com.yahoo.component.Version;
+import com.yahoo.language.Language;
+import com.yahoo.language.Linguistics;
+import com.yahoo.language.process.Normalizer;
+import com.yahoo.language.process.Segmenter;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.AndSegmentItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.DotProductItem;
+import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.Limit;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.prelude.query.NotItem;
+import com.yahoo.prelude.query.NullItem;
+import com.yahoo.prelude.query.ONearItem;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PhraseSegmentItem;
+import com.yahoo.prelude.query.PredicateQueryItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.RangeItem;
+import com.yahoo.prelude.query.RankItem;
+import com.yahoo.prelude.query.RegExpItem;
+import com.yahoo.prelude.query.SegmentItem;
+import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.SubstringItem;
+import com.yahoo.prelude.query.SuffixItem;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.prelude.query.ToolBox;
+import com.yahoo.prelude.query.ToolBox.QueryVisitor;
+import com.yahoo.prelude.query.WandItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WeightedSetItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.Query;
+import com.yahoo.search.grouping.Continuation;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.search.query.Sorting;
+import com.yahoo.search.query.Sorting.AttributeSorter;
+import com.yahoo.search.query.Sorting.FieldOrder;
+import com.yahoo.search.query.Sorting.LowerCaseSorter;
+import com.yahoo.search.query.Sorting.Order;
+import com.yahoo.search.query.Sorting.RawSorter;
+import com.yahoo.search.query.Sorting.UcaSorter;
+import com.yahoo.search.query.parser.Parsable;
+import com.yahoo.search.query.parser.Parser;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * The YQL query language.
+ *
+ * <p>
+ * This class <em>must</em> be kept in lockstep with {@link VespaSerializer}.
+ * Adding anything here will usually require a corresponding addition in
+ * VespaSerializer.
+ * </p>
+ *
+ * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author <a href="mailto:stiankri@yahoo-inc.com">Stian Kristoffersen</a>
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ */
+@Beta
+public class YqlParser implements Parser {
+
+    private static final String DESCENDING_HITS_ORDER = "descending";
+    private static final String ASCENDING_HITS_ORDER = "ascending";
+
+    private enum SegmentWhen {
+        NEVER, POSSIBLY, ALWAYS;
+    }
+
+    private static final Integer DEFAULT_HITS = 10;
+    private static final Integer DEFAULT_OFFSET = 0;
+    private static final Integer DEFAULT_TARGET_NUM_HITS = 10;
+    private static final String ACCENT_DROP_DESCRIPTION = "setting for whether to remove accents if field implies it";
+    private static final String ANNOTATIONS = "annotations";
+    private static final String FILTER_DESCRIPTION = "term filter setting";
+    private static final String IMPLICIT_TRANSFORMS_DESCRIPTION = "setting for whether built-in query transformers should touch the term";
+    private static final String NFKC = "nfkc";
+    private static final String NORMALIZE_CASE_DESCRIPTION = "setting for whether to do case normalization if field implies it";
+    private static final String ORIGIN_DESCRIPTION = "string origin for a term";
+    private static final String RANKED_DESCRIPTION = "setting for whether to use term for ranking";
+    private static final String SEGMENTER_BACKEND = "backend";
+    private static final String SEGMENTER = "segmenter";
+    private static final String SEGMENTER_VERSION = "version";
+    private static final String STEM_DESCRIPTION = "setting for whether to use stem if field implies it";
+    private static final String USE_POSITION_DATA_DESCRIPTION = "setting for whether to use position data for ranking this item";
+    private static final String USER_INPUT_ALLOW_EMPTY = "allowEmpty";
+    private static final String USER_INPUT_DEFAULT_INDEX = "defaultIndex";
+    private static final String USER_INPUT_GRAMMAR = "grammar";
+    private static final String USER_INPUT_LANGUAGE = "language";
+    private static final String USER_INPUT_RAW = "raw";
+    private static final String USER_INPUT_SEGMENT = "segment";
+    private static final String USER_INPUT = "userInput";
+    private static final String USER_QUERY = "userQuery";
+    private static final String NON_EMPTY = "nonEmpty";
+
+    public static final String SORTING_FUNCTION = "function";
+    public static final String SORTING_LOCALE = "locale";
+    public static final String SORTING_STRENGTH = "strength";
+
+    static final String ACCENT_DROP = "accentDrop";
+    static final String ALTERNATIVES = "alternatives";
+    static final String AND_SEGMENTING = "andSegmenting";
+    static final String BOUNDS = "bounds";
+    static final String BOUNDS_LEFT_OPEN = "leftOpen";
+    static final String BOUNDS_OPEN = "open";
+    static final String BOUNDS_RIGHT_OPEN = "rightOpen";
+    static final String CONNECTION_ID = "id";
+    static final String CONNECTION_WEIGHT = "weight";
+    static final String CONNECTIVITY = "connectivity";
+    static final String DISTANCE = "distance";
+    static final String DOT_PRODUCT = "dotProduct";
+    static final String EQUIV = "equiv";
+    static final String FILTER = "filter";
+    static final String HIT_LIMIT = "hitLimit";
+    static final String IMPLICIT_TRANSFORMS = "implicitTransforms";
+    static final String LABEL = "label";
+    static final String NEAR = "near";
+    static final String NORMALIZE_CASE = "normalizeCase";
+    static final String ONEAR = "onear";
+    static final String ORIGIN_LENGTH = "length";
+    static final String ORIGIN_OFFSET = "offset";
+    static final String ORIGIN = "origin";
+    static final String ORIGIN_ORIGINAL = "original";
+    static final String PHRASE = "phrase";
+    static final String PREDICATE = "predicate";
+    static final String PREFIX = "prefix";
+    static final String RANGE = "range";
+    static final String RANKED = "ranked";
+    static final String RANK = "rank";
+    static final String SCORE_THRESHOLD = "scoreThreshold";
+    static final String SIGNIFICANCE = "significance";
+    static final String STEM = "stem";
+    static final String SUBSTRING = "substring";
+    static final String SUFFIX = "suffix";
+    static final String TARGET_NUM_HITS = "targetNumHits";
+    static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor";
+    static final String UNIQUE_ID = "id";
+    static final String USE_POSITION_DATA = "usePositionData";
+    static final String WAND = "wand";
+    static final String WEAK_AND = "weakAnd";
+    static final String WEIGHTED_SET = "weightedSet";
+    static final String WEIGHT = "weight";
+
+    private final IndexFacts indexFacts;
+    private final List<ConnectedItem> connectedItems = new ArrayList<>();
+    private final List<VespaGroupingStep> groupingSteps = new ArrayList<>();
+    private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap();
+    private final Normalizer normalizer;
+    private final Segmenter segmenter;
+    private final Set<String> yqlSources = LazySet.newHashSet();
+    private final Set<String> yqlSummaryFields = LazySet.newHashSet();
+    private final String localSegmenterBackend;
+    private final Version localSegmenterVersion;
+    private Integer hits;
+    private Integer offset;
+    private Integer timeout;
+    private Query userQuery;
+    private Parsable currentlyParsing;
+    private IndexFacts.Session indexFactsSession;
+    private Set<String> docTypes;
+    private Sorting sorting;
+    private String segmenterBackend;
+    private Version segmenterVersion;
+    private boolean queryParser = true;
+    private boolean resegment = false;
+    private final Deque<OperatorNode<?>> annotationStack = new ArrayDeque<>();
+    private final ParserEnvironment environment;
+
+    private static final QueryVisitor noEmptyTerms = new QueryVisitor() {
+
+        @Override
+        public boolean visit(Item item) {
+            if (item instanceof NullItem) {
+                throw new IllegalArgumentException("Got NullItem inside nonEmpty().");
+            } else if (item instanceof WordItem) {
+                if (((WordItem) item).getIndexedString().isEmpty()) {
+                    throw new IllegalArgumentException("Searching for empty string inside nonEmpty()");
+                }
+            } else if (item instanceof CompositeItem) {
+                if (((CompositeItem) item).getItemCount() == 0) {
+                    throw new IllegalArgumentException("Empty composite operator (" + item.getName() + ") inside nonEmpty()");
+                }
+            }
+            return true;
+        }
+
+        @Override
+        public void onExit() {
+            // NOP
+        }
+    };
+
+    public YqlParser(ParserEnvironment environment) {
+        indexFacts = environment.getIndexFacts();
+        normalizer = environment.getLinguistics().getNormalizer();
+        segmenter = environment.getLinguistics().getSegmenter();
+        this.environment = environment;
+
+        Tuple2<String, Version> version = environment.getLinguistics().getVersion(Linguistics.Component.SEGMENTER);
+        localSegmenterBackend = version.first;
+        localSegmenterVersion = version.second;
+    }
+
+    @NonNull
+    @Override
+    public QueryTree parse(Parsable query) {
+        indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict());
+        connectedItems.clear();
+        groupingSteps.clear();
+        identifiedItems.clear();
+        yqlSources.clear();
+        yqlSummaryFields.clear();
+        annotationStack.clear();
+        hits = null;
+        offset = null;
+        timeout = null;
+        // userQuery set prior to calling this
+        currentlyParsing = query;
+        docTypes = null;
+        sorting = null;
+        segmenterBackend = null;
+        segmenterVersion = null;
+        // queryParser set prior to calling this
+        resegment = false;
+        return buildTree(fetchFilterPart());
+    }
+
+    private void joinDocTypesFromUserQueryAndYql() {
+        List<String> allSourceNames = new ArrayList<>(currentlyParsing.getSources().size() + yqlSources.size());
+        if ( ! yqlSources.isEmpty()) {
+            allSourceNames.addAll(currentlyParsing.getSources());
+            allSourceNames.addAll(yqlSources);
+        } else {
+            // no sources == all sources in Vespa
+        }
+        indexFactsSession = indexFacts.newSession(allSourceNames, currentlyParsing.getRestrict());
+        docTypes = new HashSet<>(indexFactsSession.documentTypes());
+    }
+
+    @NonNull
+    private QueryTree buildTree(OperatorNode<?> filterPart) {
+        Preconditions.checkArgument(filterPart.getArguments().length == 2,
+                "Expected 2 arguments to filter, got %s.",
+                filterPart.getArguments().length);
+        populateYqlSources(filterPart.<OperatorNode<?>> getArgument(0));
+        final OperatorNode<ExpressionOperator> filterExpression = filterPart
+                .getArgument(1);
+        populateLinguisticsAnnotations(filterExpression);
+        Item root = convertExpression(filterExpression);
+        connectItems();
+        userQuery = null;
+        return new QueryTree(root);
+    }
+
+    private void populateLinguisticsAnnotations(
+            OperatorNode<ExpressionOperator> filterExpression) {
+        Map<?, ?> segmenter = getAnnotation(filterExpression, SEGMENTER,
+                Map.class, null, "segmenter engine and version");
+        if (segmenter == null) {
+            segmenterVersion = null;
+            segmenterBackend = null;
+            resegment = false;
+        } else {
+            segmenterBackend = getMapValue(SEGMENTER, segmenter,
+                    SEGMENTER_BACKEND, String.class);
+            try {
+                segmenterVersion = new Version(getMapValue(SEGMENTER,
+                        segmenter, SEGMENTER_VERSION, String.class));
+            } catch (RuntimeException e) {
+                segmenterVersion = null;
+            }
+            if (localSegmenterBackend.equals(segmenterBackend)
+                    && localSegmenterVersion.equals(segmenterVersion)) {
+                resegment = false;
+            } else {
+                resegment = true;
+            }
+        }
+    }
+
+    private void populateYqlSources(OperatorNode<?> filterArgs) {
+        yqlSources.clear();
+        if (filterArgs.getOperator() == SequenceOperator.SCAN) {
+            for (String source : filterArgs.<List<String>> getArgument(0)) {
+                yqlSources.add(source);
+            }
+        } else if (filterArgs.getOperator() == SequenceOperator.ALL) {
+            // yqlSources has already been cleared
+        } else if (filterArgs.getOperator() == SequenceOperator.MULTISOURCE) {
+            for (List<String> source : filterArgs.<List<List<String>>> getArgument(0)) {
+                yqlSources.add(source.get(0));
+            }
+        } else {
+            throw newUnexpectedArgumentException(filterArgs.getOperator(),
+                    SequenceOperator.SCAN, SequenceOperator.ALL,
+                    SequenceOperator.MULTISOURCE);
+        }
+        joinDocTypesFromUserQueryAndYql();
+    }
+
+    private void populateYqlSummaryFields(
+            List<OperatorNode<ProjectOperator>> fields) {
+        yqlSummaryFields.clear();
+        for (OperatorNode<ProjectOperator> field : fields) {
+            assertHasOperator(field, ProjectOperator.FIELD);
+            yqlSummaryFields.add(field.getArgument(1, String.class));
+        }
+    }
+
+    private void connectItems() {
+        for (ConnectedItem entry : connectedItems) {
+            TaggableItem to = identifiedItems.get(entry.toId);
+            Preconditions
+                    .checkNotNull(to,
+                            "Item '%s' was specified to connect to item with ID %s, which does not "
+                                    + "exist in the query.", entry.fromItem,
+                            entry.toId);
+            entry.fromItem.setConnectivity((Item) to, entry.weight);
+        }
+    }
+
+    @NonNull
+    private Item convertExpression(OperatorNode<ExpressionOperator> ast) {
+        try {
+            annotationStack.addFirst(ast);
+            switch (ast.getOperator()) {
+            case AND:
+                return buildAnd(ast);
+            case OR:
+                return buildOr(ast);
+            case EQ:
+                return buildEquals(ast);
+            case LT:
+                return buildLessThan(ast);
+            case GT:
+                return buildGreaterThan(ast);
+            case LTEQ:
+                return buildLessThanOrEquals(ast);
+            case GTEQ:
+                return buildGreaterThanOrEquals(ast);
+            case CONTAINS:
+                return buildTermSearch(ast);
+            case MATCHES:
+                return buildRegExpSearch(ast);
+            case CALL:
+                return buildFunctionCall(ast);
+            default:
+                throw newUnexpectedArgumentException(ast.getOperator(),
+                        ExpressionOperator.AND, ExpressionOperator.CALL,
+                        ExpressionOperator.CONTAINS, ExpressionOperator.EQ,
+                        ExpressionOperator.GT, ExpressionOperator.GTEQ,
+                        ExpressionOperator.LT, ExpressionOperator.LTEQ,
+                        ExpressionOperator.OR);
+            }
+        } finally {
+            annotationStack.removeFirst();
+        }
+    }
+
+    @NonNull
+    private Item buildFunctionCall(OperatorNode<ExpressionOperator> ast) {
+        List<String> names = ast.getArgument(0);
+        Preconditions.checkArgument(names.size() == 1,
+                "Expected 1 name, got %s.", names.size());
+        switch (names.get(0)) {
+        case USER_QUERY:
+            return fetchUserQuery();
+        case RANGE:
+            return buildRange(ast);
+        case WAND:
+            return buildWand(ast);
+        case WEIGHTED_SET:
+            return buildWeightedSet(ast);
+        case DOT_PRODUCT:
+            return buildDotProduct(ast);
+        case PREDICATE:
+            return buildPredicate(ast);
+        case RANK:
+            return buildRank(ast);
+        case WEAK_AND:
+            return buildWeakAnd(ast);
+        case USER_INPUT:
+            return buildUserInput(ast);
+        case NON_EMPTY:
+            return ensureNonEmpty(ast);
+        default:
+            throw newUnexpectedArgumentException(names.get(0), DOT_PRODUCT,
+                    RANGE, RANK, USER_QUERY, WAND, WEAK_AND, WEIGHTED_SET,
+                    PREDICATE, USER_INPUT, NON_EMPTY);
+        }
+    }
+
+    private Item ensureNonEmpty(OperatorNode<ExpressionOperator> ast) {
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+        Preconditions.checkArgument(args.size() == 1,
+                "Expected 1 arguments, got %s.", args.size());
+        Item item = convertExpression(args.get(0));
+        ToolBox.visit(noEmptyTerms, item);
+        return item;
+    }
+
+    @NonNull
+    private Item buildWeightedSet(OperatorNode<ExpressionOperator> ast) {
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+        Preconditions.checkArgument(args.size() == 2,
+                "Expected 2 arguments, got %s.", args.size());
+
+        return fillWeightedSet(ast, args.get(1), new WeightedSetItem(
+                getIndex(args.get(0))));
+    }
+
+    @NonNull
+    private Item buildDotProduct(OperatorNode<ExpressionOperator> ast) {
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+        Preconditions.checkArgument(args.size() == 2,
+                "Expected 2 arguments, got %s.", args.size());
+
+        return fillWeightedSet(ast, args.get(1), new DotProductItem(
+                getIndex(args.get(0))));
+    }
+
+    @NonNull
+    private Item buildPredicate(OperatorNode<ExpressionOperator> ast) {
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+        Preconditions.checkArgument(args.size() == 3,
+                "Expected 3 arguments, got %s.", args.size());
+
+        final PredicateQueryItem item = new PredicateQueryItem();
+        item.setIndexName(getIndex(args.get(0)));
+
+        addFeatures(args.get(1),
+                (key, value, subqueryBitmap) -> item.addFeature(key, (String) value, subqueryBitmap), PredicateQueryItem.ALL_SUB_QUERIES);
+        addFeatures(args.get(2), (key, value, subqueryBitmap) -> {
+            if (value instanceof Long) {
+                item.addRangeFeature(key, (Long) value, subqueryBitmap);
+            } else {
+                item.addRangeFeature(key, (Integer) value, subqueryBitmap);
+            }
+        }, PredicateQueryItem.ALL_SUB_QUERIES);
+        return leafStyleSettings(ast, item);
+    }
+
+    interface AddFeature {
+        public void addFeature(String key, Object value, long subqueryBitmap);
+    }
+
+    private void addFeatures(OperatorNode<ExpressionOperator> map,
+            AddFeature item, long subqueryBitmap) {
+        if (map.getOperator() != ExpressionOperator.MAP) {
+            return;
+        }
+        assertHasOperator(map, ExpressionOperator.MAP);
+        List<String> keys = map.getArgument(0);
+        List<OperatorNode<ExpressionOperator>> values = map.getArgument(1);
+        for (int i = 0; i < keys.size(); ++i) {
+            String key = keys.get(i);
+            OperatorNode<ExpressionOperator> value = values.get(i);
+            if (value.getOperator() == ExpressionOperator.ARRAY) {
+                List<OperatorNode<ExpressionOperator>> multiValues = value
+                        .getArgument(0);
+                for (OperatorNode<ExpressionOperator> multiValue : multiValues) {
+                    assertHasOperator(multiValue, ExpressionOperator.LITERAL);
+                    item.addFeature(key, multiValue.getArgument(0), subqueryBitmap);
+                }
+            } else if (value.getOperator() == ExpressionOperator.LITERAL) {
+                item.addFeature(key, value.getArgument(0), subqueryBitmap);
+            } else {
+                assertHasOperator(value, ExpressionOperator.MAP); // Subquery syntax
+                Preconditions.checkArgument(key.indexOf("0x") == 0 || key.indexOf("[") == 0);
+                if (key.indexOf("0x") == 0) {
+                    String subqueryString = key.substring(2);
+                    if (subqueryString.length() > 16) {
+                        throw new NumberFormatException(
+                                "Too long subquery string: " + key);
+                    }
+                    long currentSubqueryBitmap = new BigInteger(subqueryString, 16).longValue();
+                    addFeatures(value, item, currentSubqueryBitmap);
+                } else {
+                    StringTokenizer bits = new StringTokenizer(key.substring(1, key.length() - 1), ",");
+                    long currentSubqueryBitmap = 0;
+                    while (bits.hasMoreTokens()) {
+                        int bit = Integer.parseInt(bits.nextToken().trim());
+                        currentSubqueryBitmap |= 1L << bit;
+                    }
+                    addFeatures(value, item, currentSubqueryBitmap);
+                }
+            }
+        }
+    }
+
+    @NonNull
+    private Item buildWand(OperatorNode<ExpressionOperator> ast) {
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+        Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size());
+
+        WandItem out = new WandItem(getIndex(args.get(0)), getAnnotation(ast,
+                TARGET_NUM_HITS, Integer.class, DEFAULT_TARGET_NUM_HITS,
+                "desired number of hits to accumulate in wand"));
+        Double scoreThreshold = getAnnotation(ast, SCORE_THRESHOLD,
+                Double.class, null, "min score for hit inclusion");
+        if (scoreThreshold != null) {
+            out.setScoreThreshold(scoreThreshold);
+        }
+        Double thresholdBoostFactor = getAnnotation(ast,
+                THRESHOLD_BOOST_FACTOR, Double.class, null,
+                "boost factor used to boost threshold before comparing against upper bound score");
+        if (thresholdBoostFactor != null) {
+            out.setThresholdBoostFactor(thresholdBoostFactor);
+        }
+        return fillWeightedSet(ast, args.get(1), out);
+    }
+
+    @NonNull
+    private WeightedSetItem fillWeightedSet(OperatorNode<ExpressionOperator> ast,
+                                            OperatorNode<ExpressionOperator> arg,
+                                            @NonNull WeightedSetItem out) {
+        addItems(arg, out);
+        return leafStyleSettings(ast, out);
+    }
+
+    @NonNull
+    private Item instantiatePhraseItem(String field, OperatorNode<ExpressionOperator> ast) {
+        assertHasFunctionName(ast, PHRASE);
+
+        if (getAnnotation(ast, ORIGIN, Map.class, null, ORIGIN_DESCRIPTION, false) != null) {
+            return instantiatePhraseSegmentItem(field, ast, false);
+        }
+
+        PhraseItem phrase = new PhraseItem();
+        phrase.setIndexName(field);
+        for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>> getArgument(1)) {
+            if (word.getOperator() == ExpressionOperator.CALL) {
+                List<String> names = word.getArgument(0);
+                switch (names.get(0)) {
+                case PHRASE:
+                    if (getAnnotation(word, ORIGIN, Map.class, null, ORIGIN_DESCRIPTION, false) == null) {
+                        phrase.addItem(instantiatePhraseItem(field, word));
+                    } else {
+                        phrase.addItem(instantiatePhraseSegmentItem(field, word, true));
+                    }
+                    break;
+                case ALTERNATIVES:
+                    phrase.addItem(instantiateWordAlternativesItem(field, word));
+                    break;
+                default:
+                    throw new IllegalArgumentException("Expected phrase or word alternatives, got " + names.get(0));
+                }
+            } else {
+                phrase.addItem(instantiateWordItem(field, word, phrase.getClass()));
+            }
+        }
+        return leafStyleSettings(ast, phrase);
+    }
+
+    @NonNull
+    private Item instantiatePhraseSegmentItem(String field, OperatorNode<ExpressionOperator> ast, boolean forcePhrase) {
+        Substring origin = getOrigin(ast);
+        Boolean stem = getAnnotation(ast, STEM, Boolean.class, Boolean.TRUE, STEM_DESCRIPTION);
+        Boolean andSegmenting = getAnnotation(ast, AND_SEGMENTING, Boolean.class, Boolean.FALSE,
+                                              "setting for whether to force using AND for segments on and off");
+        SegmentItem phrase;
+        List<String> words = null;
+
+        if (forcePhrase || !andSegmenting) {
+            phrase = new PhraseSegmentItem(origin.getValue(), origin.getValue(), true, !stem, origin);
+        } else {
+            phrase = new AndSegmentItem(origin.getValue(), true, !stem);
+        }
+        phrase.setIndexName(field);
+
+        if (resegment
+            && getAnnotation(ast, IMPLICIT_TRANSFORMS, Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION)) {
+            words = segmenter.segment(origin.getValue(), currentlyParsing.getLanguage());
+        }
+
+        if (words != null && words.size() > 0) {
+            for (String word : words) {
+                phrase.addItem(new WordItem(word, field, true));
+            }
+        } else {
+            for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>> getArgument(1)) {
+                phrase.addItem(instantiateWordItem(field, word, phrase.getClass(), SegmentWhen.NEVER));
+            }
+        }
+        if (phrase instanceof TaggableItem) {
+            leafStyleSettings(ast, (TaggableItem) phrase);
+        }
+        phrase.lock();
+        return phrase;
+    }
+
+    @NonNull
+    private Item instantiateNearItem(String field, OperatorNode<ExpressionOperator> ast) {
+        assertHasFunctionName(ast, NEAR);
+
+        NearItem near = new NearItem();
+        near.setIndexName(field);
+        for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>> getArgument(1)) {
+            near.addItem(instantiateWordItem(field, word, near.getClass()));
+        }
+        Integer distance = getAnnotation(ast, DISTANCE, Integer.class, null, "term distance for NEAR operator");
+        if (distance != null) {
+            near.setDistance(distance);
+        }
+        return near;
+    }
+
+    @NonNull
+    private Item instantiateONearItem(String field, OperatorNode<ExpressionOperator> ast) {
+        assertHasFunctionName(ast, ONEAR);
+
+        NearItem onear = new ONearItem();
+        onear.setIndexName(field);
+        for (OperatorNode<ExpressionOperator> word : ast.<List<OperatorNode<ExpressionOperator>>> getArgument(1)) {
+            onear.addItem(instantiateWordItem(field, word, onear.getClass()));
+        }
+        Integer distance = getAnnotation(ast, DISTANCE, Integer.class, null, "term distance for ONEAR operator");
+        if (distance != null) {
+            onear.setDistance(distance);
+        }
+        return onear;
+    }
+
+    @NonNull
+    private Item fetchUserQuery() {
+        Preconditions.checkState(!queryParser,
+                "Tried inserting user query into itself.");
+        Preconditions.checkState(userQuery != null,
+                "User query must be set before trying to build complete query "
+                        + "tree including user query.");
+        return userQuery.getModel().getQueryTree().getRoot();
+    }
+
+    @NonNull
+    private Item buildUserInput(OperatorNode<ExpressionOperator> ast) {
+
+        String grammar = getAnnotation(ast, USER_INPUT_GRAMMAR, String.class,
+                Query.Type.ALL.toString(), "grammar for handling user input");
+        String defaultIndex = getAnnotation(ast, USER_INPUT_DEFAULT_INDEX,
+                String.class, "default", "default index for user input terms");
+        Boolean allowEmpty = getAnnotation(ast, USER_INPUT_ALLOW_EMPTY, Boolean.class,
+                Boolean.FALSE, "flag for allowing NullItem to be returned");
+        String wordData;
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+
+        // TODO add support for default arguments if property results in nothing
+        wordData = getStringContents(args.get(0));
+        if (allowEmpty.booleanValue() && (wordData == null || wordData.isEmpty())) {
+            return new NullItem();
+        }
+        String languageTag = getAnnotation(ast, USER_INPUT_LANGUAGE,
+                String.class, "en",
+                "language setting for segmenting user input parameter");
+        Language language = Language.fromLanguageTag(languageTag);
+        Item item;
+        if (USER_INPUT_RAW.equals(grammar)) {
+            item = instantiateWordItem(defaultIndex, wordData, ast, null, SegmentWhen.NEVER,
+                    language);
+        } else if (USER_INPUT_SEGMENT.equals(grammar)) {
+            item = instantiateWordItem(defaultIndex, wordData, ast, null,
+                    SegmentWhen.ALWAYS, language);
+        } else {
+            item = parseUserInput(grammar, defaultIndex, wordData, language, allowEmpty.booleanValue());
+            propagateUserInputAnnotations(ast, item);
+        }
+        return item;
+    }
+
+    private String getStringContents(
+            OperatorNode<ExpressionOperator> propertySniffer) {
+        String wordData;
+
+        switch (propertySniffer.getOperator()) {
+        case LITERAL:
+            wordData = propertySniffer.getArgument(0, String.class);
+            break;
+        case VARREF:
+            Preconditions
+                    .checkState(userQuery != null,
+                            "properties must be available when trying to fetch user input");
+            wordData = userQuery.properties().getString(
+                    propertySniffer.getArgument(0, String.class));
+            break;
+        default:
+            throw newUnexpectedArgumentException(propertySniffer.getOperator(),
+                    ExpressionOperator.LITERAL, ExpressionOperator.VARREF);
+        }
+        return wordData;
+    }
+
+    private class AnnotationPropagator extends QueryVisitor {
+        private final Boolean isRanked;
+        private final Boolean filter;
+        private final Boolean stem;
+        private final Boolean normalizeCase;
+        private final Boolean accentDrop;
+        private final Boolean usePositionData;
+
+        public AnnotationPropagator(OperatorNode<ExpressionOperator> ast) {
+            isRanked = getAnnotation(ast, RANKED, Boolean.class, null,
+                    RANKED_DESCRIPTION);
+            filter = getAnnotation(ast, FILTER, Boolean.class, null,
+                    FILTER_DESCRIPTION);
+            stem = getAnnotation(ast, STEM, Boolean.class, null,
+                    STEM_DESCRIPTION);
+            normalizeCase = getAnnotation(ast, NORMALIZE_CASE, Boolean.class,
+                    Boolean.TRUE, NORMALIZE_CASE_DESCRIPTION);
+            accentDrop = getAnnotation(ast, ACCENT_DROP, Boolean.class, null,
+                    ACCENT_DROP_DESCRIPTION);
+            usePositionData = getAnnotation(ast, USE_POSITION_DATA,
+                    Boolean.class, null, USE_POSITION_DATA_DESCRIPTION);
+        }
+
+        @Override
+        public boolean visit(Item item) {
+            if (item instanceof WordItem) {
+                WordItem w = (WordItem) item;
+                if (usePositionData != null) {
+                    w.setPositionData(usePositionData);
+                }
+                if (stem != null) {
+                    w.setStemmed(!stem);
+                }
+                if (normalizeCase != null) {
+                    w.setLowercased(!normalizeCase);
+                }
+                if (accentDrop != null) {
+                    w.setNormalizable(accentDrop);
+                }
+            }
+            if (item instanceof TaggableItem) {
+                if (isRanked != null) {
+                    item.setRanked(isRanked);
+                }
+                if (filter != null) {
+                    item.setFilter(filter);
+                }
+            }
+            return true;
+        }
+
+        @Override
+        public void onExit() {
+            // intentionally left blank
+        }
+    }
+
+    private void propagateUserInputAnnotations(
+            OperatorNode<ExpressionOperator> ast, Item item) {
+        ToolBox.visit(new AnnotationPropagator(ast), item);
+
+    }
+
+    @NonNull
+    private Item parseUserInput(String grammar, String defaultIndex, String wordData,
+            Language language, boolean allowNullItem) {
+        Item item;
+        Query.Type parseAs = Query.Type.getType(grammar);
+        Parser parser = ParserFactory.newInstance(parseAs, environment);
+        // perhaps not use already resolved doctypes, but respect source and
+        // restrict
+        item = parser.parse(
+                new Parsable().setQuery(wordData).addSources(docTypes)
+                        .setLanguage(language)
+                        .setDefaultIndexName(defaultIndex)).getRoot();
+        // the null check should be unnecessary, but is there to avoid having to
+        // suppress null warnings
+        if (!allowNullItem && (item == null || item instanceof NullItem)) {
+            throw new IllegalArgumentException("Parsing \"" + wordData
+                    + "\" only resulted in NullItem.");
+        }
+        return item;
+    }
+
+    @NonNull
+    private OperatorNode<?> fetchFilterPart() {
+        ProgramParser parser = new ProgramParser();
+        OperatorNode<?> ast;
+        try {
+            ast = parser.parse("query", currentlyParsing.getQuery());
+        } catch (Exception e) {
+            throw new IllegalArgumentException(e);
+        }
+        assertHasOperator(ast, StatementOperator.PROGRAM);
+        Preconditions.checkArgument(ast.getArguments().length == 1,
+                "Expected only a single argument to the root node, got %s.",
+                ast.getArguments().length);
+        // TODO: should we check size of first argument as well?
+        ast = ast.<List<OperatorNode<?>>> getArgument(0).get(0);
+        assertHasOperator(ast, StatementOperator.EXECUTE);
+
+        ast = ast.getArgument(0);
+        ast = fetchTimeout(ast);
+        ast = fetchPipe(ast);
+        ast = fetchSummaryFields(ast);
+        ast = fetchOffsetAndHits(ast);
+        ast = fetchSorting(ast);
+        assertHasOperator(ast, SequenceOperator.FILTER);
+        return ast;
+    }
+
+    @SuppressWarnings("unchecked")
+    private OperatorNode<?> fetchPipe(OperatorNode<?> toScan) {
+        OperatorNode<?> ast = toScan;
+        while (ast.getOperator() == SequenceOperator.PIPE) {
+            OperatorNode<ExpressionOperator> groupingAst = ast
+                    .<List<OperatorNode<ExpressionOperator>>> getArgument(2)
+                    .get(0);
+            GroupingOperation groupingOperation = GroupingOperation
+                    .fromString(groupingAst.<String> getArgument(0));
+            VespaGroupingStep groupingStep = new VespaGroupingStep(
+                    groupingOperation);
+            List<String> continuations = getAnnotation(groupingAst,
+                    "continuations", List.class, Collections.emptyList(),
+                    "grouping continuations");
+            for (String continuation : continuations) {
+                groupingStep.continuations().add(
+                        Continuation.fromString(continuation));
+            }
+            groupingSteps.add(groupingStep);
+            ast = ast.getArgument(0);
+        }
+        Collections.reverse(groupingSteps);
+        return ast;
+    }
+
+    @NonNull
+    private OperatorNode<?> fetchSorting(OperatorNode<?> ast) {
+        if (ast.getOperator() != SequenceOperator.SORT) {
+            return ast;
+        }
+        List<FieldOrder> sortingInit = new ArrayList<>();
+        List<OperatorNode<?>> sortArguments = ast.getArgument(1);
+        for (OperatorNode<?> op : sortArguments) {
+            final OperatorNode<ExpressionOperator> fieldNode = op
+                    .<OperatorNode<ExpressionOperator>> getArgument(0);
+            String field = fetchFieldRead(fieldNode);
+            String locale = getAnnotation(fieldNode, SORTING_LOCALE,
+                    String.class, null, "locale used by sorting function");
+            String function = getAnnotation(fieldNode, SORTING_FUNCTION,
+                    String.class, null,
+                    "sorting function for the specified attribute");
+            String strength = getAnnotation(fieldNode, SORTING_STRENGTH,
+                    String.class, null, "strength for sorting function");
+            AttributeSorter sorter;
+            if (function == null) {
+                sorter = new AttributeSorter(field);
+            } else if (Sorting.LOWERCASE.equals(function)) {
+                sorter = new LowerCaseSorter(field);
+            } else if (Sorting.RAW.equals(function)) {
+                sorter = new RawSorter(field);
+            } else if (Sorting.UCA.equals(function)) {
+                if (locale != null) {
+                    UcaSorter.Strength ucaStrength = UcaSorter.Strength.UNDEFINED;
+                    if (strength != null) {
+                        if (Sorting.STRENGTH_PRIMARY.equalsIgnoreCase(strength)) {
+                            ucaStrength = UcaSorter.Strength.PRIMARY;
+                        } else if (Sorting.STRENGTH_SECONDARY
+                                .equalsIgnoreCase(strength)) {
+                            ucaStrength = UcaSorter.Strength.SECONDARY;
+                        } else if (Sorting.STRENGTH_TERTIARY
+                                .equalsIgnoreCase(strength)) {
+                            ucaStrength = UcaSorter.Strength.TERTIARY;
+                        } else if (Sorting.STRENGTH_QUATERNARY
+                                .equalsIgnoreCase(strength)) {
+                            ucaStrength = UcaSorter.Strength.QUATERNARY;
+                        } else if (Sorting.STRENGTH_IDENTICAL
+                                .equalsIgnoreCase(strength)) {
+                            ucaStrength = UcaSorter.Strength.IDENTICAL;
+                        } else {
+                            throw newUnexpectedArgumentException(function,
+                                    Sorting.STRENGTH_PRIMARY,
+                                    Sorting.STRENGTH_SECONDARY,
+                                    Sorting.STRENGTH_TERTIARY,
+                                    Sorting.STRENGTH_QUATERNARY,
+                                    Sorting.STRENGTH_IDENTICAL);
+                        }
+                        sorter = new UcaSorter(field, locale, ucaStrength);
+                    } else {
+                        sorter = new UcaSorter(field, locale, ucaStrength);
+                    }
+                } else {
+                    sorter = new UcaSorter(field);
+                }
+            } else {
+                throw newUnexpectedArgumentException(function, "lowercase",
+                        "raw", "uca");
+            }
+            switch ((SortOperator) op.getOperator()) {
+            case ASC:
+                sortingInit.add(new FieldOrder(sorter, Order.ASCENDING));
+                break;
+            case DESC:
+                sortingInit.add(new FieldOrder(sorter, Order.DESCENDING));
+                break;
+            default:
+                throw newUnexpectedArgumentException(op.getOperator(),
+                        SortOperator.ASC, SortOperator.DESC);
+            }
+        }
+        sorting = new Sorting(sortingInit);
+        return ast.getArgument(0);
+    }
+
+    @NonNull
+    private OperatorNode<?> fetchOffsetAndHits(OperatorNode<?> ast) {
+        if (ast.getOperator() == SequenceOperator.OFFSET) {
+            offset = ast.<OperatorNode<?>> getArgument(1)
+                    .<Integer> getArgument(0);
+            hits = DEFAULT_HITS;
+            return ast.getArgument(0);
+        }
+        if (ast.getOperator() == SequenceOperator.SLICE) {
+            offset = ast.<OperatorNode<?>> getArgument(1)
+                    .<Integer> getArgument(0);
+            hits = ast.<OperatorNode<?>> getArgument(2)
+                    .<Integer> getArgument(0) - offset;
+            return ast.getArgument(0);
+        }
+        if (ast.getOperator() == SequenceOperator.LIMIT) {
+            hits = ast.<OperatorNode<?>> getArgument(1)
+                    .<Integer> getArgument(0);
+            offset = DEFAULT_OFFSET;
+            return ast.getArgument(0);
+        }
+        return ast;
+    }
+
+    @NonNull
+    private OperatorNode<?> fetchSummaryFields(OperatorNode<?> ast) {
+        if (ast.getOperator() != SequenceOperator.PROJECT) {
+            return ast;
+        }
+        Preconditions.checkArgument(ast.getArguments().length == 2,
+                "Expected 2 arguments to PROJECT, got %s.",
+                ast.getArguments().length);
+        populateYqlSummaryFields(ast
+                .<List<OperatorNode<ProjectOperator>>> getArgument(1));
+        return ast.getArgument(0);
+    }
+
+    private OperatorNode<?> fetchTimeout(OperatorNode<?> ast) {
+        if (ast.getOperator() != SequenceOperator.TIMEOUT) {
+            return ast;
+        }
+        timeout = ast.<OperatorNode<?>> getArgument(1).<Integer> getArgument(0);
+        return ast.getArgument(0);
+    }
+
+    @NonNull
+    private static String fetchFieldRead(OperatorNode<ExpressionOperator> ast) {
+        assertHasOperator(ast, ExpressionOperator.READ_FIELD);
+        return ast.getArgument(1);
+    }
+
+    @NonNull
+    private IntItem buildGreaterThanOrEquals(
+            OperatorNode<ExpressionOperator> ast) {
+        IntItem number;
+        if (isIndexOnLeftHandSide(ast)) {
+            number = new IntItem("[" + fetchConditionWord(ast) + ";]",
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(1, OperatorNode.class),
+                    number);
+        } else {
+            number = new IntItem("[;" + fetchConditionWord(ast) + "]",
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(0, OperatorNode.class),
+                    number);
+        }
+        return number;
+    }
+
+    @NonNull
+    private IntItem buildLessThanOrEquals(OperatorNode<ExpressionOperator> ast) {
+        IntItem number;
+        if (isIndexOnLeftHandSide(ast)) {
+            number = new IntItem("[;" + fetchConditionWord(ast) + "]",
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(1, OperatorNode.class),
+                    number);
+        } else {
+            number = new IntItem("[" + fetchConditionWord(ast) + ";]",
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(0, OperatorNode.class),
+                    number);
+        }
+        return number;
+    }
+
+    @NonNull
+    private IntItem buildGreaterThan(OperatorNode<ExpressionOperator> ast) {
+        IntItem number;
+        if (isIndexOnLeftHandSide(ast)) {
+            number = new IntItem(">" + fetchConditionWord(ast),
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(1, OperatorNode.class),
+                    number);
+        } else {
+            number = new IntItem("<" + fetchConditionWord(ast),
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(0, OperatorNode.class),
+                    number);
+        }
+        return number;
+    }
+
+    @NonNull
+    private IntItem buildLessThan(OperatorNode<ExpressionOperator> ast) {
+        IntItem number;
+        if (isIndexOnLeftHandSide(ast)) {
+            number = new IntItem("<" + fetchConditionWord(ast),
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(1, OperatorNode.class),
+                    number);
+        } else {
+            number = new IntItem(">" + fetchConditionWord(ast),
+                    fetchConditionIndex(ast));
+            number = leafStyleSettings(ast.getArgument(0, OperatorNode.class),
+                    number);
+        }
+        return number;
+    }
+
+    @NonNull
+    private IntItem buildEquals(OperatorNode<ExpressionOperator> ast) {
+        IntItem number = new IntItem(fetchConditionWord(ast),
+                fetchConditionIndex(ast));
+        if (isIndexOnLeftHandSide(ast)) {
+            number = leafStyleSettings(ast.getArgument(1, OperatorNode.class),
+                    number);
+        } else {
+            number = leafStyleSettings(ast.getArgument(0, OperatorNode.class),
+                    number);
+        }
+        return number;
+    }
+
+    @NonNull
+    private String fetchConditionIndex(OperatorNode<ExpressionOperator> ast) {
+        OperatorNode<ExpressionOperator> lhs = ast.getArgument(0);
+        OperatorNode<ExpressionOperator> rhs = ast.getArgument(1);
+        if (lhs.getOperator() == ExpressionOperator.LITERAL
+                || lhs.getOperator() == ExpressionOperator.NEGATE) {
+            assertHasOperator(rhs, ExpressionOperator.READ_FIELD);
+            return getIndex(rhs);
+        }
+        if (rhs.getOperator() == ExpressionOperator.LITERAL
+                || rhs.getOperator() == ExpressionOperator.NEGATE) {
+            assertHasOperator(lhs, ExpressionOperator.READ_FIELD);
+            return getIndex(lhs);
+        }
+        throw new IllegalArgumentException(
+                "Expected LITERAL and READ_FIELD, got " + lhs.getOperator()
+                        + " and " + rhs.getOperator() + ".");
+    }
+
+    private static String getNumberAsString(OperatorNode<ExpressionOperator> ast) {
+        String negative = "";
+        OperatorNode<ExpressionOperator> currentAst = ast;
+        if (currentAst.getOperator() == ExpressionOperator.NEGATE) {
+            negative = "-";
+            currentAst = currentAst.getArgument(0);
+        }
+        assertHasOperator(currentAst, ExpressionOperator.LITERAL);
+        return negative + currentAst.getArgument(0).toString();
+    }
+
+    @NonNull
+    private static String fetchConditionWord(
+            OperatorNode<ExpressionOperator> ast) {
+        OperatorNode<ExpressionOperator> lhs = ast.getArgument(0);
+        OperatorNode<ExpressionOperator> rhs = ast.getArgument(1);
+        if (lhs.getOperator() == ExpressionOperator.LITERAL
+                || lhs.getOperator() == ExpressionOperator.NEGATE) {
+            assertHasOperator(rhs, ExpressionOperator.READ_FIELD);
+            return getNumberAsString(lhs);
+        }
+        if (rhs.getOperator() == ExpressionOperator.LITERAL
+                || rhs.getOperator() == ExpressionOperator.NEGATE) {
+            assertHasOperator(lhs, ExpressionOperator.READ_FIELD);
+            return getNumberAsString(rhs);
+        }
+        throw new IllegalArgumentException(
+                "Expected LITERAL/NEGATE and READ_FIELD, got "
+                        + lhs.getOperator() + " and " + rhs.getOperator() + ".");
+    }
+
+    private static boolean isIndexOnLeftHandSide(
+            OperatorNode<ExpressionOperator> ast) {
+        return ast.getArgument(0, OperatorNode.class).getOperator() == ExpressionOperator.READ_FIELD;
+    }
+
+    @NonNull
+    private CompositeItem buildAnd(OperatorNode<ExpressionOperator> ast) {
+        AndItem andItem = new AndItem();
+        NotItem notItem = new NotItem();
+        convertVarArgsAnd(ast, 0, andItem, notItem);
+        Preconditions
+                .checkArgument(andItem.getItemCount() > 0,
+                        "Vespa does not support AND with no logically positive branches.");
+        if (notItem.getItemCount() == 0) {
+            return andItem;
+        }
+        if (andItem.getItemCount() == 1) {
+            notItem.setPositiveItem(andItem.getItem(0));
+        } else {
+            notItem.setPositiveItem(andItem);
+        }
+        return notItem;
+    }
+
+    @NonNull
+    private CompositeItem buildOr(OperatorNode<ExpressionOperator> spec) {
+        return convertVarArgs(spec, 0, new OrItem());
+    }
+
+    @NonNull
+    private CompositeItem buildWeakAnd(OperatorNode<ExpressionOperator> spec) {
+        WeakAndItem weakAnd = new WeakAndItem();
+        Integer targetNumHits = getAnnotation(spec, TARGET_NUM_HITS,
+                Integer.class, null, "desired minimum hits to produce");
+        if (targetNumHits != null) {
+            weakAnd.setN(targetNumHits);
+        }
+        Integer scoreThreshold = getAnnotation(spec, SCORE_THRESHOLD,
+                Integer.class, null, "min dot product score for hit inclusion");
+        if (scoreThreshold != null) {
+            weakAnd.setScoreThreshold(scoreThreshold);
+        }
+        return convertVarArgs(spec, 1, weakAnd);
+    }
+
+    @NonNull
+    private CompositeItem buildRank(OperatorNode<ExpressionOperator> spec) {
+        return convertVarArgs(spec, 1, new RankItem());
+    }
+
+    @NonNull
+    private CompositeItem convertVarArgs(OperatorNode<ExpressionOperator> ast,
+            int argIdx, @NonNull
+            CompositeItem out) {
+        Iterable<OperatorNode<ExpressionOperator>> args = ast
+                .getArgument(argIdx);
+        for (OperatorNode<ExpressionOperator> arg : args) {
+            assertHasOperator(arg, ExpressionOperator.class);
+            out.addItem(convertExpression(arg));
+        }
+        return out;
+    }
+
+    private void convertVarArgsAnd(OperatorNode<ExpressionOperator> ast,
+            int argIdx, AndItem outAnd, NotItem outNot) {
+        Iterable<OperatorNode<ExpressionOperator>> args = ast
+                .getArgument(argIdx);
+        for (OperatorNode<ExpressionOperator> arg : args) {
+            assertHasOperator(arg, ExpressionOperator.class);
+            if (arg.getOperator() == ExpressionOperator.NOT) {
+                OperatorNode<ExpressionOperator> exp = arg.getArgument(0);
+                assertHasOperator(exp, ExpressionOperator.class);
+                outNot.addNegativeItem(convertExpression(exp));
+            } else {
+                outAnd.addItem(convertExpression(arg));
+            }
+        }
+    }
+
+    @NonNull
+    private Item buildTermSearch(OperatorNode<ExpressionOperator> ast) {
+        assertHasOperator(ast, ExpressionOperator.CONTAINS);
+        return instantiateLeafItem(
+                getIndex(ast.<OperatorNode<ExpressionOperator>> getArgument(0)),
+                ast.<OperatorNode<ExpressionOperator>> getArgument(1));
+    }
+
+    @NonNull
+    private Item buildRegExpSearch(OperatorNode<ExpressionOperator> ast) {
+        assertHasOperator(ast, ExpressionOperator.MATCHES);
+        String field = getIndex(ast.<OperatorNode<ExpressionOperator>> getArgument(0));
+        OperatorNode<ExpressionOperator> ast1 = ast.<OperatorNode<ExpressionOperator>> getArgument(1);
+        String wordData = getStringContents(ast1);
+        RegExpItem regExp = new RegExpItem(field, true, wordData);
+        return leafStyleSettings(ast1, regExp);
+    }
+
+
+    @NonNull
+    private Item buildRange(OperatorNode<ExpressionOperator> spec) {
+        assertHasOperator(spec, ExpressionOperator.CALL);
+        assertHasFunctionName(spec, RANGE);
+
+        IntItem range = instantiateRangeItem(
+                spec.<List<OperatorNode<ExpressionOperator>>> getArgument(1),
+                spec);
+        return leafStyleSettings(spec, range);
+    }
+
+    private static Number negate(Number x) {
+        if (x.getClass() == Integer.class) {
+            int x1 = x.intValue();
+            return Integer.valueOf(-x1);
+        } else if (x.getClass() == Long.class) {
+            long x1 = x.longValue();
+            return Long.valueOf(-x1);
+        } else if (x.getClass() == Float.class) {
+            float x1 = x.floatValue();
+            return Float.valueOf(-x1);
+        } else if (x.getClass() == Double.class) {
+            double x1 = x.doubleValue();
+            return Double.valueOf(-x1);
+        } else {
+            throw newUnexpectedArgumentException(x.getClass(), Integer.class,
+                    Long.class, Float.class, Double.class);
+        }
+    }
+
+    @NonNull
+    private IntItem instantiateRangeItem(
+            List<OperatorNode<ExpressionOperator>> args,
+            OperatorNode<ExpressionOperator> spec) {
+        Preconditions.checkArgument(args.size() == 3,
+                "Expected 3 arguments, got %s.", args.size());
+
+        Number lowerArg = getBound(args.get(1));
+        Number upperArg = getBound(args.get(2));
+        String bounds = getAnnotation(spec, BOUNDS, String.class, null,
+                "whether bounds should be open or closed");
+        // TODO: add support for implicit transforms
+        if (bounds == null) {
+            return new RangeItem(lowerArg, upperArg, getIndex(args.get(0)));
+        } else {
+            Limit from;
+            Limit to;
+            if (BOUNDS_OPEN.equals(bounds)) {
+                from = new Limit(lowerArg, false);
+                to = new Limit(upperArg, false);
+            } else if (BOUNDS_LEFT_OPEN.equals(bounds)) {
+                from = new Limit(lowerArg, false);
+                to = new Limit(upperArg, true);
+            } else if (BOUNDS_RIGHT_OPEN.equals(bounds)) {
+                from = new Limit(lowerArg, true);
+                to = new Limit(upperArg, false);
+            } else {
+                throw newUnexpectedArgumentException(bounds, BOUNDS_OPEN,
+                        BOUNDS_LEFT_OPEN, BOUNDS_RIGHT_OPEN);
+            }
+            return new IntItem(from, to, getIndex(args.get(0)));
+        }
+    }
+
+    private Number getBound(OperatorNode<ExpressionOperator> bound) {
+        Number boundValue;
+        OperatorNode<ExpressionOperator> currentBound = bound;
+        boolean negate = false;
+        if (currentBound.getOperator() == ExpressionOperator.NEGATE) {
+            currentBound = currentBound.getArgument(0);
+            negate = true;
+        }
+        assertHasOperator(currentBound, ExpressionOperator.LITERAL);
+        boundValue = currentBound.getArgument(0, Number.class);
+        if (negate) {
+            boundValue = negate(boundValue);
+        }
+        return boundValue;
+    }
+
+    @NonNull
+    private Item instantiateLeafItem(String field,
+            OperatorNode<ExpressionOperator> ast) {
+        switch (ast.getOperator()) {
+        case LITERAL:
+        case VARREF:
+            return instantiateWordItem(field, ast, null);
+        case CALL:
+            return instantiateCompositeLeaf(field, ast);
+        default:
+            throw newUnexpectedArgumentException(ast.getOperator().name(),
+                    ExpressionOperator.CALL, ExpressionOperator.LITERAL);
+        }
+    }
+
+    @NonNull
+    private Item instantiateCompositeLeaf(String field,
+            OperatorNode<ExpressionOperator> ast) {
+        List<String> names = ast.getArgument(0);
+        Preconditions.checkArgument(names.size() == 1,
+                "Expected 1 name, got %s.", names.size());
+        switch (names.get(0)) {
+        case PHRASE:
+            return instantiatePhraseItem(field, ast);
+        case NEAR:
+            return instantiateNearItem(field, ast);
+        case ONEAR:
+            return instantiateONearItem(field, ast);
+        case EQUIV:
+            return instantiateEquivItem(field, ast);
+        case ALTERNATIVES:
+            return instantiateWordAlternativesItem(field, ast);
+        default:
+            throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR,
+                    ONEAR, PHRASE);
+        }
+    }
+
+    private Item instantiateWordAlternativesItem(String field, OperatorNode<ExpressionOperator> ast) {
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+        Preconditions.checkArgument(args.size() >= 1, "Expected 1 or more arguments, got %s.", args.size());
+        Preconditions.checkArgument(args.get(0).getOperator() == ExpressionOperator.MAP, "Expected MAP, got %s.", args.get(0)
+                .getOperator());
+
+        List<WordAlternativesItem.Alternative> terms = new ArrayList<>();
+        List<String> keys = args.get(0).getArgument(0);
+        List<OperatorNode<ExpressionOperator>> values = args.get(0).getArgument(1);
+        for (int i = 0; i < keys.size(); ++i) {
+            String term = keys.get(i);
+            double exactness;
+            OperatorNode<ExpressionOperator> value = values.get(i);
+            switch (value.getOperator()) {
+            case LITERAL:
+                exactness = value.getArgument(0, Double.class);
+                break;
+            default:
+                throw newUnexpectedArgumentException(value.getOperator(), ExpressionOperator.LITERAL);
+            }
+            terms.add(new WordAlternativesItem.Alternative(term, exactness));
+        }
+        Substring origin = getOrigin(ast);
+        final Boolean isFromQuery = getAnnotation(ast, IMPLICIT_TRANSFORMS, Boolean.class, Boolean.TRUE,
+                IMPLICIT_TRANSFORMS_DESCRIPTION);
+        return leafStyleSettings(ast, new WordAlternativesItem(field, isFromQuery, origin, terms));
+    }
+
+    @NonNull
+    private Item instantiateEquivItem(String field,
+            OperatorNode<ExpressionOperator> ast) {
+        List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+        Preconditions.checkArgument(args.size() >= 2,
+                "Expected 2 or more arguments, got %s.", args.size());
+
+        EquivItem equiv = new EquivItem();
+        equiv.setIndexName(field);
+        for (OperatorNode<ExpressionOperator> arg : args) {
+            switch (arg.getOperator()) {
+            case LITERAL:
+                equiv.addItem(instantiateWordItem(field, arg, equiv.getClass()));
+                break;
+            case CALL:
+                assertHasFunctionName(arg, PHRASE);
+                equiv.addItem(instantiatePhraseItem(field, arg));
+                break;
+            default:
+                throw newUnexpectedArgumentException(arg.getOperator(),
+                        ExpressionOperator.CALL, ExpressionOperator.LITERAL);
+            }
+        }
+        return leafStyleSettings(ast, equiv);
+    }
+
+    @NonNull
+    private Item instantiateWordItem(String field,
+            OperatorNode<ExpressionOperator> ast, Class<?> parent) {
+        return instantiateWordItem(field, ast, parent, SegmentWhen.POSSIBLY);
+    }
+
+    @NonNull
+    private Item instantiateWordItem(String field,
+            OperatorNode<ExpressionOperator> ast, Class<?> parent,
+            SegmentWhen segmentPolicy) {
+        String wordData = getStringContents(ast);
+        return instantiateWordItem(field, wordData, ast, parent,
+                segmentPolicy, null);
+    }
+
+    @NonNull
+    private Item instantiateWordItem(String field,
+            String rawWord,
+            OperatorNode<ExpressionOperator> ast, Class<?> parent,
+            SegmentWhen segmentPolicy, Language language) {
+        String wordData = rawWord;
+        if (getAnnotation(ast, NFKC, Boolean.class, Boolean.TRUE,
+                "setting for whether to NFKC normalize input data")) {
+            wordData = normalizer.normalize(wordData);
+        }
+        boolean fromQuery = getAnnotation(ast, IMPLICIT_TRANSFORMS,
+                Boolean.class, Boolean.TRUE, IMPLICIT_TRANSFORMS_DESCRIPTION);
+        boolean prefixMatch = getAnnotation(ast, PREFIX, Boolean.class,
+                Boolean.FALSE,
+                "setting for whether to use prefix match of input data");
+        boolean suffixMatch = getAnnotation(ast, SUFFIX, Boolean.class,
+                Boolean.FALSE,
+                "setting for whether to use suffix match of input data");
+        boolean substrMatch = getAnnotation(ast, SUBSTRING, Boolean.class,
+                Boolean.FALSE,
+                "setting for whether to use substring match of input data");
+        Preconditions.checkArgument((prefixMatch ? 1 : 0)
+                + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2,
+                "Only one of prefix, substring and suffix can be set.");
+        @NonNull
+        final TaggableItem wordItem;
+
+        if (prefixMatch) {
+            wordItem = new PrefixItem(wordData, fromQuery);
+        } else if (suffixMatch) {
+            wordItem = new SuffixItem(wordData, fromQuery);
+        } else if (substrMatch) {
+            wordItem = new SubstringItem(wordData, fromQuery);
+        } else {
+            switch (segmentPolicy) {
+            case NEVER:
+                wordItem = new WordItem(wordData, fromQuery);
+                break;
+            case POSSIBLY:
+                if (shouldResegmentWord(field, fromQuery)) {
+                    wordItem = resegment(field, ast, wordData, fromQuery,
+                            parent, language);
+                } else {
+                    wordItem = new WordItem(wordData, fromQuery);
+                }
+                break;
+            case ALWAYS:
+                wordItem = resegment(field, ast, wordData, fromQuery, parent,
+                        language);
+                break;
+            default:
+                throw new IllegalArgumentException(
+                        "Unexpected segmenting rule: " + segmentPolicy);
+            }
+        }
+        if (wordItem instanceof WordItem) {
+            prepareWord(field, ast, fromQuery, (WordItem) wordItem);
+        }
+        return (Item) leafStyleSettings(ast, wordItem);
+    }
+
+    @SuppressWarnings({"deprecation"})
+    private boolean shouldResegmentWord(String field, boolean fromQuery) {
+        return resegment && fromQuery &&  ! indexFactsSession.getIndex(field).isAttribute();
+    }
+
+    @NonNull
+    private TaggableItem resegment(String field,
+            OperatorNode<ExpressionOperator> ast, String wordData,
+            boolean fromQuery, Class<?> parent, Language language) {
+        final TaggableItem wordItem;
+        String toSegment = wordData;
+        final Substring s = getOrigin(ast);
+        final Language usedLanguage = language == null ? currentlyParsing.getLanguage() : language;
+        if (s != null) {
+            toSegment = s.getValue();
+        }
+        List<String> words = segmenter.segment(toSegment,
+                usedLanguage);
+        if (words.size() == 0) {
+            wordItem = new WordItem(wordData, fromQuery);
+        } else if (words.size() == 1 || !phraseArgumentSupported(parent)) {
+            wordItem = new WordItem(words.get(0), fromQuery);
+        } else {
+            wordItem = new PhraseSegmentItem(toSegment, fromQuery, false);
+            ((PhraseSegmentItem) wordItem).setIndexName(field);
+            for (String w : words) {
+                WordItem segment = new WordItem(w, fromQuery);
+                prepareWord(field, ast, fromQuery, segment);
+                ((PhraseSegmentItem) wordItem).addItem(segment);
+            }
+            ((PhraseSegmentItem) wordItem).lock();
+        }
+        return wordItem;
+    }
+
+    private boolean phraseArgumentSupported(Class<?> parent) {
+        if (parent == null) {
+            return true;
+        } else if (parent == PhraseItem.class) {
+            // not supported in backend, but the container flattens the
+            // arguments itself
+            return true;
+        } else if (parent == EquivItem.class) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private void prepareWord(String field,
+            OperatorNode<ExpressionOperator> ast, boolean fromQuery,
+            WordItem wordItem) {
+        wordItem.setIndexName(field);
+        wordStyleSettings(ast, wordItem);
+        if (shouldResegmentWord(field, fromQuery)) {
+            // force re-stemming, new case normalization, etc
+            wordItem.setStemmed(false);
+            wordItem.setLowercased(false);
+            wordItem.setNormalizable(true);
+        }
+    }
+
+    @NonNull
+    private <T extends TaggableItem> T leafStyleSettings(OperatorNode<?> ast,
+            @NonNull
+            T out) {
+        {
+            Map<?, ?> connectivity = getAnnotation(ast, CONNECTIVITY,
+                    Map.class, null, "connectivity settings");
+            if (connectivity != null) {
+                connectedItems.add(new ConnectedItem(out, getMapValue(
+                        CONNECTIVITY, connectivity, CONNECTION_ID,
+                        Integer.class), getMapValue(CONNECTIVITY, connectivity,
+                        CONNECTION_WEIGHT, Number.class).doubleValue()));
+            }
+            Number significance = getAnnotation(ast, SIGNIFICANCE,
+                    Number.class, null, "term significance");
+            if (significance != null) {
+                out.setSignificance(significance.doubleValue());
+            }
+            Integer uniqueId = getAnnotation(ast, UNIQUE_ID, Integer.class,
+                    null, "term ID", false);
+            if (uniqueId != null) {
+                out.setUniqueID(uniqueId);
+                identifiedItems.put(uniqueId, out);
+            }
+        }
+        {
+            Item leaf = (Item) out;
+            Map<?, ?> itemAnnotations = getAnnotation(ast, ANNOTATIONS,
+                    Map.class, Collections.emptyMap(), "item annotation map");
+            for (Map.Entry<?, ?> entry : itemAnnotations.entrySet()) {
+                Preconditions.checkArgument(entry.getKey() instanceof String,
+                        "Expected String annotation key, got %s.", entry
+                                .getKey().getClass());
+                Preconditions.checkArgument(entry.getValue() instanceof String,
+                        "Expected String annotation value, got %s.", entry
+                                .getValue().getClass());
+                leaf.addAnnotation((String) entry.getKey(), entry.getValue());
+            }
+            Boolean filter = getAnnotation(ast, FILTER, Boolean.class, null,
+                    FILTER_DESCRIPTION);
+            if (filter != null) {
+                leaf.setFilter(filter);
+            }
+            Boolean isRanked = getAnnotation(ast, RANKED, Boolean.class, null,
+                    RANKED_DESCRIPTION);
+            if (isRanked != null) {
+                leaf.setRanked(isRanked);
+            }
+            String label = getAnnotation(ast, LABEL, String.class, null,
+                    "item label");
+            if (label != null) {
+                leaf.setLabel(label);
+            }
+            Integer weight = getAnnotation(ast, WEIGHT, Integer.class, null,
+                    "term weight for ranking");
+            if (weight != null) {
+                leaf.setWeight(weight);
+            }
+        }
+        if (out instanceof IntItem) {
+            IntItem number = (IntItem) out;
+            Integer hitLimit = getCappedRangeSearchParameter(ast);
+            if (hitLimit != null) {
+                number.setHitLimit(hitLimit.intValue());
+            }
+        }
+
+        return out;
+    }
+
+    private Integer getCappedRangeSearchParameter(OperatorNode<?> ast) {
+        Integer hitLimit = getAnnotation(ast, HIT_LIMIT, Integer.class, null, "hit limit");
+
+        if (hitLimit != null) {
+            Boolean ascending = getAnnotation(ast, ASCENDING_HITS_ORDER, Boolean.class, null,
+                    "ascending population ordering for capped range search");
+            Boolean descending = getAnnotation(ast, DESCENDING_HITS_ORDER, Boolean.class, null,
+                    "descending population ordering for capped range search");
+            Preconditions.checkArgument(ascending == null || descending == null,
+                    "Settings for both ascending and descending ordering set, only one of these expected.");
+            if (Boolean.TRUE.equals(descending) || Boolean.FALSE.equals(ascending)) {
+                hitLimit = Integer.valueOf(hitLimit.intValue() * -1);
+            }
+        }
+        return hitLimit;
+    }
+
+    @Beta
+    public boolean isQueryParser() {
+        return queryParser;
+    }
+
+    @Beta
+    public void setQueryParser(boolean queryParser) {
+        this.queryParser = queryParser;
+    }
+
+    @Beta
+    public void setUserQuery(@NonNull Query userQuery) {
+        this.userQuery = userQuery;
+    }
+
+    @Beta
+    public Set<String> getYqlSummaryFields() {
+        return yqlSummaryFields;
+    }
+
+    @Beta
+    public List<VespaGroupingStep> getGroupingSteps() {
+        return groupingSteps;
+    }
+
+    /**
+     * Give the offset expected from the latest parsed query if anything is
+     * explicitly specified.
+     *
+     * @return an Integer instance or null
+     */
+    public Integer getOffset() {
+        return offset;
+    }
+
+    /**
+     * Give the number of hits expected from the latest parsed query if anything
+     * is explicitly specified.
+     *
+     * @return an Integer instance or null
+     */
+    public Integer getHits() {
+        return hits;
+    }
+
+    /**
+     * The timeout specified in the YQL+ query last parsed.
+     *
+     * @return an Integer instance or null
+     */
+    public Integer getTimeout() {
+        return timeout;
+    }
+
+    /**
+     * The sorting specified in the YQL+ query last parsed.
+     *
+     * @return a Sorting instance or null
+     */
+    public Sorting getSorting() {
+        return sorting;
+    }
+
+    Set<String> getDocTypes() {
+        return docTypes;
+    }
+
+    Set<String> getYqlSources() {
+        return yqlSources;
+    }
+
+    private static void assertHasOperator(OperatorNode<?> ast,
+            Class<? extends Operator> expectedOperatorClass) {
+        Preconditions.checkArgument(
+                expectedOperatorClass.isInstance(ast.getOperator()),
+                "Expected operator class %s, got %s.",
+                expectedOperatorClass.getName(), ast.getOperator().getClass()
+                        .getName());
+    }
+
+    private static void assertHasOperator(OperatorNode<?> ast,
+            Operator expectedOperator) {
+        Preconditions.checkArgument(ast.getOperator() == expectedOperator,
+                "Expected operator %s, got %s.", expectedOperator,
+                ast.getOperator());
+    }
+
+    private static void assertHasFunctionName(OperatorNode<?> ast,
+            String expectedFunctionName) {
+        List<String> names = ast.getArgument(0);
+        Preconditions.checkArgument(expectedFunctionName.equals(names.get(0)),
+                "Expected function '%s', got '%s'.", expectedFunctionName,
+                names.get(0));
+    }
+
+    private static void addItems(OperatorNode<ExpressionOperator> ast,
+            WeightedSetItem out) {
+        switch (ast.getOperator()) {
+        case MAP:
+            addStringItems(ast, out);
+            break;
+        case ARRAY:
+            addLongItems(ast, out);
+            break;
+        default:
+            throw newUnexpectedArgumentException(ast.getOperator(),
+                    ExpressionOperator.ARRAY, ExpressionOperator.MAP);
+        }
+    }
+
+    private static void addStringItems(OperatorNode<ExpressionOperator> ast,
+            WeightedSetItem out) {
+        List<String> keys = ast.getArgument(0);
+        List<OperatorNode<ExpressionOperator>> values = ast.getArgument(1);
+        for (int i = 0; i < keys.size(); ++i) {
+            OperatorNode<ExpressionOperator> tokenWeight = values.get(i);
+            assertHasOperator(tokenWeight, ExpressionOperator.LITERAL);
+            out.addToken(keys.get(i), tokenWeight.getArgument(0, Integer.class));
+        }
+    }
+
+    private static void addLongItems(OperatorNode<ExpressionOperator> ast,
+            WeightedSetItem out) {
+        List<OperatorNode<ExpressionOperator>> values = ast.getArgument(0);
+        for (OperatorNode<ExpressionOperator> value : values) {
+            assertHasOperator(value, ExpressionOperator.ARRAY);
+            List<OperatorNode<ExpressionOperator>> args = value.getArgument(0);
+            Preconditions.checkArgument(args.size() == 2,
+                    "Expected item and weight, got %s.", args);
+
+            OperatorNode<ExpressionOperator> tokenValueNode = args.get(0);
+            assertHasOperator(tokenValueNode, ExpressionOperator.LITERAL);
+            Number tokenValue = tokenValueNode.getArgument(0, Number.class);
+            Preconditions.checkArgument(tokenValue instanceof Integer
+                    || tokenValue instanceof Long,
+                    "Expected Integer or Long, got %s.", tokenValue.getClass()
+                            .getName());
+
+            OperatorNode<ExpressionOperator> tokenWeightNode = args.get(1);
+            assertHasOperator(tokenWeightNode, ExpressionOperator.LITERAL);
+            Integer tokenWeight = tokenWeightNode.getArgument(0, Integer.class);
+
+            out.addToken(tokenValue.longValue(), tokenWeight);
+        }
+    }
+
+    private void wordStyleSettings(OperatorNode<ExpressionOperator> ast,
+            WordItem out) {
+        Substring origin = getOrigin(ast);
+        if (origin != null) {
+            out.setOrigin(origin);
+        }
+        Boolean usePositionData = getAnnotation(ast, USE_POSITION_DATA,
+                Boolean.class, null,
+                USE_POSITION_DATA_DESCRIPTION);
+        if (usePositionData != null) {
+            out.setPositionData(usePositionData);
+        }
+        Boolean stem = getAnnotation(ast, STEM, Boolean.class, null,
+                STEM_DESCRIPTION);
+        if (stem != null) {
+            out.setStemmed(!stem);
+        }
+        Boolean normalizeCase = getAnnotation(ast, NORMALIZE_CASE,
+                Boolean.class, null,
+                NORMALIZE_CASE_DESCRIPTION);
+        if (normalizeCase != null) {
+            out.setLowercased(!normalizeCase);
+        }
+        Boolean accentDrop = getAnnotation(ast, ACCENT_DROP, Boolean.class,
+                null,
+                ACCENT_DROP_DESCRIPTION);
+        if (accentDrop != null) {
+            out.setNormalizable(accentDrop);
+        }
+        Boolean andSegmenting = getAnnotation(ast, AND_SEGMENTING,
+                Boolean.class, null,
+                "setting for whether to force using AND for segments on and off");
+        if (andSegmenting != null) {
+            if (andSegmenting) {
+                out.setSegmentingRule(SegmentingRule.BOOLEAN_AND);
+            } else {
+                out.setSegmentingRule(SegmentingRule.PHRASE);
+            }
+        }
+    }
+
+    @NonNull
+    private String getIndex(OperatorNode<ExpressionOperator> operatorNode) {
+        String index = fetchFieldRead(operatorNode);
+        Preconditions.checkArgument(indexFactsSession.isIndex(index), "Field '%s' does not exist.", index);
+        return indexFactsSession.getCanonicName(index);
+    }
+
+    private Substring getOrigin(OperatorNode<ExpressionOperator> ast) {
+        Map<?, ?> origin = getAnnotation(ast, ORIGIN, Map.class, null,
+                ORIGIN_DESCRIPTION);
+        if (origin == null) {
+            return null;
+        }
+        String original = getMapValue(ORIGIN, origin, ORIGIN_ORIGINAL,
+                String.class);
+        int offset = getMapValue(ORIGIN, origin, ORIGIN_OFFSET, Integer.class);
+        int length = getMapValue(ORIGIN, origin, ORIGIN_LENGTH, Integer.class);
+        return new Substring(offset, length + offset, original);
+    }
+
+    private static <T> T getMapValue(String mapName, Map<?, ?> map, String key,
+            Class<T> expectedValueClass) {
+        Object value = map.get(key);
+        Preconditions.checkArgument(value != null,
+                "Map annotation '%s' must contain an entry with key '%s'.",
+                mapName, key);
+        assert value != null;
+        Preconditions.checkArgument(expectedValueClass.isInstance(value),
+                "Expected %s for entry '%s' in map annotation '%s', got %s.",
+                expectedValueClass.getName(), key, mapName, value.getClass()
+                        .getName());
+        return expectedValueClass.cast(value);
+    }
+
+    private <T> T getAnnotation(OperatorNode<?> ast, String key,
+    Class<T> expectedClass, T defaultValue, String description) {
+        return getAnnotation(ast, key, expectedClass, defaultValue,
+                description, true);
+    }
+
+    private <T> T getAnnotation(OperatorNode<?> ast, String key,
+            Class<T> expectedClass, T defaultValue, String description, boolean considerParents) {
+        Object value = ast.getAnnotation(key);
+        for (Iterator<OperatorNode<?>> i = annotationStack.iterator(); value == null
+                && considerParents && i.hasNext();) {
+            value = i.next().getAnnotation(key);
+        }
+        if (value == null) {
+            return defaultValue;
+        }
+        Preconditions.checkArgument(expectedClass.isInstance(value),
+                "Expected %s for annotation '%s' (%s), got %s.", expectedClass
+                        .getName(), key, description, value.getClass()
+                        .getName());
+        return expectedClass.cast(value);
+    }
+
+    private static IllegalArgumentException newUnexpectedArgumentException(
+            Object actual, Object... expected) {
+        StringBuilder out = new StringBuilder("Expected ");
+        for (int i = 0, len = expected.length; i < len; ++i) {
+            out.append(expected[i]);
+            if (i < len - 2) {
+                out.append(", ");
+            } else if (i < len - 1) {
+                out.append(" or ");
+            }
+        }
+        out.append(", got ").append(actual).append(".");
+        return new IllegalArgumentException(out.toString());
+    }
+
+    String getSegmenterBackend() {
+        return segmenterBackend;
+    }
+
+    Version getSegmenterVersion() {
+        return segmenterVersion;
+    }
+
+    private static final class ConnectedItem {
+
+        final double weight;
+        final int toId;
+        final TaggableItem fromItem;
+
+        ConnectedItem(TaggableItem fromItem, int toId, double weight) {
+            this.weight = weight;
+            this.toId = toId;
+            this.fromItem = fromItem;
+        }
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlQuery.java b/container-search/src/main/java/com/yahoo/search/yql/YqlQuery.java
new file mode 100644
index 00000000000..27c27b88d24
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlQuery.java
@@ -0,0 +1,22 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.yql;
+
+/**
+ * A Yql query. These usually contains variables, which allows the yql query to be parsed once at configuration
+ * time and turned into fully specified queries at request time without reparsing.
+ *
+ * @author bratseth
+ */
+// TODO: This is just a skeleton
+public class YqlQuery {
+
+    private YqlQuery(String yqlQuery) {
+        // TODO
+    }
+
+    /** Creates a YQl query form a string */
+    public static YqlQuery from(String yqlQueryString) {
+        return new YqlQuery(yqlQueryString);
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/package-info.java b/container-search/src/main/java/com/yahoo/search/yql/package-info.java
new file mode 100644
index 00000000000..79cf983e471
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/yql/package-info.java
@@ -0,0 +1,11 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * YQL+ integration.
+ *
+ * <p>Not a public API.</p>
+ */
+@ExportPackage
+package com.yahoo.search.yql;
+
+import com.yahoo.osgi.annotation.ExportPackage;
+