Merge pull request #6496 from vespa-engine/henrhoi/api-for-new-parameter-with-yql-and-grouping

Henrhoi/api-for-new-parameter-with-yql-and-grouping

13 files changed, 2171 insertions, 6 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 4c7005811ee..cfb1c9a26be 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -20,6 +20,7 @@ import com.yahoo.search.query.Presentation;
 import com.yahoo.search.query.Properties;
 import com.yahoo.search.query.QueryTree;
 import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.Select;
 import com.yahoo.search.query.SessionId;
 import com.yahoo.search.query.Sorting;
 import com.yahoo.search.query.Sorting.AttributeSorter;
@@ -100,7 +101,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         ADVANCED(3,"adv"),
         WEB(4,"web"),
         PROGRAMMATIC(5, "prog"),
-        YQL(6, "yql");
+        YQL(6, "yql"),
+        SELECT(7, "select");;
 
         private final int intValue;
         private final String stringValue;
@@ -170,6 +172,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
     /** How results of this query should be presented */
     private Presentation presentation = new Presentation(this);
 
+    /** The selection of where-clause and grouping */
+    private Select select = new Select(this);
+
     //---------------- Tracing ----------------------------------------------------
 
     private static Logger log = Logger.getLogger(Query.class.getName());
@@ -207,6 +212,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType())));
         argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType())));
         argumentType.addField(new FieldDescription(Model.MODEL, new QueryProfileFieldType(Model.getArgumentType())));
+        argumentType.addField(new FieldDescription(Select.SELECT, new QueryProfileFieldType(Select.getArgumentType())));
         argumentType.freeze();
     }
     public static QueryProfileType getArgumentType() { return argumentType; }
@@ -220,6 +226,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         addAliases(Ranking.getArgumentType(), propertyAliasesBuilder);
         addAliases(Model.getArgumentType(), propertyAliasesBuilder);
         addAliases(Presentation.getArgumentType(), propertyAliasesBuilder);
+        addAliases(Select.getArgumentType(), propertyAliasesBuilder);
         propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder);
     }
     private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) {
@@ -239,6 +246,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         registry.register(Query.getArgumentType().unfrozen());
         registry.register(Ranking.getArgumentType().unfrozen());
         registry.register(Model.getArgumentType().unfrozen());
+        registry.register(Select.getArgumentType().unfrozen());
         registry.register(Presentation.getArgumentType().unfrozen());
         registry.register(DefaultProperties.argumentType.unfrozen());
     }
@@ -383,6 +391,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         setFrom(properties,Model.getArgumentType(), context);
         setFrom(properties,Presentation.getArgumentType(), context);
         setFrom(properties,Ranking.getArgumentType(), context);
+        setFrom(properties, Select.getArgumentType(), context);
     }
 
     /**
@@ -982,6 +991,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
     /** Returns the presentation to be used for this query, never null */
     public Presentation getPresentation() { return presentation; }
 
+    /** Returns the select to be used for this query, never null */
+    public Select getSelect() { return select; }
+
     /** Returns the ranking to be used for this query, never null */
     public Ranking getRanking() { return ranking; }
 
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
index b59578ab6a3..fdbee9c8f11 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
@@ -140,6 +140,8 @@ public class VespaSearcher extends ConfiguredHTTPProviderSearcher {
             return Query.Type.PROGRAMMATIC;
         } else if (providerQueryType == ProviderConfig.QueryType.YQL) {
             return Query.Type.YQL;
+        } else if (providerQueryType == ProviderConfig.QueryType.SELECT) {
+            return Query.Type.SELECT;
         } else {
             throw new RuntimeException("Query type " + providerQueryType + " unsupported.");
         }
diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
index 9eaecea008e..5ac1f834031 100644
--- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
+++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
@@ -580,8 +580,18 @@ public class SearchHandler extends LoggingRequestHandler {
             // Create request-mapping
             Map<String, String> requestMap = new HashMap<>();
             createRequestMapping(inspector, requestMap, "");
-            return requestMap;
 
+            // Throws QueryException if query contains both yql- and select-parameter
+            if (requestMap.containsKey("yql") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) {
+                throw new QueryException("Illegal query: Query contains both yql- and select-parameter");
+            }
+
+            // Throws QueryException if query contains both query- and select-parameter
+            if (requestMap.containsKey("query") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) {
+                throw new QueryException("Illegal query: Query contains both query- and select-parameter");
+            }
+
+            return requestMap;
 
         } else {
             return request.propertyMap();
@@ -609,6 +619,10 @@ public class SearchHandler extends LoggingRequestHandler {
                     map.put(qualifiedKey, value.asString());
                     break;
                 case OBJECT:
+                    if (qualifiedKey.equals("select.where") || qualifiedKey.equals("select.grouping")){
+                        map.put(qualifiedKey, value.toString());
+                        break;
+                    }
                     createRequestMapping(value, map, qualifiedKey+".");
                     break;
             }
diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java
index 167bb312f61..ddd33cb7c78 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Model.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Model.java
@@ -249,7 +249,7 @@ public class Model implements Cloneable {
     public String getQueryString() { return queryString; }
 
     /**
-     * Returns the query as an object structure.
+     * Returns the query as an object structure. Remember to have the correct Query.Type set.
      * This causes parsing of the query string if it has changed since this was last called
      * (i.e query parsing is lazy)
      */
diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java
new file mode 100644
index 00000000000..3ffc6bddb24
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/Select.java
@@ -0,0 +1,110 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.yql.VespaGroupingStep;
+
+
+
+/**
+ * The parameters defining the where-clause and groping of a query
+ *
+ * @author henrhoi
+ */
+public class Select implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+    private static final CompoundName argumentTypeName;
+
+    public static final String SELECT = "select";
+    public static final String WHERE = "where";
+    public static final String GROUPING = "grouping";
+
+
+    private static Model model;
+    private Query parent;
+    private String where = "";
+    private String grouping = "";
+
+    static {
+        argumentType = new QueryProfileType(SELECT);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(WHERE, "string", "where"));
+        argumentType.addField(new FieldDescription(GROUPING, "string", "grouping"));
+        argumentType.freeze();
+        argumentTypeName=new CompoundName(argumentType.getId().getName());
+    }
+
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    public Select(String where, String grouping){
+        this.where = where;
+        this.grouping = grouping;
+    }
+
+    public Select(Query query) {
+        setParent(query);
+        model = query.getModel();
+    }
+
+
+    /** Returns the query owning this, never null */
+    private Query getParent() { return parent; }
+
+
+    /** Assigns the query owning this */
+    public void setParent(Query parent) {
+        if (parent==null) throw new NullPointerException("A query models owner cannot be null");
+        this.parent = parent;
+    }
+
+
+    /** Set the where-clause for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */
+    public void setWhere(String where) {
+        this.where = where;
+        model.setType(SELECT);
+
+        // Setting the queryTree to null
+        model.setQueryString(null);
+    }
+
+
+    /** Returns the where-clause in the query */
+    public String getWhereString(){
+        return this.where;
+    }
+
+
+    /** Set the grouping-string for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */
+    public void setGrouping(String grouping){
+        this.grouping = grouping;
+        SelectParser parser = (SelectParser) ParserFactory.newInstance(Query.Type.SELECT, new ParserEnvironment());
+
+        for (VespaGroupingStep step : parser.getGroupingSteps(grouping)) {
+            GroupingRequest.newInstance(parent)
+                    .setRootOperation(step.getOperation())
+                    .continuations().addAll(step.continuations());
+        }
+    }
+
+
+    /** Returns the grouping in the query */
+    public String getGroupingString(){
+        return this.grouping;
+    }
+
+
+    @Override
+    public String toString() {
+        return "where: [" + where + "], grouping: [" + grouping+ "]";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
new file mode 100644
index 00000000000..13ebacb62ef
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
@@ -0,0 +1,1185 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+
+import com.google.common.base.Preconditions;
+import com.yahoo.collections.LazyMap;
+import com.yahoo.language.Language;
+import com.yahoo.language.process.Normalizer;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.DotProductItem;
+import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.ExactStringItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.Limit;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.prelude.query.NotItem;
+import com.yahoo.prelude.query.ONearItem;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PredicateQueryItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.QueryException;
+import com.yahoo.prelude.query.RangeItem;
+import com.yahoo.prelude.query.RankItem;
+import com.yahoo.prelude.query.RegExpItem;
+import com.yahoo.prelude.query.SameElementItem;
+import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.SubstringItem;
+import com.yahoo.prelude.query.SuffixItem;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.prelude.query.WandItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WeightedSetItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.query.parser.Parsable;
+import com.yahoo.search.query.parser.Parser;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.yql.VespaGroupingStep;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.vespa.config.SlimeUtils;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.slime.Type.ARRAY;
+import static com.yahoo.slime.Type.DOUBLE;
+import static com.yahoo.slime.Type.LONG;
+import static com.yahoo.slime.Type.OBJECT;
+import static com.yahoo.slime.Type.STRING;
+
+/**
+ * The Select query language.
+ *
+ * This class will be parsing the Select parameters, and will be used when the query has the SELECT-type.
+ *
+ * @author henrhoi
+ */
+
+
+public class SelectParser implements Parser {
+
+    Parsable query;
+    private final IndexFacts indexFacts;
+    private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap();
+    private final List<ConnectedItem> connectedItems = new ArrayList<>();
+    private final Normalizer normalizer;
+    private final ParserEnvironment environment;
+    private IndexFacts.Session indexFactsSession;
+
+
+
+    /** YQL parameters and functions */
+
+    private static final String DESCENDING_HITS_ORDER = "descending";
+    private static final String ASCENDING_HITS_ORDER = "ascending";
+    private static final Integer DEFAULT_TARGET_NUM_HITS = 10;
+    private static final String ORIGIN_LENGTH = "length";
+    private static final String ORIGIN_OFFSET = "offset";
+    private static final String ORIGIN = "origin";
+    private static final String ORIGIN_ORIGINAL = "original";
+    private static final String CONNECTION_ID = "id";
+    private static final String CONNECTION_WEIGHT = "weight";
+    private static final String CONNECTIVITY = "connectivity";
+    private static final String ANNOTATIONS = "annotations";
+    private static final String NFKC = "nfkc";
+    private static final String USER_INPUT_LANGUAGE = "language";
+    private static final String ACCENT_DROP = "accentDrop";
+    private static final String ALTERNATIVES = "alternatives";
+    private static final String AND_SEGMENTING = "andSegmenting";
+    private static final String DISTANCE = "distance";
+    private static final String DOT_PRODUCT = "dotProduct";
+    private static final String EQUIV = "equiv";
+    private static final String FILTER = "filter";
+    private static final String HIT_LIMIT = "hitLimit";
+    private static final String IMPLICIT_TRANSFORMS = "implicitTransforms";
+    private static final String LABEL = "label";
+    private static final String NEAR = "near";
+    private static final String NORMALIZE_CASE = "normalizeCase";
+    private static final String ONEAR = "onear";
+    private static final String PHRASE = "phrase";
+    private static final String PREDICATE = "predicate";
+    private static final String PREFIX = "prefix";
+    private static final String RANKED = "ranked";
+    private static final String RANK = "rank";
+    private static final String SAME_ELEMENT = "sameElement";
+    private static final String SCORE_THRESHOLD = "scoreThreshold";
+    private static final String SIGNIFICANCE = "significance";
+    private static final String STEM = "stem";
+    private static final String SUBSTRING = "substring";
+    private static final String SUFFIX = "suffix";
+    private static final String TARGET_NUM_HITS = "targetNumHits";
+    private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor";
+    private static final String UNIQUE_ID = "id";
+    private static final String USE_POSITION_DATA = "usePositionData";
+    private static final String WAND = "wand";
+    private static final String WEAK_AND = "weakAnd";
+    private static final String WEIGHTED_SET = "weightedSet";
+    private static final String WEIGHT = "weight";
+    private static final String AND = "and";
+    private static final String AND_NOT = "and_not";
+    private static final String OR = "or";
+    private static final String EQ = "equals";
+    private static final String RANGE = "range";
+    private static final String CONTAINS = "contains";
+    private static final String MATCHES = "matches";
+    private static final String CALL = "call";
+    private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND);
+
+    /**************************************/
+
+
+
+    public SelectParser(ParserEnvironment environment) {
+        indexFacts = environment.getIndexFacts();
+        normalizer = environment.getLinguistics().getNormalizer();
+
+        this.environment = environment;
+    }
+
+
+    @Override
+    public QueryTree parse(Parsable query) {
+        indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict());
+        connectedItems.clear();
+        identifiedItems.clear();
+        this.query = query;
+
+        return buildTree();
+    }
+
+
+
+    private QueryTree buildTree() {
+        Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get();
+        if (inspector.field("error_message").valid()){
+            throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8));
+        }
+
+        Item root = walkJson(inspector);
+        connectItems();
+        QueryTree newTree = new QueryTree(root);
+
+        return newTree;
+    }
+
+
+    private Item walkJson(Inspector inspector){
+        final Item[] item = {null};
+        inspector.traverse((ObjectTraverser) (key, value) -> {
+            String type = (FUNCTION_CALLS.contains(key)) ? CALL : key;
+
+            switch (type) {
+
+                case AND:
+                    item[0] = buildAnd(key, value);
+                    break;
+                case AND_NOT:
+                    item[0] = buildNotAnd(key, value);
+                    break;
+                case OR:
+                    item[0] = buildOr(key, value);
+                    break;
+                case EQ:
+                    item[0] = buildEquals(key, value);
+                    break;
+                case RANGE:
+                    item[0] = buildRange(key, value);
+                    break;
+                case CONTAINS:
+                    item[0] = buildTermSearch(key, value);
+                    break;
+                case MATCHES:
+                    item[0] = buildRegExpSearch(key, value);
+                    break;
+                case CALL:
+                    item[0] = buildFunctionCall(key, value);
+                    break;
+                default:
+                    throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT);
+            }
+        });
+        return item[0];
+    }
+
+
+    public List<VespaGroupingStep> getGroupingSteps(String grouping){
+        List<VespaGroupingStep> groupingSteps = new ArrayList<>();
+        List<String> groupingOperations = getOperations(grouping);
+        for (String groupingString : groupingOperations){
+            GroupingOperation groupingOperation = GroupingOperation.fromString(groupingString);
+            VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation);
+            groupingSteps.add(groupingStep);
+        }
+        return groupingSteps;
+    }
+
+    private List<String> getOperations(String grouping) {
+        List<String> operations = new ArrayList<>();
+        Inspector inspector = SlimeUtils.jsonToSlime(grouping.getBytes()).get();
+        if (inspector.field("error_message").valid()){
+            throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8));
+        }
+
+        inspector.traverse( (ArrayTraverser) (key, value) -> {
+            String groupingString = value.toString();
+            groupingString = groupingString.replace(" ", "").replace("\"", "").replace("\'", "").replace(":{", "(").replace(":", "(").replace("}", ")").replace(",", ")");
+            groupingString = groupingString.substring(1, groupingString.length());
+            operations.add(groupingString);
+        });
+
+        return operations;
+
+    }
+
+
+    @NonNull
+    private Item buildFunctionCall(String key, Inspector value) {
+        switch (key) {
+            case WAND:
+                return buildWand(key, value);
+            case WEIGHTED_SET:
+                return buildWeightedSet(key, value);
+            case DOT_PRODUCT:
+                return buildDotProduct(key, value);
+            case PREDICATE:
+                return buildPredicate(key, value);
+            case RANK:
+                return buildRank(key, value);
+            case WEAK_AND:
+                return buildWeakAnd(key, value);
+            default:
+                throw newUnexpectedArgumentException(key, DOT_PRODUCT, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE);
+        }
+    }
+
+
+    private void addItemsFromInspector(CompositeItem item, Inspector inspector){
+        if (inspector.type() == ARRAY){
+            inspector.traverse((ArrayTraverser) (index, new_value) -> {
+                item.addItem(walkJson(new_value));
+            });
+
+        } else if (inspector.type() == OBJECT){
+            if (inspector.field("children").valid()){
+                inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> {
+                    item.addItem(walkJson(new_value));
+                });
+            }
+
+        }
+    }
+
+
+    private Inspector getChildren(Inspector inspector){
+        if (inspector.type() == ARRAY){
+            return inspector;
+
+        } else if (inspector.type() == OBJECT){
+            if (inspector.field("children").valid()){
+                return inspector.field("children");
+            }
+            if (inspector.field(1).valid()){
+                return inspector.field(1);
+            }
+        }
+        return null;
+    }
+
+
+    private HashMap<Integer, Inspector> getChildrenMap(Inspector inspector){
+        HashMap<Integer, Inspector> children = new HashMap<>();
+            if (inspector.type() == ARRAY){
+                inspector.traverse((ArrayTraverser) (index, new_value) -> {
+                    children.put(index, new_value);
+                });
+
+            } else if (inspector.type() == OBJECT){
+                if (inspector.field("children").valid()){
+                    inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> {
+                        children.put(index, new_value);
+                    });
+                }
+            }
+        return children;
+    }
+
+
+    private Inspector getAnnotations(Inspector inspector){
+        if (inspector.type() == OBJECT && inspector.field("attributes").valid()){
+            return inspector.field("attributes");
+        }
+        return null;
+    }
+
+
+    private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){
+        HashMap<String, Inspector> attributes = new HashMap<>();
+        if (annotation.type() == OBJECT){
+            annotation.traverse((ObjectTraverser) (index, new_value) -> {
+                attributes.put(index, new_value);
+            });
+        }
+        return attributes;
+    }
+
+
+    private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){
+        HashMap<String, Inspector> attributes = new HashMap<>();
+        if (inspector.type() == OBJECT && inspector.field("attributes").valid()){
+            inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> {
+                attributes.put(index, new_value);
+            });
+        }
+        return attributes;
+    }
+
+
+    private <T> T getAnnotation(String annotationName, HashMap<String, Inspector> annotations, Class<T> expectedClass, T defaultValue) {
+        return (annotations.get(annotationName) == null) ? defaultValue : expectedClass.cast(annotations.get(annotationName).asString());
+    }
+
+
+    private Boolean getBoolAnnotation(String annotationName, HashMap<String, Inspector> annotations, Boolean defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return annotation.asBool();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Integer getIntegerAnnotation(String annotationName, HashMap<String, Inspector> annotations, Integer defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return (int)annotation.asLong();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Double getDoubleAnnotation(String annotationName, HashMap<String, Inspector> annotations, Double defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return annotation.asDouble();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Inspector getAnnotationAsInspectorOrNull(String annotationName, HashMap<String, Inspector> annotations) {
+        return annotations.get(annotationName);
+    }
+
+
+    @NonNull
+    private CompositeItem buildAnd(String key, Inspector value) {
+        AndItem andItem = new AndItem();
+        addItemsFromInspector(andItem, value);
+
+        return andItem;
+    }
+
+
+    @NonNull
+    private CompositeItem buildNotAnd(String key, Inspector value) {
+        NotItem notItem = new NotItem();
+        addItemsFromInspector(notItem, value);
+
+        return notItem;
+    }
+
+
+    @NonNull
+    private CompositeItem buildOr(String key, Inspector value) {
+        OrItem orItem = new OrItem();
+        addItemsFromInspector(orItem, value);
+        return orItem;
+    }
+
+
+    @NonNull
+    private CompositeItem buildWeakAnd(String key, Inspector value) {
+        WeakAndItem weakAnd = new WeakAndItem();
+        addItemsFromInspector(weakAnd, value);
+        Inspector annotations = getAnnotations(value);
+
+        if (annotations != null){
+            annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+                if (TARGET_NUM_HITS.equals(annotation_name)){
+                    weakAnd.setN((int)(annotation_value.asDouble()));
+                }
+                if (SCORE_THRESHOLD.equals(annotation_name)){
+                    weakAnd.setScoreThreshold((int)(annotation_value.asDouble()));
+                }
+            });
+        }
+
+        return weakAnd;
+    }
+
+
+    @NonNull
+    private <T extends TaggableItem> T leafStyleSettings(Inspector annotations, @NonNull T out) {
+        {
+            if (annotations != null) {
+                Inspector  itemConnectivity= getAnnotationAsInspectorOrNull(CONNECTIVITY, getAnnotationMapFromAnnotationInspector(annotations));
+                if (itemConnectivity != null) {
+                    Integer[] id = {null};
+                    Double[] weight = {null};
+                    itemConnectivity.traverse((ObjectTraverser) (key, value) -> {
+                        switch (key){
+                            case CONNECTION_ID:
+                                id[0] = (int) value.asLong();
+                                break;
+                            case CONNECTION_WEIGHT:
+                                weight[0] = value.asDouble();
+                                break;
+                        }
+                    });
+                    connectedItems.add(new ConnectedItem(out, id[0], weight[0]));
+                }
+
+                annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+
+                    if (SIGNIFICANCE.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            out.setSignificance(annotation_value.asDouble());
+                        }
+                    }
+                    if (UNIQUE_ID.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            out.setUniqueID((int)annotation_value.asLong());
+                            identifiedItems.put((int)annotation_value.asLong(), out);
+                        }
+                    }
+                });
+            }
+        }
+        {
+            Item leaf = (Item) out;
+            if (annotations != null) {
+                Inspector itemAnnotations = getAnnotationAsInspectorOrNull(ANNOTATIONS, getAnnotationMapFromAnnotationInspector(annotations));
+                if (itemAnnotations != null) {
+                    itemAnnotations.traverse((ObjectTraverser) (key, value) -> {
+                        leaf.addAnnotation(key, value.asString());
+                    });
+                }
+
+                annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+                    if (FILTER.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setFilter(annotation_value.asBool());
+                        }
+                    }
+                    if (RANKED.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setRanked(annotation_value.asBool());
+                        }
+                    }
+                    if (LABEL.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setLabel(annotation_value.asString());
+                        }
+                    }
+                    if (WEIGHT.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setWeight((int)annotation_value.asDouble());
+                        }
+                    }
+                });
+            }
+            if (out instanceof IntItem && annotations != null) {
+                IntItem number = (IntItem) out;
+                Integer hitLimit = getCappedRangeSearchParameter(annotations);
+                if (hitLimit != null) {
+                    number.setHitLimit(hitLimit);
+                }
+
+            }
+        }
+
+        return out;
+    }
+
+
+    private Integer getCappedRangeSearchParameter(Inspector annotations) {
+        final Integer[] hitLimit = {null};
+        annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+            if (HIT_LIMIT.equals(annotation_name)) {
+                if (annotation_value != null) {
+                    hitLimit[0] = (int)(annotation_value.asDouble());
+                }
+            }
+        });
+        final Boolean[] ascending = {null};
+        final Boolean[] descending = {null};
+
+        if (hitLimit[0] != null) {
+            annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+                if (ASCENDING_HITS_ORDER.equals(annotation_name)) {
+                    ascending[0] = annotation_value.asBool();
+                }
+                if (DESCENDING_HITS_ORDER.equals(annotation_name)) {
+                    descending[0] = annotation_value.asBool();
+                }
+
+            });
+            Preconditions.checkArgument(ascending[0] == null || descending[0] == null,
+                    "Settings for both ascending and descending ordering set, only one of these expected.");
+
+            if (Boolean.TRUE.equals(descending[0]) || Boolean.FALSE.equals(ascending[0])) {
+                hitLimit[0] = hitLimit[0] * -1;
+            }
+        }
+        return hitLimit[0];
+    }
+
+
+    @NonNull
+    private Item buildRange(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        Inspector annotations = getAnnotations(value);
+
+        final boolean[] equals = {false};
+
+        String field;
+        Inspector boundInspector;
+        if (children.get(0).type() == STRING){
+            field = children.get(0).asString();
+            boundInspector = children.get(1);
+        } else {
+            field = children.get(1).asString();
+            boundInspector = children.get(0);
+        }
+
+        final Number[] bounds = {null, null};
+        final String[] operators = {null, null};
+        boundInspector.traverse((ObjectTraverser) (operator, bound) -> {
+            if (bound.type() == STRING) {
+                throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.");
+            }
+            if (operator.equals("=")) {
+                bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
+                operators[0] = operator;
+                equals[0] = true;
+            }
+            if (operator.equals(">=") || operator.equals(">")){
+                bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
+                operators[0] = operator;
+            } else if (operator.equals("<=") || operator.equals("<")){
+                bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
+                operators[1] = operator;
+            }
+
+        });
+        IntItem range = null;
+        if (equals[0]){
+            range = new IntItem(bounds[0].toString(), field);
+        } else if (operators[0]==null || operators[1]==null){
+            Integer index = (operators[0] == null) ? 1 : 0;
+            switch (operators[index]){
+                case ">=":
+                    range = buildGreaterThanOrEquals(field, bounds[index].toString());
+                    break;
+                case ">":
+                    range = buildGreaterThan(field, bounds[index].toString());
+                    break;
+                case "<":
+                    range = buildLessThan(field, bounds[index].toString());
+                    break;
+                case "<=":
+                    range = buildLessThanOrEquals(field, bounds[index].toString());
+                    break;
+            }
+        }
+        else {
+            range = instantiateRangeItem(bounds[0], bounds[1], field, operators[0].equals(">"), operators[1].equals("<"));
+        }
+
+        return leafStyleSettings(annotations, range);
+    }
+
+    @NonNull
+    private IntItem buildGreaterThanOrEquals(String field, String bound) {
+        return new IntItem("[" + bound + ";]", field);
+
+    }
+
+
+    @NonNull
+    private IntItem buildLessThanOrEquals(String field, String bound) {
+        return new IntItem("[;" + bound + "]", field);
+    }
+
+
+    @NonNull
+    private IntItem buildGreaterThan(String field, String bound) {
+        return new IntItem(">" + bound, field);
+
+    }
+
+
+    @NonNull
+    private IntItem buildLessThan(String field, String bound) {
+        return new IntItem("<" + bound, field);
+    }
+
+
+    @NonNull
+    private IntItem instantiateRangeItem(Number lowerBound, Number upperBound, String field, boolean bounds_left_open, boolean bounds_right_open) {
+        Preconditions.checkArgument(lowerBound != null && upperBound != null && field != null,
+                "Expected 3 NonNull-arguments");
+
+        if (!bounds_left_open && !bounds_right_open) {
+            return new RangeItem(lowerBound, upperBound, field);
+        } else {
+            Limit from;
+            Limit to;
+            if (bounds_left_open && bounds_right_open) {
+                from = new Limit(lowerBound, false);
+                to = new Limit(upperBound, false);
+            } else if (bounds_left_open) {
+                from = new Limit(lowerBound, false);
+                to = new Limit(upperBound, true);
+            } else {
+                from = new Limit(lowerBound, true);
+                to = new Limit(upperBound, false);
+            }
+            return new IntItem(from, to, field);
+        }
+    }
+
+
+    @NonNull
+    private Item buildEquals(String key, Inspector value) {
+         return buildRange(key, value);
+    }
+
+
+    @NonNull
+    private Item buildWand(String key, Inspector value) {
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size());
+        Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS);
+
+        WandItem out = new WandItem(children.get(0).asString(), target_num_hits);
+
+        Double scoreThreshold = getDoubleAnnotation(SCORE_THRESHOLD, annotations, null);
+
+        if (scoreThreshold != null) {
+            out.setScoreThreshold(scoreThreshold);
+        }
+
+        Double thresholdBoostFactor = getDoubleAnnotation(THRESHOLD_BOOST_FACTOR, annotations, null);
+        if (thresholdBoostFactor != null) {
+            out.setThresholdBoostFactor(thresholdBoostFactor);
+        }
+        return fillWeightedSet(value, children, out);
+    }
+
+
+    @NonNull
+    private WeightedSetItem fillWeightedSet(Inspector value, HashMap<Integer, Inspector> children, @NonNull WeightedSetItem out) {
+        addItems(children, out);
+
+        return leafStyleSettings(getAnnotations(value), out);
+    }
+
+
+    private static void addItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
+        switch (children.get(1).type()) {
+            case OBJECT:
+                addStringItems(children, out);
+                break;
+            case ARRAY:
+                addLongItems(children, out);
+                break;
+            default:
+                throw newUnexpectedArgumentException(children.get(1).type(), ARRAY, OBJECT);
+        }
+    }
+
+
+    private static void addStringItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
+        //{"a":1, "b":2}
+        children.get(1).traverse((ObjectTraverser) (key, value) -> {
+            if (value.type() == STRING){
+                throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.");
+            }
+            out.addToken(key, (int)value.asLong());
+        });
+    }
+
+
+    private static void addLongItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
+        //[[11,1], [37,2]]
+        children.get(1).traverse((ArrayTraverser) (index, pair) -> {
+            List<Integer> pairValues = new ArrayList<>();
+            pair.traverse((ArrayTraverser) (pairIndex, pairValue) -> {
+                pairValues.add((int)pairValue.asLong());
+            });
+            Preconditions.checkArgument(pairValues.size() == 2,
+                    "Expected item and weight, got %s.", pairValues);
+            out.addToken(pairValues.get(0).longValue(), pairValues.get(1));
+        });
+    }
+
+
+    @NonNull
+    private Item buildRegExpSearch(String key, Inspector value) {
+        assertHasOperator(key, MATCHES);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        String wordData = children.get(1).asString();
+        RegExpItem regExp = new RegExpItem(field, true, wordData);
+        return leafStyleSettings(getAnnotations(value), regExp);
+    }
+
+
+    @NonNull
+    private Item buildWeightedSet(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size());
+        return fillWeightedSet(value, children, new WeightedSetItem(field));
+    }
+
+
+    @NonNull
+    private Item buildDotProduct(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size());
+        return fillWeightedSet(value, children, new DotProductItem(field));
+    }
+
+
+    @NonNull
+    private Item buildPredicate(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        Inspector args = children.get(1);
+
+        Preconditions.checkArgument(children.size() == 3, "Expected 3 arguments, got %s.", children.size());
+
+        PredicateQueryItem item = new PredicateQueryItem();
+        item.setIndexName(field);
+
+        List<Inspector> argumentList = valueListFromInspector(getChildren(value));
+
+        // Adding attributes
+        argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> {
+            if (attrValue.type() == ARRAY){
+                List<Inspector> attributes = valueListFromInspector(attrValue);
+                attributes.forEach( (attribute) -> item.addFeature(attrKey, attribute.asString()));
+            } else {
+                item.addFeature(attrKey, attrValue.asString());
+            }
+        });
+
+        // Adding range attributes
+        argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, (int)attrValue.asDouble()));
+
+        return leafStyleSettings(getAnnotations(value), item);
+    }
+
+
+    @NonNull
+    private CompositeItem buildRank(String key, Inspector value) {
+        RankItem rankItem = new RankItem();
+        addItemsFromInspector(rankItem, value);
+        return rankItem;
+    }
+
+
+    @NonNull
+    private Item buildTermSearch(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+
+        return instantiateLeafItem(field, key, value);
+    }
+
+
+    private String getInspectorKey(Inspector inspector){
+        String[] actualKey = {""};
+        if (inspector.type() == OBJECT){
+            inspector.traverse((ObjectTraverser) (key, value) -> {
+                actualKey[0] = key;
+
+            });
+        }
+        return actualKey[0];
+    }
+
+
+    @NonNull
+    private Item instantiateLeafItem(String field, String key, Inspector value) {
+        List<Inspector> possibleLeafFunction = valueListFromInspector(value);
+        String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : "";
+        if (FUNCTION_CALLS.contains(key)) {
+            return instantiateCompositeLeaf(field, key, value);
+        } else if(!possibleLeafFunctionName.equals("")){
+           return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName));
+        } else {
+            return instantiateWordItem(field, key, value);
+        }
+    }
+
+
+    @NonNull
+    private Item instantiateCompositeLeaf(String field, String key, Inspector value) {
+        switch (key) {
+            case SAME_ELEMENT:
+                return instantiateSameElementItem(field, key, value);
+            case PHRASE:
+                return instantiatePhraseItem(field, key, value);
+            case NEAR:
+                return instantiateNearItem(field, key, value);
+            case ONEAR:
+                return instantiateONearItem(field, key, value);
+            case EQUIV:
+                return instantiateEquivItem(field, key, value);
+            case ALTERNATIVES:
+                return instantiateWordAlternativesItem(field, key, value);
+            default:
+                throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT);
+        }
+    }
+
+
+    @NonNull
+    private Item instantiateWordItem(String field, String key, Inspector value) {
+        String wordData = getChildrenMap(value).get(1).asString();
+        return instantiateWordItem(field, wordData, key, value, false, decideParsingLanguage(value, wordData));
+    }
+
+
+    @NonNull
+    private Item instantiateWordItem(String field, String rawWord, String key, Inspector value, boolean exactMatch, Language language) {
+        String wordData = rawWord;
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+
+        if (getBoolAnnotation(NFKC, annotations, Boolean.FALSE)) {
+            // NOTE: If this is set to FALSE (default), we will still NFKC normalize text data
+            // during tokenization/segmentation, as that is always turned on also on the indexing side.
+            wordData = normalizer.normalize(wordData);
+        }
+        boolean fromQuery = getBoolAnnotation(IMPLICIT_TRANSFORMS, annotations,  Boolean.TRUE);
+        boolean prefixMatch = getBoolAnnotation(PREFIX, annotations, Boolean.FALSE);
+        boolean suffixMatch = getBoolAnnotation(SUFFIX, annotations,  Boolean.FALSE);
+        boolean substrMatch = getBoolAnnotation(SUBSTRING,annotations, Boolean.FALSE);
+
+        Preconditions.checkArgument((prefixMatch ? 1 : 0)
+                        + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2,
+                "Only one of prefix, substring and suffix can be set.");
+        @NonNull
+        final TaggableItem wordItem;
+
+        if (exactMatch) {
+            wordItem = new ExactStringItem(wordData, fromQuery);
+        } else if (prefixMatch) {
+            wordItem = new PrefixItem(wordData, fromQuery);
+        } else if (suffixMatch) {
+            wordItem = new SuffixItem(wordData, fromQuery);
+        } else if (substrMatch) {
+            wordItem = new SubstringItem(wordData, fromQuery);
+        } else {
+            wordItem = new WordItem(wordData, fromQuery);
+        }
+
+        if (wordItem instanceof WordItem) {
+            prepareWord(field, value, (WordItem) wordItem);
+        }
+        if (language != Language.ENGLISH)
+            ((Item)wordItem).setLanguage(language);
+
+        return (Item) leafStyleSettings(getAnnotations(value), wordItem);
+    }
+
+
+    private Language decideParsingLanguage(Inspector value, String wordData) {
+        String languageTag = getAnnotation(USER_INPUT_LANGUAGE, getAnnotationMap(value), String.class, null);
+
+        Language language = Language.fromLanguageTag(languageTag);
+        if (language != Language.UNKNOWN) return language;
+
+        Optional<Language> explicitLanguage = query.getExplicitLanguage();
+        if (explicitLanguage.isPresent()) return explicitLanguage.get();
+
+        return Language.ENGLISH;
+    }
+
+
+    private void prepareWord(String field, Inspector value, WordItem wordItem) {
+        wordItem.setIndexName(field);
+        wordStyleSettings(value, wordItem);
+    }
+
+
+    private void wordStyleSettings(Inspector value, WordItem out) {
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+
+        Substring origin = getOrigin(getAnnotations(value));
+        if (origin != null) {
+            out.setOrigin(origin);
+        }
+        if (annotations !=  null){
+            Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null));
+            if (usePositionData != null) {
+                out.setPositionData(usePositionData);
+            }
+            Boolean stem = getBoolAnnotation(STEM, annotations, null);
+            if (stem != null) {
+                out.setStemmed(!stem);
+            }
+
+            Boolean normalizeCase = getBoolAnnotation(NORMALIZE_CASE, annotations, null);
+            if (normalizeCase != null) {
+                out.setLowercased(!normalizeCase);
+            }
+            Boolean accentDrop = getBoolAnnotation(ACCENT_DROP, annotations, null);
+            if (accentDrop != null) {
+                out.setNormalizable(accentDrop);
+            }
+            Boolean andSegmenting = getBoolAnnotation(AND_SEGMENTING, annotations,  null);
+            if (andSegmenting != null) {
+                if (andSegmenting) {
+                    out.setSegmentingRule(SegmentingRule.BOOLEAN_AND);
+                } else {
+                    out.setSegmentingRule(SegmentingRule.PHRASE);
+                }
+            }
+        }
+    }
+
+
+    private Substring getOrigin(Inspector annotations) {
+        if (annotations != null) {
+            Inspector origin = getAnnotationAsInspectorOrNull(ORIGIN, getAnnotationMapFromAnnotationInspector(annotations));
+            if (origin == null) {
+                return null;
+            }
+            final String[] original = {null};
+            final Integer[] offset = {null};
+            final Integer[] length = {null};
+
+            origin.traverse((ObjectTraverser) (key, value) -> {
+                switch (key) {
+                    case (ORIGIN_ORIGINAL):
+                        original[0] = value.asString();
+                        break;
+                    case (ORIGIN_OFFSET):
+                        offset[0] = (int) value.asDouble();
+                        break;
+                    case (ORIGIN_LENGTH):
+                        length[0] = (int) value.asDouble();
+                        break;
+                }
+
+
+            });
+            return new Substring(offset[0], length[0] + offset[0], original[0]);
+        }
+        return null;
+    }
+
+
+    @NonNull
+    private Item instantiateSameElementItem(String field, String key, Inspector value) {
+        assertHasOperator(key, SAME_ELEMENT);
+
+        SameElementItem sameElement = new SameElementItem(field);
+        // All terms below sameElement are relative to this.
+        getChildren(value).traverse((ArrayTraverser) (index, term) -> {
+            sameElement.addItem(walkJson(term));
+        });
+
+        return sameElement;
+    }
+
+
+    @NonNull
+    private Item instantiatePhraseItem(String field, String key, Inspector value) {
+        assertHasOperator(key, PHRASE);
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+
+        PhraseItem phrase = new PhraseItem();
+        phrase.setIndexName(field);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        for (Inspector word :  children.values())
+            if (word.type() == STRING) phrase.addItem(new WordItem(word.asString()));
+            else if (word.type() == OBJECT && word.field(PHRASE).valid()) {
+                    phrase.addItem(instantiatePhraseItem(field, key, getChildren(word)));
+            }
+        return leafStyleSettings(getAnnotations(value), phrase);
+    }
+
+
+    @NonNull
+    private Item instantiateNearItem(String field, String key, Inspector value) {
+        assertHasOperator(key, NEAR);
+
+        NearItem near = new NearItem();
+        near.setIndexName(field);
+
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        for (Inspector word :  children.values()){
+            near.addItem(new WordItem(word.asString(), field));
+        }
+
+        Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null);
+
+        if (distance != null) {
+            near.setDistance((int)distance);
+        }
+        return near;
+    }
+
+
+    @NonNull
+    private Item instantiateONearItem(String field, String key, Inspector value) {
+        assertHasOperator(key, ONEAR);
+
+        NearItem onear = new ONearItem();
+        onear.setIndexName(field);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        for (Inspector word :  children.values()){
+            onear.addItem(new WordItem(word.asString(), field));
+        }
+
+        Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null);
+        if (distance != null) {
+            onear.setDistance(distance);
+        }
+        return onear;
+    }
+
+
+    @NonNull
+    private Item instantiateEquivItem(String field, String key, Inspector value) {
+
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        Preconditions.checkArgument(children.size() >= 2, "Expected 2 or more arguments, got %s.", children.size());
+
+        EquivItem equiv = new EquivItem();
+        equiv.setIndexName(field);
+
+        for (Inspector word :  children.values()){
+            if (word.type() == STRING || word.type() == LONG || word.type() == DOUBLE){
+                equiv.addItem(new WordItem(word.asString(), field));
+            }
+            if (word.type() == OBJECT){
+                word.traverse((ObjectTraverser) (key2, value2) -> {
+                    assertHasOperator(key2, PHRASE);
+                    equiv.addItem(instantiatePhraseItem(field, key2, value2));
+                });
+            }
+        }
+
+        return leafStyleSettings(getAnnotations(value), equiv);
+    }
+
+
+    private Item instantiateWordAlternativesItem(String field, String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size());
+        Preconditions.checkArgument(children.get(0).type() == OBJECT, "Expected OBJECT, got %s.", children.get(0).type());
+
+        List<WordAlternativesItem.Alternative> terms = new ArrayList<>();
+
+        children.get(0).traverse((ObjectTraverser) (keys, values) -> {
+            terms.add(new WordAlternativesItem.Alternative(keys, values.asDouble()));
+        });
+        return leafStyleSettings(getAnnotations(value), new WordAlternativesItem(field, Boolean.TRUE, null, terms));
+    }
+
+
+    //  Not in use yet
+    @NonNull
+    private String getIndex(String field) {
+        Preconditions.checkArgument(indexFactsSession.isIndex(field), "Field '%s' does not exist.", field);
+        //return indexFactsSession.getCanonicName(field);
+        return field;
+    }
+
+
+    private static void assertHasOperator(String key, String expectedKey) {
+        Preconditions.checkArgument(key.equals(expectedKey), "Expected operator %s, got %s.", expectedKey, key);
+    }
+
+
+    private static IllegalArgumentException newUnexpectedArgumentException(Object actual, Object... expected) {
+        StringBuilder out = new StringBuilder("Expected ");
+        for (int i = 0, len = expected.length; i < len; ++i) {
+            out.append(expected[i]);
+            if (i < len - 2) {
+                out.append(", ");
+            } else if (i < len - 1) {
+                out.append(" or ");
+            }
+        }
+        out.append(", got ").append(actual).append(".");
+        return new IllegalArgumentException(out.toString());
+    }
+
+
+    private List<Inspector> valueListFromInspector(Inspector inspector){
+        List<Inspector> inspectorList = new ArrayList<>();
+        inspector.traverse((ArrayTraverser) (key, value) -> inspectorList.add(value));
+        return inspectorList;
+    }
+
+
+    private void connectItems() {
+        for (ConnectedItem entry : connectedItems) {
+            TaggableItem to = identifiedItems.get(entry.toId);
+            Preconditions.checkNotNull(to,
+                    "Item '%s' was specified to connect to item with ID %s, which does not "
+                            + "exist in the query.", entry.fromItem,
+                    entry.toId);
+            entry.fromItem.setConnectivity((Item) to, entry.weight);
+        }
+    }
+
+
+    private static final class ConnectedItem {
+
+        final double weight;
+        final int toId;
+        final TaggableItem fromItem;
+
+        ConnectedItem(TaggableItem fromItem, int toId, double weight) {
+            this.weight = weight;
+            this.toId = toId;
+            this.fromItem = fromItem;
+        }
+    }
+
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
index e5941a90b83..64fb201fe21 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
@@ -3,6 +3,7 @@ package com.yahoo.search.query.parser;
 
 import com.yahoo.language.Language;
 import com.yahoo.search.query.Model;
+import com.yahoo.search.query.Select;
 
 import java.util.Collection;
 import java.util.HashSet;
@@ -36,6 +37,7 @@ public final class Parsable {
     private String defaultIndexName;
     private Language language; // TODO: Initialize to UNKNOWN
     private Optional<Language> explicitLanguage = Optional.empty();
+    private Select select;
 
     /** If this is set it will be used to determine the language, if not set explicitly */
     private Optional<Model> model = Optional.empty();
@@ -133,6 +135,15 @@ public final class Parsable {
         return this;
     }
 
+    public Parsable setSelect(Select select){
+        this.select = select;
+        return this;
+    }
+
+    public Select getSelect(){
+        return this.select;
+    }
+
     public static Parsable fromQueryModel(Model model) {
         return new Parsable()
                 .setModel(model)
@@ -141,7 +152,11 @@ public final class Parsable {
                 .setExplicitLanguage(Optional.ofNullable(model.getLanguage()))
                 .setDefaultIndexName(model.getDefaultIndex())
                 .addSources(model.getSources())
-                .addRestricts(model.getRestrict());
+                .addRestricts(model.getRestrict())
+                .setSelect(model.getParent().getSelect());
     }
 
+
+
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
index 8d008abaac2..69d46527255 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
@@ -3,6 +3,7 @@ package com.yahoo.search.query.parser;
 
 import com.yahoo.prelude.query.parser.*;
 import com.yahoo.search.Query;
+import com.yahoo.search.query.SelectParser;
 import com.yahoo.search.yql.YqlParser;
 
 /**
@@ -40,6 +41,8 @@ public final class ParserFactory {
                 return new ProgrammaticParser();
             case YQL:
                 return new YqlParser(environment);
+            case SELECT:
+                return new SelectParser(environment);
             default:
                 throw new UnsupportedOperationException(type.toString());
         }
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index 0aea5e96161..71002166b11 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -113,6 +113,10 @@ public class QueryProperties extends Properties {
                 if (key.get(1).equals(Ranking.PROPERTIES)) return ranking.getProperties().get(key.rest().rest().toString());
             }
         }
+        else if (key.size()==2 && key.first().equals(Select.SELECT)) {
+            if (key.last().equals(Select.WHERE)) return query.getSelect().getWhereString();
+            if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString();
+        }
         else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) {
             if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding();
             if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary();
@@ -247,6 +251,13 @@ public class QueryProperties extends Properties {
                 else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7
                     throwIllegalParameter(key.last(), Presentation.PRESENTATION);
             }
+            else if (key.size()==2 && key.first().equals(Select.SELECT)) {
+                if (key.last().equals(Select.WHERE)){
+                    query.getSelect().setWhere(asString(value, ""));
+                } else if (key.last().equals(Select.GROUPING)) {
+                    query.getSelect().setGrouping(asString(value, ""));
+                }
+            }
             else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated
                 setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features")));
             } else if (key.first().equals("rankproperty")) {
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
index 6bad032600c..e0e9042e1a3 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
@@ -789,7 +789,7 @@ public class YqlParser implements Parser {
             OperatorNode<ExpressionOperator> groupingAst = ast.<List<OperatorNode<ExpressionOperator>>> getArgument(2).get(0);
             GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.<String> getArgument(0));
             VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation);
-            List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, 
+            List<String> continuations = getAnnotation(groupingAst, "continuations", List.class,
                                                        Collections.emptyList(), "grouping continuations");
             for (String continuation : continuations) {
                 groupingStep.continuations().add(Continuation.fromString(continuation));
diff --git a/container-search/src/main/resources/configdefinitions/provider.def b/container-search/src/main/resources/configdefinitions/provider.def
index 79b09913b49..f9ab305b114 100644
--- a/container-search/src/main/resources/configdefinitions/provider.def
+++ b/container-search/src/main/resources/configdefinitions/provider.def
@@ -35,7 +35,7 @@ yca.ttl int default=0
 yca.retry int default=0
 
 # The form of the serialized query.
-queryType enum { LEGACY, PROGRAMMATIC, YQL } default=LEGACY
+queryType enum { LEGACY, PROGRAMMATIC, YQL, SELECT } default=LEGACY
 
 # How to do pinging against a backend.
 pingOption enum { DISABLE, NORMAL, YCA } default=NORMAL
diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
index eea58d5444e..e85a945cc67 100644
--- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
@@ -11,7 +11,10 @@ import com.yahoo.io.IOUtils;
 import com.yahoo.net.HostName;
 import com.yahoo.search.handler.SearchHandler;
 import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase;
+import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Type;
 import com.yahoo.vespa.config.SlimeUtils;
 import org.json.JSONObject;
 import org.junit.After;
@@ -21,8 +24,10 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.stream.Collectors;
 
 import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -338,6 +343,35 @@ public class JSONSearchHandlerTestCase {
     }
 
 
+    @Test
+    public void testSelectParameter() throws Exception {
+        JSONObject json = new JSONObject();
+
+        JSONObject select = new JSONObject();
+
+            JSONObject where = new JSONObject();
+            where.put("where", "where");
+
+            JSONObject grouping = new JSONObject();
+            grouping.put("grouping", "grouping");
+
+        select.put("where", where);
+        select.put("grouping", grouping);
+
+        json.put("select", select);
+
+
+        // Create mapping
+        Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get();
+        Map<String, String> map = new HashMap<>();
+        searchHandler.createRequestMapping(inspector, map, "");
+
+        JSONObject processedWhere = new JSONObject(map.get("select.where"));
+        assertEquals(where.toString(), processedWhere.toString());
+
+        JSONObject processedGrouping = new JSONObject(map.get("select.grouping"));
+        assertEquals(grouping.toString(), processedGrouping.toString());
+    }
 
     @Test
     public void testRequestMapping() throws Exception {
diff --git a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java
new file mode 100644
index 00000000000..031ba386ad4
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java
@@ -0,0 +1,779 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.select;
+
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.ExactStringItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.RegExpItem;
+import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.SubstringItem;
+import com.yahoo.prelude.query.SuffixItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.Query;
+import com.yahoo.search.federation.ProviderConfig;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.search.query.Select;
+import com.yahoo.search.query.SelectParser;
+import com.yahoo.search.query.parser.Parsable;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.yql.VespaGroupingStep;
+import org.apache.http.client.utils.URIBuilder;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+
+/**
+ * Specification for the conversion of Select expressions to Vespa search queries.
+ *
+ * @author henrhoi
+ */
+
+public class SelectParserTestCase {
+
+    private final SelectParser parser = new SelectParser(new ParserEnvironment());
+
+
+    /** WHERE TESTS */
+
+    @Test
+    public void test_contains() throws Exception {
+        JSONObject json = new JSONObject();
+        List<String> contains = Arrays.asList("default", "foo");
+        json.put("contains", contains);
+        assertParse(json.toString(), "default:foo");
+    }
+
+    @Test
+    public void test() {
+        assertParse("{'contains' : ['title', 'madonna']}",
+                "title:madonna");
+    }
+
+
+    @Test
+    public void testDottedFieldNames() {
+        assertParse("{ 'contains' : ['my.nested.title', 'madonna']}",
+                "my.nested.title:madonna");
+    }
+
+
+
+    @Test
+    public void testOr() throws Exception {
+        JSONObject json_two_or = new JSONObject();
+        JSONObject json_three_or = new JSONObject();
+        List<String> contains1 = Arrays.asList("title", "madonna");
+        List<String> contains2 = Arrays.asList("title", "saint");
+        List<String> contains3 = Arrays.asList("title", "angel");
+
+        JSONObject contains_json1 = new JSONObject();
+        JSONObject contains_json2 = new JSONObject();
+        JSONObject contains_json3 = new JSONObject();
+        contains_json1.put("contains", contains1);
+        contains_json2.put("contains", contains2);
+        contains_json3.put("contains", contains3);
+
+        json_two_or.put("or", Arrays.asList(contains_json1, contains_json2));
+        json_three_or.put("or", Arrays.asList(contains_json1, contains_json2, contains_json3));
+
+        assertParse(json_two_or.toString(), "OR title:madonna title:saint");
+        assertParse(json_three_or.toString(), "OR title:madonna title:saint title:angel");
+    }
+
+    @Test
+    public void testAnd() throws Exception{
+        JSONObject json_two_and = new JSONObject();
+        JSONObject json_three_and = new JSONObject();
+        List<String> contains1 = Arrays.asList("title", "madonna");
+        List<String> contains2 = Arrays.asList("title", "saint");
+        List<String> contains3 = Arrays.asList("title", "angel");
+
+        JSONObject contains_json1 = new JSONObject();
+        JSONObject contains_json2 = new JSONObject();
+        JSONObject contains_json3 = new JSONObject();
+        contains_json1.put("contains", contains1);
+        contains_json2.put("contains", contains2);
+        contains_json3.put("contains", contains3);
+
+        json_two_and.put("and", Arrays.asList(contains_json1, contains_json2));
+        json_three_and.put("and", Arrays.asList(contains_json1, contains_json2, contains_json3));
+
+        assertParse(json_two_and.toString(), "AND title:madonna title:saint");
+        assertParse(json_three_and.toString(), "AND title:madonna title:saint title:angel");
+    }
+
+    @Test
+    public void testAndNot() throws JSONException {
+        JSONObject json_and_not = new JSONObject();
+        List<String> contains1 = Arrays.asList("title", "madonna");
+        List<String> contains2 = Arrays.asList("title", "saint");
+
+        JSONObject contains_json1 = new JSONObject();
+        JSONObject contains_json2 = new JSONObject();
+        contains_json1.put("contains", contains1);
+        contains_json2.put("contains", contains2);
+
+        json_and_not.put("and_not", Arrays.asList(contains_json1, contains_json2));
+
+        assertParse(json_and_not.toString(),
+                "+title:madonna -title:saint");
+    }
+
+
+    @Test
+    public void testLessThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:<500");
+    }
+
+    @Test
+    public void testGreaterThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:>500");
+    }
+
+
+    @Test
+    public void testLessThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<=", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[;500]");
+    }
+
+    @Test
+    public void testGreaterThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">=", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[500;]");
+    }
+
+    @Test
+    public void testEquality() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("=", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:500");
+    }
+
+    @Test
+    public void testNegativeLessThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:<-500");
+    }
+
+    @Test
+    public void testNegativeGreaterThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:>-500");
+    }
+
+    @Test
+    public void testNegativeLessThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<=", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[;-500]");
+    }
+
+    @Test
+    public void testNegativeGreaterThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">=", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[-500;]");
+    }
+
+    @Test
+    public void testNegativeEquality() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("=", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:-500");
+    }
+
+    @Test
+    public void testAnnotatedLessThan() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<\" : -500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:<-500");
+    }
+
+    @Test
+    public void testAnnotatedGreaterThan() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">\" : 500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:>500");
+    }
+
+    @Test
+    public void testAnnotatedLessThanOrEqual() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<=\" : -500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:[;-500]");
+    }
+
+    @Test
+    public void testAnnotatedGreaterThanOrEqual() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">=\" : 500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:[500;]");
+    }
+
+
+    @Test
+    public void testAnnotatedEquality() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"=\" : -500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:-500");
+    }
+
+    @Test
+    public void testTermAnnotations() {
+        assertEquals("merkelapp",
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"label\" : \"merkelapp\"} } }").getLabel());
+        assertEquals("another",
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"annotations\" : {\"cox\" : \"another\"} } } }").getAnnotation("cox"));
+        assertEquals(23.0, getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"significance\" : 23.0 } } }").getSignificance(), 1E-6);
+        assertEquals(150, getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"weight\" : 150 } } }").getWeight());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"usePositionData\" : false } } }").usePositionData());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"filter\" : true } } }").isFilter());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"ranked\" : false } } }").isRanked());
+        Substring origin = getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"origin\": {\"original\": \"abc\", \"offset\": 1, \"length\": 2}} } }").getOrigin();
+        assertEquals("abc", origin.string);
+        assertEquals(1, origin.start);
+        assertEquals(3, origin.end);
+    }
+
+
+    @Test
+    public void testSameElement() {
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }",
+                "baz:{f1:a f2:b}");
+
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, {\"range\":[\"f2\",{\"=\":10}] } ]} ] }",
+                "baz:{f1:a f2:10}");
+
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"key\", \"a\"] }, {\"range\":[\"value.f2\",{\"=\":10}] } ]} ] }",
+                "baz:{key:a value.f2:10}");
+    }
+
+    @Test
+    public void testPhrase() {
+        assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\"] } ] }",
+                "baz:\"a b\"");
+    }
+
+    @Test
+    public void testNestedPhrase() {
+        assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\", {\"phrase\" : [ \"c\", \"d\"] }] } ] }",
+                "baz:\"a b c d\"");
+    }
+
+    @Test
+    public void testStemming() {
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : false} } }").isStemmed());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : true} } }").isStemmed());
+        assertFalse(getRootWord("{ \"contains\": [\"baz\", \"colors\"] }").isStemmed());
+    }
+
+    @Test
+    public void testRaw() {
+        Item root = parseWhere("{ \"contains\":[ \"baz\", \"yoni jo dima\" ] }").getRoot();
+        assertTrue(root instanceof WordItem);
+        assertFalse(root instanceof ExactStringItem);
+        assertEquals("yoni jo dima", ((WordItem)root).getWord());
+
+        root = parseWhere("{ \"contains\": { \"children\" : [\"baz\", \"yoni jo dima\"], \"attributes\" : {\"grammar\" : \"raw\"} } }").getRoot();
+        assertTrue(root instanceof WordItem);
+        assertFalse(root instanceof ExactStringItem);
+        assertEquals("yoni jo dima", ((WordItem)root).getWord());
+    }
+
+    @Test
+    public void testAccentDropping() {
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : false} } }").isNormalizable());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : true} } }").isNormalizable());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isNormalizable());
+    }
+
+    @Test
+    public void testCaseNormalization() {
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : false} } }").isLowercased());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : true} } }").isLowercased());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isLowercased());
+    }
+
+    @Test
+    public void testSegmentingRule() {
+        assertEquals(SegmentingRule.PHRASE,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : false} } }").getSegmentingRule());
+        assertEquals(SegmentingRule.BOOLEAN_AND,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : true} } }").getSegmentingRule());
+        assertEquals(SegmentingRule.LANGUAGE_DEFAULT,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").getSegmentingRule());
+    }
+
+    @Test
+    public void testNfkc() {
+        assertEquals("a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : false} } }").getWord());
+        assertEquals("\u00e5", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : true} } }").getWord());
+        assertEquals("No NKFC by default", "a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"] } } ").getWord());
+    }
+
+    @Test
+    public void testImplicitTransforms() {
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : false} } }").isFromQuery());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : true} } }").isFromQuery());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"] } }").isFromQuery());
+    }
+
+    @Test
+    public void testConnectivity() {
+        QueryTree parsed = parseWhere("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }");
+        assertEquals("AND title:madonna title:saint title:angel", parsed.toString());
+
+        AndItem root = (AndItem)parsed.getRoot();
+        WordItem first = (WordItem)root.getItem(0);
+        WordItem second = (WordItem)root.getItem(1);
+        WordItem third = (WordItem)root.getItem(2);
+        assertTrue(first.getConnectedItem() == third);
+        assertEquals(first.getConnectivity(), 7.0d, 1E-6);
+        assertNull(second.getConnectedItem());
+
+        assertParseFail("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }",
+                new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " +
+                "which does not exist in the query."));
+    }
+
+    @Test
+    public void testAnnotatedPhrase() {
+        QueryTree parsed = parseWhere("{ \"contains\": [\"baz\", { \"phrase\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"label\": \"hello world\" } } }] }");
+        assertEquals("baz:\"a b\"", parsed.toString());
+        PhraseItem phrase = (PhraseItem)parsed.getRoot();
+        assertEquals("hello world", phrase.getLabel());
+    }
+
+    @Test
+    public void testRange() {
+        QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": 1, \"<=\": 8 }] }");
+        assertEquals("baz:[1;8]", parsed.toString());
+    }
+
+    @Test
+    public void testNegativeRange() {
+        QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": -8, \"<=\": -1 }] }");
+        assertEquals("baz:[-8;-1]", parsed.toString());
+    }
+
+    @Test
+    public void testRangeIllegalArguments() {
+        assertParseFail("{ \"range\": [\"baz\", { \">=\": \"cox\", \"<=\": -1 }] }",
+                new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."));
+    }
+
+    @Test
+    public void testNear() {
+        assertParse("{ \"contains\": [\"description\", { \"near\": [\"a\", \"b\"] }] }",
+                "NEAR(2) description:a description:b");
+        assertParse("{ \"contains\": [\"description\", { \"near\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }",
+                "NEAR(100) description:a description:b");
+    }
+
+    @Test
+    public void testOrderedNear() {
+        assertParse("{ \"contains\": [\"description\", { \"onear\": [\"a\", \"b\"] }] }",
+                "ONEAR(2) description:a description:b");
+        assertParse("{ \"contains\": [\"description\", { \"onear\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }",
+                "ONEAR(100) description:a description:b");
+    }
+
+    @Test
+    public void testWand() {
+        assertParse("{ \"wand\": [\"description\", { \"a\": 1, \"b\": 2 }] }",
+                "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}");
+        assertParse("{ \"wand\": { \"children\": [\"description\", { \"a\": 1, \"b\": 2 }], \"attributes\": { \"scoreThreshold\": 13.3, \"targetNumHits\": 7, \"thresholdBoostFactor\": 2.3 } } }",
+                "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}");
+    }
+
+    @Test
+    public void testNumericWand() {
+        String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}";
+        assertParse("{ \"wand\" : [\"description\", [[11,1], [37,2]] ]}", numWand);
+        assertParseFail("{ \"wand\" : [\"description\", 12] }",
+                new IllegalArgumentException("Expected ARRAY or OBJECT, got LONG."));
+    }
+
+    @Test
+    public void testWeightedSet() {
+        assertParse("{ \"weightedSet\" : [\"description\", {\"a\":1, \"b\":2} ]}",
+                "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}");
+        assertParseFail("{ \"weightedSet\" : [\"description\", {\"a\":\"g\", \"b\":2} ]}",
+                new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."));
+        assertParseFail("{ \"weightedSet\" : [\"description\" ]}",
+                new IllegalArgumentException("Expected 2 arguments, got 1."));
+    }
+
+    @Test
+    public void testDotProduct() {
+        assertParse("{ \"dotProduct\" : [\"description\", {\"a\":1, \"b\":2} ]}",
+                "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}");
+        assertParse("{ \"dotProduct\" : [\"description\", {\"a\":2} ]}",
+                "DOTPRODUCT description{[2]:\"a\"}");
+    }
+
+    @Test
+    public void testPredicate() {
+        assertParse("{ \"predicate\" : [\"predicate_field\", {\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23} ]}",
+                "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23");
+        assertParse("{ \"predicate\" : [\"predicate_field\", 0, \"void\" ]}",
+                "PREDICATE_QUERY_ITEM ");
+    }
+
+    @Test
+    public void testRank() {
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }",
+                "RANK a:A b:B");
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] } ] }",
+                "RANK a:A b:B c:C");
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"or\": [{ \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] }] }] }",
+                "RANK a:A (OR b:B c:C)");
+    }
+
+    @Test
+    public void testWeakAnd() {
+        assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }",
+                "WAND(100) a:A b:B");
+        assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetNumHits\": 37} }}",
+                "WAND(37) a:A b:B");
+
+        QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}");
+        assertEquals("WAND(100) a:A b:B", tree.toString());
+        assertEquals(WeakAndItem.class, tree.getRoot().getClass());
+        assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold());
+    }
+
+    @Test
+    public void testEquiv() {
+        assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"A\",\"B\"]}]}",
+                "EQUIV fieldName:A fieldName:B");
+
+        assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"phrase\" : [ \"new\",\"york\" ] } ] } ] }",
+                "EQUIV fieldName:ny fieldName:\"new york\"");
+
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\"] } ] }",
+                new IllegalArgumentException("Expected 2 or more arguments, got 1."));
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"nalle\" : [ \"void\" ] } ] } ] }",
+                new IllegalArgumentException("Expected operator phrase, got nalle."));
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\", 42]}]}",
+                new IllegalArgumentException("Word item word can not be empty"));
+    }
+
+    @Test
+    public void testAffixItems() {
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true} } }",
+                SuffixItem.class);
+
+
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"prefix\": true} } }",
+                PrefixItem.class);
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"substring\": true} } }",
+                SubstringItem.class);
+        assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"prefix\" : true} } }",
+                new IllegalArgumentException("Only one of prefix, substring and suffix can be set."));
+        assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"substring\" : true} } }",
+                new IllegalArgumentException("Only one of prefix, substring and suffix can be set."));
+    }
+
+    @Test
+    public void testLongNumberInSimpleExpression() {
+        assertParse("{ \"range\" : [ \"price\", { \"=\" : 8589934592 }]}",
+                "price:8589934592");
+    }
+
+    @Test
+    public void testNegativeLongNumberInSimpleExpression() {
+        assertParse("{ \"range\" : [ \"price\", { \"=\" : -8589934592 }]}",
+                "price:-8589934592");
+    }
+
+    @Test
+    public void testNegativeHitLimit() {
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": -38 } } }",
+                "foo:[0;1;-38]");
+    }
+
+    @Test
+    public void testRangeSearchHitPopulationOrdering() {
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": true} } }", "foo:[0;1;38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": false} } }", "foo:[0;1;-38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": true} } }", "foo:[0;1;-38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": false} } }", "foo:[0;1;38]");
+
+        boolean gotExceptionFromParse = false;
+        try {
+            parseWhere("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38, \"ascending\": true, \"descending\": false} } }");
+        } catch (IllegalArgumentException e) {
+            assertTrue("Expected information about abuse of settings.",
+                    e.getMessage().contains("both ascending and descending ordering set"));
+            gotExceptionFromParse = true;
+        }
+        assertTrue(gotExceptionFromParse);
+    }
+
+    // NB: Uses operator-keys to set bounds, not annotations
+    @Test
+    public void testOpenIntervals() {
+        assertParse("{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<=\" : 500.0 }] } }" +
+                        "select * from sources * where range(title, 0.0, 500.0);",
+                "title:[0.0;500.0]");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<\" : 500.0 }] } }",
+                "title:<0.0;500.0>");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<=\" : 500.0 }]  } }",
+                "title:<0.0;500.0]");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<\" : 500.0 }] } }",
+                "title:[0.0;500.0>");
+    }
+
+    @Test
+    public void testRegexp() {
+        QueryTree x = parseWhere("{ \"matches\" : [\"foo\", \"a b\"]}");
+        Item root = x.getRoot();
+        assertSame(RegExpItem.class, root.getClass());
+        assertEquals("a b", ((RegExpItem) root).stringValue());
+    }
+
+    @Test
+    public void testWordAlternatives() {
+        QueryTree x = parseWhere("{\"contains\" : [\"foo\", {\"alternatives\" : [{\"trees\": 1.0, \"tree\": 0.7}]}]}");
+        Item root = x.getRoot();
+        assertSame(WordAlternativesItem.class, root.getClass());
+        WordAlternativesItem alternatives = (WordAlternativesItem) root;
+        checkWordAlternativesContent(alternatives);
+    }
+
+    /** GROUPING TESTS */
+
+    @Test
+    public void testGrouping(){
+        String grouping = "[ { \"all\" : { \"group\" : \"time.year(a)\", \"each\" : { \"output\" : \"count()\" } } } ]";
+        String expected = "[[]all(group(time.year(a)) each(output(count())))]";
+        assertGrouping(expected, parseGrouping(grouping));
+    }
+
+
+    @Test
+    public void testMultipleGroupings() {
+        String grouping = "[ { \"all\" : { \"group\" : \"a\", \"each\" : { \"output\" : \"count()\"}}}, { \"all\" : { \"group\" : \"b\", \"each\" : { \"output\" : \"count()\"}}} ]";
+        String expected = "[[]all(group(a) each(output(count()))), []all(group(b) each(output(count())))]";
+
+        assertGrouping(expected, parseGrouping(grouping));
+    }
+
+
+
+    /** OTHER TESTS */
+
+    @Test
+    public void testOverridingOtherQueryTree() {
+        Query query = new Query("?query=default:query");
+        assertEquals("default:query", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.ALL, query.getModel().getType());
+
+        query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }");
+        assertEquals("default:select", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.SELECT, query.getModel().getType());
+    }
+
+
+    @Test
+    public void testOverridingWhereQueryTree() {
+        Query query = new Query();
+        query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }");
+        assertEquals("default:select", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.SELECT, query.getModel().getType());
+
+        query.getModel().setQueryString("default:query");
+        query.getModel().setType("all");
+        assertEquals("default:query", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.ALL, query.getModel().getType());
+    }
+
+
+
+
+    /** Assert-methods */
+    private void assertParse(String where, String expectedQueryTree) {
+        String queryTree = parseWhere(where).toString();
+        assertEquals(expectedQueryTree, queryTree);
+    }
+
+    private void assertParseFail(String where, Throwable expectedException) {
+        try {
+            parseWhere(where).toString();
+        } catch (Throwable t) {
+            assertEquals(expectedException.getClass(), t.getClass());
+            assertEquals(expectedException.getMessage(), t.getMessage());
+            return;
+        }
+        fail("Parse succeeded: " + where);
+    }
+
+    private void assertRootClass(String where, Class<? extends Item> expectedRootClass) {
+        assertEquals(expectedRootClass, parseWhere(where).getRoot().getClass());
+    }
+
+    private void assertGrouping(String expected, List<VespaGroupingStep> steps) {
+        List<String> actual = new ArrayList<>(steps.size());
+        for (VespaGroupingStep step : steps) {
+            actual.add(step.continuations().toString() +
+                    step.getOperation());
+        }
+        assertEquals(expected, actual.toString());
+    }
+
+
+
+
+    /** Parse-methods*/
+
+    private QueryTree parseWhere(String where) {
+        Select select = new Select(where, "");
+
+        return parser.parse(new Parsable().setSelect(select));
+    }
+
+    private List<VespaGroupingStep> parseGrouping(String grouping) {
+
+        return parser.getGroupingSteps(grouping);
+    }
+
+    private QueryTree parse(String where, String grouping) {
+        Select select = new Select(where, grouping);
+
+        return parser.parse(new Parsable().setSelect(select));
+    }
+
+
+
+
+
+    /** Other methods */
+    private WordItem getRootWord(String yqlQuery) {
+        Item root = parseWhere(yqlQuery).getRoot();
+        assertTrue(root instanceof WordItem);
+        return (WordItem)root;
+    }
+
+    private void checkWordAlternativesContent(WordAlternativesItem alternatives) {
+        boolean seenTree = false;
+        boolean seenForest = false;
+        final String forest = "trees";
+        final String tree = "tree";
+        assertEquals(2, alternatives.getAlternatives().size());
+        for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) {
+            if (tree.equals(alternative.word)) {
+                assertFalse("Duplicate term introduced", seenTree);
+                seenTree = true;
+                assertEquals(.7d, alternative.exactness, 1e-15d);
+            } else if (forest.equals(alternative.word)) {
+                assertFalse("Duplicate term introduced", seenForest);
+                seenForest = true;
+                assertEquals(1.0d, alternative.exactness, 1e-15d);
+            } else {
+                fail("Unexpected term: " + alternative.word);
+            }
+        }
+    }
+
+
+}