SelectParser now handles ''all'' testcases from YQLParserTestCase. Class may need some cleaning and removal of unuses code

2 files changed, 588 insertions, 146 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
index 559e0ab034e..779223754f4 100644
--- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
+++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
@@ -5,6 +5,7 @@ import com.google.common.base.Preconditions;
 import com.yahoo.collections.LazyMap;
 import com.yahoo.collections.Tuple2;
 import com.yahoo.component.Version;
+import com.yahoo.data.access.simple.Value;
 import com.yahoo.language.Language;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.detect.Detector;
@@ -45,6 +46,7 @@ import com.yahoo.search.query.parser.Parsable;
 import com.yahoo.search.query.parser.Parser;
 
 import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.yql.YqlParser;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
@@ -69,11 +71,8 @@ public class SelectParser implements Parser {
     Parsable query;
     private final IndexFacts indexFacts;
     private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap();
+    private final List<ConnectedItem> connectedItems = new ArrayList<>();
     private final Normalizer normalizer;
-    private final Segmenter segmenter;
-    private final Detector detector;
-    private final String localSegmenterBackend;
-    private final Version localSegmenterVersion;
     private final ParserEnvironment environment;
     private IndexFacts.Session indexFactsSession;
 
@@ -83,15 +82,14 @@ public class SelectParser implements Parser {
 
     private static final String DESCENDING_HITS_ORDER = "descending";
     private static final String ASCENDING_HITS_ORDER = "ascending";
-
-    private static final Integer DEFAULT_HITS = 10;
-    private static final Integer DEFAULT_OFFSET = 0;
     private static final Integer DEFAULT_TARGET_NUM_HITS = 10;
-    static final String ORIGIN_LENGTH = "length";
-    static final String ORIGIN_OFFSET = "offset";
-    static final String ORIGIN = "origin";
-    static final String ORIGIN_ORIGINAL = "original";
-
+    private static final String ORIGIN_LENGTH = "length";
+    private static final String ORIGIN_OFFSET = "offset";
+    private static final String ORIGIN = "origin";
+    private static final String ORIGIN_ORIGINAL = "original";
+    private static final String CONNECTION_ID = "id";
+    private static final String CONNECTION_WEIGHT = "weight";
+    private static final String CONNECTIVITY = "connectivity";
     private static final String ANNOTATIONS = "annotations";
     private static final String NFKC = "nfkc";
     private static final String USER_INPUT_LANGUAGE = "language";
@@ -127,30 +125,40 @@ public class SelectParser implements Parser {
     private static final String WEAK_AND = "weakAnd";
     private static final String WEIGHTED_SET = "weightedSet";
     private static final String WEIGHT = "weight";
+    private static final String AND = "and";
+    private static final String AND_NOT = "and_not";
+    private static final String OR = "or";
+    private static final String EQ = "equals";
+    private static final String RANGE = "range";
+    private static final String CONTAINS = "contains";
+    private static final String MATCHES = "matches";
+    private static final String CALL = "call";
+    private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND);
 
     /**************************************/
 
+
+
     public SelectParser(ParserEnvironment environment) {
         indexFacts = environment.getIndexFacts();
         normalizer = environment.getLinguistics().getNormalizer();
-        segmenter = environment.getLinguistics().getSegmenter();
-        detector = environment.getLinguistics().getDetector();
-        this.environment = environment;
 
-        Tuple2<String, Version> version = environment.getLinguistics().getVersion(Linguistics.Component.SEGMENTER);
-        localSegmenterBackend = version.first;
-        localSegmenterVersion = version.second;
+        this.environment = environment;
     }
 
+
     @Override
     public QueryTree parse(Parsable query) {
         indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict());
+        connectedItems.clear();
+        identifiedItems.clear();
         this.query = query;
 
         return buildTree();
 
     }
 
+
     public QueryTree buildTree() {
         Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getBytes()).get();
         if (inspector.field("error_message").valid()){
@@ -158,26 +166,16 @@ public class SelectParser implements Parser {
         }
 
         Item root = walkJson(inspector);
+        connectItems();
         QueryTree newTree = new QueryTree(root);
 
         return newTree;
     }
-    private static final String AND = "and";
-    private static final String AND_NOT = "and_not";
-    private static final String OR = "or";
-    private static final String EQ = "equals";
-    private static final String RANGE = "range";
-    private static final String CONTAINS = "contains";
-    private static final String MATCHES = "matches";
-    private static final String CALL = "call";
-    private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND);
 
 
     public Item walkJson(Inspector inspector){
         final Item[] item = {null};
         inspector.traverse((ObjectTraverser) (key, value) -> {
-
-
             String type = (FUNCTION_CALLS.contains(key)) ? CALL : key;
 
             switch (type) {
@@ -208,8 +206,6 @@ public class SelectParser implements Parser {
                 default:
                     throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT);
             }
-
-
         });
         return item[0];
     }
@@ -236,7 +232,6 @@ public class SelectParser implements Parser {
     }
 
 
-
     private void addItemsFromInspector(CompositeItem item, Inspector inspector){
         if (inspector.type() == ARRAY){
             inspector.traverse((ArrayTraverser) (index, new_value) -> {
@@ -253,6 +248,7 @@ public class SelectParser implements Parser {
         }
     }
 
+
     private Inspector getChildren(Inspector inspector){
         HashMap<Integer, Inspector> children = new HashMap<>();
         if (inspector.type() == ARRAY){
@@ -262,6 +258,9 @@ public class SelectParser implements Parser {
             if (inspector.field("children").valid()){
                 return inspector.field("children");
             }
+            if (inspector.field(1).valid()){
+                return inspector.field(1);
+            }
         }
         return null;
     }
@@ -284,6 +283,7 @@ public class SelectParser implements Parser {
         return children;
     }
 
+
     private Inspector getAnnotations(Inspector inspector){
         if (inspector.type() == OBJECT && inspector.field("attributes").valid()){
             return inspector.field("attributes");
@@ -291,6 +291,7 @@ public class SelectParser implements Parser {
         return null;
     }
 
+
     private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){
         HashMap<String, Inspector> attributes = new HashMap<>();
         if (annotation.type() == OBJECT){
@@ -301,6 +302,7 @@ public class SelectParser implements Parser {
         return attributes;
     }
 
+
     private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){
         HashMap<String, Inspector> attributes = new HashMap<>();
         if (inspector.type() == OBJECT && inspector.field("attributes").valid()){
@@ -311,14 +313,50 @@ public class SelectParser implements Parser {
         return attributes;
     }
 
+
     private <T> T getAnnotation(String annotationName, HashMap<String, Inspector> annotations, Class<T> expectedClass, T defaultValue) {
         return (annotations.get(annotationName) == null) ? defaultValue : expectedClass.cast(annotations.get(annotationName).asString());
     }
 
+
+    private Boolean getBoolAnnotation(String annotationName, HashMap<String, Inspector> annotations, Boolean defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return annotation.asBool();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Integer getIntegerAnnotation(String annotationName, HashMap<String, Inspector> annotations, Integer defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return (int)annotation.asLong();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Double getDoubleAnnotation(String annotationName, HashMap<String, Inspector> annotations, Double defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return annotation.asDouble();
+            }
+        }
+        return defaultValue;
+    }
+
+
     private Inspector getAnnotationAsInspectorOrNull(String annotationName, HashMap<String, Inspector> annotations) {
         return annotations.get(annotationName);
     }
 
+
     @NonNull
     private CompositeItem buildAnd(String key, Inspector value) {
         AndItem andItem = new AndItem();
@@ -327,6 +365,7 @@ public class SelectParser implements Parser {
         return andItem;
     }
 
+
     @NonNull
     private CompositeItem buildNotAnd(String key, Inspector value) {
         NotItem notItem = new NotItem();
@@ -336,7 +375,6 @@ public class SelectParser implements Parser {
     }
 
 
-
     @NonNull
     private CompositeItem buildOr(String key, Inspector value) {
         OrItem orItem = new OrItem();
@@ -344,6 +382,7 @@ public class SelectParser implements Parser {
         return orItem;
     }
 
+
     @NonNull
     private CompositeItem buildWeakAnd(String key, Inspector value) {
         WeakAndItem weakAnd = new WeakAndItem();
@@ -353,10 +392,10 @@ public class SelectParser implements Parser {
         if (annotations != null){
             annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
                 if (TARGET_NUM_HITS.equals(annotation_name)){
-                    weakAnd.setN((Integer.class.cast(annotation_value.asDouble())));
+                    weakAnd.setN((int)(annotation_value.asDouble()));
                 }
                 if (SCORE_THRESHOLD.equals(annotation_name)){
-                    weakAnd.setScoreThreshold((Integer.class.cast(annotation_value.asDouble())));
+                    weakAnd.setScoreThreshold((int)(annotation_value.asDouble()));
                 }
             });
         }
@@ -364,11 +403,28 @@ public class SelectParser implements Parser {
         return weakAnd;
     }
 
+
     @NonNull
     private <T extends TaggableItem> T leafStyleSettings(Inspector annotations, @NonNull T out) {
         {
             if (annotations != null) {
+                Inspector  itemConnectivity= getAnnotationAsInspectorOrNull(CONNECTIVITY, getAnnotationMapFromAnnotationInspector(annotations));
+                if (itemConnectivity != null) {
+                    Integer[] id = {null};
+                    Double[] weight = {null};
+                    itemConnectivity.traverse((ObjectTraverser) (key, value) -> {
+                        switch (key){
+                            case CONNECTION_ID:
+                                id[0] = (int) value.asLong();
+                            case CONNECTION_WEIGHT:
+                                weight[0] = value.asDouble();
+                        }
+                    });
+                    connectedItems.add(new ConnectedItem(out, id[0], weight[0]));
+                }
+
                 annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+
                     if (SIGNIFICANCE.equals(annotation_name)) {
                         if (annotation_value != null) {
                             out.setSignificance(annotation_value.asDouble());
@@ -376,8 +432,8 @@ public class SelectParser implements Parser {
                     }
                     if (UNIQUE_ID.equals(annotation_name)) {
                         if (annotation_value != null) {
-                            out.setUniqueID(Integer.class.cast(annotation_name));
-                            identifiedItems.put(Integer.class.cast(annotation_name), out);
+                            out.setUniqueID((int)annotation_value.asLong());
+                            identifiedItems.put((int)annotation_value.asLong(), out);
                         }
                     }
                 });
@@ -435,7 +491,7 @@ public class SelectParser implements Parser {
         annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
             if (HIT_LIMIT.equals(annotation_name)) {
                 if (annotation_value != null) {
-                    hitLimit[0] = Integer.class.cast(annotation_value);
+                    hitLimit[0] = (int)(annotation_value.asDouble());
                 }
             }
         });
@@ -445,10 +501,10 @@ public class SelectParser implements Parser {
         if (hitLimit[0] != null) {
             annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
                 if (ASCENDING_HITS_ORDER.equals(annotation_name)) {
-                    ascending[0] = Boolean.class.cast(annotation_value);
+                    ascending[0] = annotation_value.asBool();
                 }
                 if (DESCENDING_HITS_ORDER.equals(annotation_name)) {
-                    descending[0] = Boolean.class.cast(annotation_value);
+                    descending[0] = annotation_value.asBool();
                 }
 
             });
@@ -463,7 +519,6 @@ public class SelectParser implements Parser {
     }
 
 
-
     @NonNull
     private Item buildRange(String key, Inspector value) {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
@@ -484,16 +539,19 @@ public class SelectParser implements Parser {
         final Number[] bounds = {null, null};
         final String[] operators = {null, null};
         boundInspector.traverse((ObjectTraverser) (operator, bound) -> {
+            if (bound.type() == STRING) {
+                throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.");
+            }
             if (operator.equals("=")) {
-                bounds[0] = Number.class.cast(bound.asLong());
+                bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
                 operators[0] = operator;
                 equals[0] = true;
             }
             if (operator.equals(">=") || operator.equals(">")){
-                bounds[0] = Number.class.cast(bound.asLong());
+                bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
                 operators[0] = operator;
             } else if (operator.equals("<=") || operator.equals("<")){
-                bounds[1] = Number.class.cast(bound.asLong());
+                bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
                 operators[1] = operator;
             }
 
@@ -531,32 +589,25 @@ public class SelectParser implements Parser {
 
     }
 
+
     @NonNull
     private IntItem buildLessThanOrEquals(String field, String bound) {
         return new IntItem("[;" + bound + "]", field);
     }
 
+
     @NonNull
     private IntItem buildGreaterThan(String field, String bound) {
         return new IntItem(">" + bound, field);
 
     }
 
+
     @NonNull
     private IntItem buildLessThan(String field, String bound) {
         return new IntItem("<" + bound, field);
     }
 
-    /*
-    @NonNull
-    private IntItem buildEquals(OperatorNode<ExpressionOperator> ast) {
-        IntItem number = new IntItem(fetchConditionWord(ast), fetchConditionIndex(ast));
-        if (isIndexOnLeftHandSide(ast)) {
-            return leafStyleSettings(ast.getArgument(1, OperatorNode.class), number);
-        } else {
-            return leafStyleSettings(ast.getArgument(0, OperatorNode.class), number);
-        }
-    }*/
 
     @NonNull
     private IntItem instantiateRangeItem(Number lowerBound, Number upperBound, String field, boolean bounds_left_open, boolean bounds_right_open) {
@@ -595,22 +646,24 @@ public class SelectParser implements Parser {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
 
         Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size());
-        Integer target_num_hits= getAnnotation(TARGET_NUM_HITS, annotations, Integer.class, DEFAULT_TARGET_NUM_HITS);
+        Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS);
 
         WandItem out = new WandItem(children.get(0).asString(), target_num_hits);
 
-        Double scoreThreshold = getAnnotation(SCORE_THRESHOLD, annotations, Double.class, null);
+        Double scoreThreshold = getDoubleAnnotation(SCORE_THRESHOLD, annotations, null);
+
         if (scoreThreshold != null) {
             out.setScoreThreshold(scoreThreshold);
         }
 
-        Double thresholdBoostFactor = getAnnotation(THRESHOLD_BOOST_FACTOR, annotations, Double.class, null);
+        Double thresholdBoostFactor = getDoubleAnnotation(THRESHOLD_BOOST_FACTOR, annotations, null);
         if (thresholdBoostFactor != null) {
             out.setThresholdBoostFactor(thresholdBoostFactor);
         }
         return fillWeightedSet(value, children, out);
     }
 
+
     @NonNull
     private WeightedSetItem fillWeightedSet(Inspector value, HashMap<Integer, Inspector> children, @NonNull WeightedSetItem out) {
         addItems(children, out);
@@ -632,17 +685,24 @@ public class SelectParser implements Parser {
         }
     }
 
+
     private static void addStringItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
         //{"a":1, "b":2}
-        children.get(1).traverse((ObjectTraverser) (key, value) -> out.addToken(key, Integer.class.cast(value.asLong())));
+        children.get(1).traverse((ObjectTraverser) (key, value) -> {
+            if (value.type() == STRING){
+                throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.");
+            }
+            out.addToken(key, (int)value.asLong());
+        });
     }
 
+
     private static void addLongItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
         //[[11,1], [37,2]]
         children.get(1).traverse((ArrayTraverser) (index, pair) -> {
             List<Integer> pairValues = new ArrayList<>();
             pair.traverse((ArrayTraverser) (pairIndex, pairValue) -> {
-                pairValues.add(Integer.class.cast(pairValue.asLong()));
+                pairValues.add((int)pairValue.asLong());
             });
             Preconditions.checkArgument(pairValues.size() == 2,
                     "Expected item and weight, got %s.", pairValues);
@@ -661,6 +721,7 @@ public class SelectParser implements Parser {
         return leafStyleSettings(getAnnotations(value), regExp);
     }
 
+
     @NonNull
     private Item buildWeightedSet(String key, Inspector value) {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
@@ -669,6 +730,7 @@ public class SelectParser implements Parser {
         return fillWeightedSet(value, children, new WeightedSetItem(field));
     }
 
+
     @NonNull
     private Item buildDotProduct(String key, Inspector value) {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
@@ -677,6 +739,7 @@ public class SelectParser implements Parser {
         return fillWeightedSet(value, children, new DotProductItem(field));
     }
 
+
     @NonNull
     private Item buildPredicate(String key, Inspector value) {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
@@ -691,10 +754,17 @@ public class SelectParser implements Parser {
         List<Inspector> argumentList = valueListFromInspector(getChildren(value));
 
         // Adding attributes
-        argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addFeature(attrKey, attrValue.asString()));
+        argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> {
+            if (attrValue.type() == ARRAY){
+                List<Inspector> attributes = valueListFromInspector(attrValue);
+                attributes.forEach( (attribute) -> item.addFeature(attrKey, attribute.asString()));
+            } else {
+                item.addFeature(attrKey, attrValue.asString());
+            }
+        });
 
         // Adding range attributes
-        argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, Integer.class.cast(attrValue.asDouble())));
+        argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, (int)attrValue.asDouble()));
 
         return leafStyleSettings(getAnnotations(value), item);
     }
@@ -707,6 +777,7 @@ public class SelectParser implements Parser {
         return rankItem;
     }
 
+
     @NonNull
     private Item buildTermSearch(String key, Inspector value) {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
@@ -714,15 +785,34 @@ public class SelectParser implements Parser {
 
         return instantiateLeafItem(field, key, value);
     }
+
+
+    private String getInspectorKey(Inspector inspector){
+        String[] actualKey = {""};
+        if (inspector.type() == OBJECT){
+            inspector.traverse((ObjectTraverser) (key, value) -> {
+                actualKey[0] = key;
+
+            });
+        }
+        return actualKey[0];
+    }
+
+
     @NonNull
     private Item instantiateLeafItem(String field, String key, Inspector value) {
-        if (CALL.contains(key)) {
+        List<Inspector> possibleLeafFunction = valueListFromInspector(value);
+        String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : "";
+        if (FUNCTION_CALLS.contains(key)) {
             return instantiateCompositeLeaf(field, key, value);
+        } else if(!possibleLeafFunctionName.equals("")){
+           return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName));
         } else {
             return instantiateWordItem(field, key, value);
         }
     }
 
+
     @NonNull
     private Item instantiateCompositeLeaf(String field, String key, Inspector value) {
         switch (key) {
@@ -750,20 +840,21 @@ public class SelectParser implements Parser {
         return instantiateWordItem(field, wordData, key, value, false, decideParsingLanguage(value, wordData));
     }
 
+
     @NonNull
     private Item instantiateWordItem(String field, String rawWord, String key, Inspector value, boolean exactMatch, Language language) {
         String wordData = rawWord;
         HashMap<String, Inspector> annotations = getAnnotationMap(value);
 
-        if (getAnnotation(NFKC, annotations, Boolean.class, Boolean.FALSE)) {
+        if (getBoolAnnotation(NFKC, annotations, Boolean.FALSE)) {
             // NOTE: If this is set to FALSE (default), we will still NFKC normalize text data
             // during tokenization/segmentation, as that is always turned on also on the indexing side.
             wordData = normalizer.normalize(wordData);
         }
-        boolean fromQuery = getAnnotation(IMPLICIT_TRANSFORMS, annotations, Boolean.class, Boolean.TRUE);
-        boolean prefixMatch = getAnnotation(PREFIX, annotations, Boolean.class, Boolean.FALSE);
-        boolean suffixMatch = getAnnotation(SUFFIX, annotations, Boolean.class, Boolean.FALSE);
-        boolean substrMatch = getAnnotation(SUBSTRING,annotations, Boolean.class, Boolean.FALSE);
+        boolean fromQuery = getBoolAnnotation(IMPLICIT_TRANSFORMS, annotations,  Boolean.TRUE);
+        boolean prefixMatch = getBoolAnnotation(PREFIX, annotations, Boolean.FALSE);
+        boolean suffixMatch = getBoolAnnotation(SUFFIX, annotations,  Boolean.FALSE);
+        boolean substrMatch = getBoolAnnotation(SUBSTRING,annotations, Boolean.FALSE);
 
         Preconditions.checkArgument((prefixMatch ? 1 : 0)
                         + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2,
@@ -786,7 +877,7 @@ public class SelectParser implements Parser {
         if (wordItem instanceof WordItem) {
             prepareWord(field, value, (WordItem) wordItem);
         }
-        if (language != Language.ENGLISH) // mark the language used, unless it's the default
+        if (language != Language.ENGLISH)
             ((Item)wordItem).setLanguage(language);
 
         return (Item) leafStyleSettings(getAnnotations(value), wordItem);
@@ -811,6 +902,7 @@ public class SelectParser implements Parser {
         wordStyleSettings(value, wordItem);
     }
 
+
     private void wordStyleSettings(Inspector value, WordItem out) {
         HashMap<String, Inspector> annotations = getAnnotationMap(value);
 
@@ -818,33 +910,36 @@ public class SelectParser implements Parser {
         if (origin != null) {
             out.setOrigin(origin);
         }
+        if (annotations !=  null){
+            Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null));
+            if (usePositionData != null) {
+                out.setPositionData(usePositionData);
+            }
+            Boolean stem = getBoolAnnotation(STEM, annotations, null);
+            if (stem != null) {
+                out.setStemmed(!stem);
+            }
 
-        Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null));
-        if (usePositionData != null) {
-            out.setPositionData(usePositionData);
-        }
-        Boolean stem = Boolean.getBoolean(getAnnotation(STEM, annotations, String.class, null));
-        if (stem != null) {
-            out.setStemmed(!stem);
-        }
-        Boolean normalizeCase = Boolean.getBoolean(getAnnotation(NORMALIZE_CASE, annotations, String.class, null));
-        if (normalizeCase != null) {
-            out.setLowercased(!normalizeCase);
-        }
-        Boolean accentDrop = Boolean.getBoolean(getAnnotation(ACCENT_DROP, annotations, String.class, null));
-        if (accentDrop != null) {
-            out.setNormalizable(accentDrop);
-        }
-        Boolean andSegmenting = Boolean.getBoolean(getAnnotation(AND_SEGMENTING, annotations, String.class, null));
-        if (andSegmenting != null) {
-            if (andSegmenting) {
-                out.setSegmentingRule(SegmentingRule.BOOLEAN_AND);
-            } else {
-                out.setSegmentingRule(SegmentingRule.PHRASE);
+            Boolean normalizeCase = getBoolAnnotation(NORMALIZE_CASE, annotations, null);
+            if (normalizeCase != null) {
+                out.setLowercased(!normalizeCase);
+            }
+            Boolean accentDrop = getBoolAnnotation(ACCENT_DROP, annotations, null);
+            if (accentDrop != null) {
+                out.setNormalizable(accentDrop);
+            }
+            Boolean andSegmenting = getBoolAnnotation(AND_SEGMENTING, annotations,  null);
+            if (andSegmenting != null) {
+                if (andSegmenting) {
+                    out.setSegmentingRule(SegmentingRule.BOOLEAN_AND);
+                } else {
+                    out.setSegmentingRule(SegmentingRule.PHRASE);
+                }
             }
         }
     }
 
+
     private Substring getOrigin(Inspector annotations) {
         if (annotations != null) {
             Inspector origin = getAnnotationAsInspectorOrNull(ORIGIN, getAnnotationMapFromAnnotationInspector(annotations));
@@ -875,6 +970,7 @@ public class SelectParser implements Parser {
         return null;
     }
 
+
     @NonNull
     private Item instantiateSameElementItem(String field, String key, Inspector value) {
         assertHasOperator(key, SAME_ELEMENT);
@@ -888,6 +984,7 @@ public class SelectParser implements Parser {
         return sameElement;
     }
 
+
     @NonNull
     private Item instantiatePhraseItem(String field, String key, Inspector value) {
         assertHasOperator(key, PHRASE);
@@ -897,15 +994,15 @@ public class SelectParser implements Parser {
         phrase.setIndexName(field);
         HashMap<Integer, Inspector> children = getChildrenMap(value);
 
-        for (Inspector word :  children.values()){
-            phrase.addItem(instantiateWordItem(word.asString(), key, value));
-
-        }
+        for (Inspector word :  children.values())
+            if (word.type() == STRING) phrase.addItem(new WordItem(word.asString()));
+            else if (word.type() == OBJECT && word.field(PHRASE).valid()) {
+                    phrase.addItem(instantiatePhraseItem(field, key, getChildren(word)));
+            }
         return leafStyleSettings(getAnnotations(value), phrase);
     }
 
 
-
     @NonNull
     private Item instantiateNearItem(String field, String key, Inspector value) {
         assertHasOperator(key, NEAR);
@@ -916,17 +1013,18 @@ public class SelectParser implements Parser {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
 
         for (Inspector word :  children.values()){
-            near.addItem(instantiateWordItem(word.asString(), key, value));
+            near.addItem(new WordItem(word.asString(), field));
         }
 
-        Integer distance = getAnnotation(DISTANCE, getAnnotationMap(value), Integer.class, null);
+        Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null);
 
         if (distance != null) {
-            near.setDistance(distance);
+            near.setDistance((int)distance);
         }
         return near;
     }
 
+
     @NonNull
     private Item instantiateONearItem(String field, String key, Inspector value) {
         assertHasOperator(key, ONEAR);
@@ -936,10 +1034,10 @@ public class SelectParser implements Parser {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
 
         for (Inspector word :  children.values()){
-            onear.addItem(instantiateWordItem(word.asString(), key, value));
+            onear.addItem(new WordItem(word.asString(), field));
         }
 
-        Integer distance = getAnnotation(DISTANCE, getAnnotationMap(value), Integer.class, null);
+        Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null);
         if (distance != null) {
             onear.setDistance(distance);
         }
@@ -958,12 +1056,12 @@ public class SelectParser implements Parser {
 
         for (Inspector word :  children.values()){
             if (word.type() == STRING || word.type() == LONG || word.type() == DOUBLE){
-                equiv.addItem(instantiateWordItem(word.asString(), key, value));
+                equiv.addItem(new WordItem(word.asString(), field));
             }
             if (word.type() == OBJECT){
                 word.traverse((ObjectTraverser) (key2, value2) -> {
                     assertHasOperator(key2, PHRASE);
-                    equiv.addItem(instantiatePhraseItem(word.asString(), key, value2));
+                    equiv.addItem(instantiatePhraseItem(field, key2, value2));
                 });
             }
         }
@@ -971,6 +1069,7 @@ public class SelectParser implements Parser {
         return leafStyleSettings(getAnnotations(value), equiv);
     }
 
+
     private Item instantiateWordAlternativesItem(String field, String key, Inspector value) {
         HashMap<Integer, Inspector> children = getChildrenMap(value);
         Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size());
@@ -985,9 +1084,7 @@ public class SelectParser implements Parser {
     }
 
 
-
-
-
+    //  Not in use yet
     @NonNull
     private String getIndex(String field) {
         Preconditions.checkArgument(indexFactsSession.isIndex(field), "Field '%s' does not exist.", field);
@@ -996,9 +1093,8 @@ public class SelectParser implements Parser {
     }
 
 
-
     private static void assertHasOperator(String key, String expectedKey) {
-        Preconditions.checkArgument(key.equals(expectedKey), "Expected operator class %s, got %s.", expectedKey, key);
+        Preconditions.checkArgument(key.equals(expectedKey), "Expected operator %s, got %s.", expectedKey, key);
     }
 
 
@@ -1016,6 +1112,7 @@ public class SelectParser implements Parser {
         return new IllegalArgumentException(out.toString());
     }
 
+
     private List<Inspector> valueListFromInspector(Inspector inspector){
         List<Inspector> inspectorList = new ArrayList<>();
         inspector.traverse((ArrayTraverser) (key, value) -> inspectorList.add(value));
@@ -1023,4 +1120,30 @@ public class SelectParser implements Parser {
     }
 
 
+    private void connectItems() {
+        for (ConnectedItem entry : connectedItems) {
+            TaggableItem to = identifiedItems.get(entry.toId);
+            Preconditions.checkNotNull(to,
+                    "Item '%s' was specified to connect to item with ID %s, which does not "
+                            + "exist in the query.", entry.fromItem,
+                    entry.toId);
+            entry.fromItem.setConnectivity((Item) to, entry.weight);
+        }
+    }
+
+
+    private static final class ConnectedItem {
+
+        final double weight;
+        final int toId;
+        final TaggableItem fromItem;
+
+        ConnectedItem(TaggableItem fromItem, int toId, double weight) {
+            this.weight = weight;
+            this.toId = toId;
+            this.fromItem = fromItem;
+        }
+    }
+
+
 }
diff --git a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java
index 324bd8e4387..78a52acbedf 100644
--- a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java
@@ -1,8 +1,18 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.select;
 
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.ExactStringItem;
 import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.RegExpItem;
+import com.yahoo.prelude.query.SegmentingRule;
 import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.SubstringItem;
+import com.yahoo.prelude.query.SuffixItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
 import com.yahoo.prelude.query.WordItem;
 import com.yahoo.search.query.QueryTree;
 import com.yahoo.search.query.Select;
@@ -12,13 +22,15 @@ import com.yahoo.search.query.parser.ParserEnvironment;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.junit.Test;
-
 import java.util.Arrays;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 
 /**
@@ -34,11 +46,9 @@ public class SelectParserTestCase {
     @Test
     public void test_contains() throws Exception {
         JSONObject json = new JSONObject();
-        List<String> contains = Arrays.asList("default", "henrik");
+        List<String> contains = Arrays.asList("default", "foo");
         json.put("contains", contains);
-
-        System.out.println(json.toString());
-        assertParse(json.toString(), "default:henrik");
+        assertParse(json.toString(), "default:foo");
     }
 
     @Test
@@ -113,7 +123,6 @@ public class SelectParserTestCase {
 
         json_and_not.put("and_not", Arrays.asList(contains_json1, contains_json2));
 
-        System.out.println(json_and_not.toString());
         assertParse(json_and_not.toString(),
                 "+title:madonna -title:saint");
     }
@@ -129,7 +138,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:<500");
     }
@@ -144,7 +152,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:>500");
     }
@@ -160,7 +167,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:[;500]");
     }
@@ -175,7 +181,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:[500;]");
     }
@@ -190,7 +195,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:500");
     }
@@ -205,7 +209,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:<-500");
     }
@@ -220,7 +223,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:>-500");
     }
@@ -235,7 +237,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:[;-500]");
     }
@@ -250,7 +251,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:[-500;]");
     }
@@ -265,7 +265,6 @@ public class SelectParserTestCase {
 
         range_json.put("range", range);
 
-        System.out.println(range_json.toString());
         assertParse(range_json.toString(),
                 "price:-500");
     }
@@ -273,32 +272,24 @@ public class SelectParserTestCase {
     @Test
     public void testAnnotatedLessThan() {
         String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<\" : -500}], \"attributes\" : {\"filter\" : true} } }";
-        System.out.println(jsonString);
-
         assertParse(jsonString, "|price:<-500");
     }
 
     @Test
     public void testAnnotatedGreaterThan() {
         String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">\" : 500}], \"attributes\" : {\"filter\" : true} } }";
-        System.out.println(jsonString);
-
         assertParse(jsonString, "|price:>500");
     }
 
     @Test
     public void testAnnotatedLessThanOrEqual() {
         String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<=\" : -500}], \"attributes\" : {\"filter\" : true} } }";
-        System.out.println(jsonString);
-
         assertParse(jsonString, "|price:[;-500]");
     }
 
     @Test
     public void testAnnotatedGreaterThanOrEqual() {
         String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">=\" : 500}], \"attributes\" : {\"filter\" : true} } }";
-        System.out.println(jsonString);
-
         assertParse(jsonString, "|price:[500;]");
     }
 
@@ -306,8 +297,6 @@ public class SelectParserTestCase {
     @Test
     public void testAnnotatedEquality() {
         String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"=\" : -500}], \"attributes\" : {\"filter\" : true} } }";
-        System.out.println(jsonString);
-
         assertParse(jsonString, "|price:-500");
     }
 
@@ -331,31 +320,316 @@ public class SelectParserTestCase {
 
     @Test
     public void testSameElement() {
-        //System.out.println("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }");
-       // assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }",
-                //"baz:{f1:a f2:b}");
-        //assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 = 10);",
-        //        "baz:{f1:a f2:10}");
-        //assertParse("select foo from bar where baz contains sameElement(key contains \"a\", value.f2 = 10);",
-        //        "baz:{key:a value.f2:10}");
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }",
+                "baz:{f1:a f2:b}");
+
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, {\"range\":[\"f2\",{\"=\":10}] } ]} ] }",
+                "baz:{f1:a f2:10}");
+
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"key\", \"a\"] }, {\"range\":[\"value.f2\",{\"=\":10}] } ]} ] }",
+                "baz:{key:a value.f2:10}");
+    }
+
+    @Test
+    public void testPhrase() {
+        assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\"] } ] }",
+                "baz:\"a b\"");
+    }
+
+    @Test
+    public void testNestedPhrase() {
+        assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\", {\"phrase\" : [ \"c\", \"d\"] }] } ] }",
+                "baz:\"a b c d\"");
+    }
+
+    @Test
+    public void testStemming() {
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : false} } }").isStemmed());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : true} } }").isStemmed());
+        assertFalse(getRootWord("{ \"contains\": [\"baz\", \"colors\"] }").isStemmed());
+    }
+
+    @Test
+    public void testRaw() {
+        Item root = parseWhere("{ \"contains\":[ \"baz\", \"yoni jo dima\" ] }").getRoot();
+        assertTrue(root instanceof WordItem);
+        assertFalse(root instanceof ExactStringItem);
+        assertEquals("yoni jo dima", ((WordItem)root).getWord());
+
+        root = parseWhere("{ \"contains\": { \"children\" : [\"baz\", \"yoni jo dima\"], \"attributes\" : {\"grammar\" : \"raw\"} } }").getRoot();
+        assertTrue(root instanceof WordItem);
+        assertFalse(root instanceof ExactStringItem);
+        assertEquals("yoni jo dima", ((WordItem)root).getWord());
+    }
+
+    @Test
+    public void testAccentDropping() {
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : false} } }").isNormalizable());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : true} } }").isNormalizable());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isNormalizable());
+    }
+
+    @Test
+    public void testCaseNormalization() {
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : false} } }").isLowercased());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : true} } }").isLowercased());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isLowercased());
     }
 
+    @Test
+    public void testSegmentingRule() {
+        assertEquals(SegmentingRule.PHRASE,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : false} } }").getSegmentingRule());
+        assertEquals(SegmentingRule.BOOLEAN_AND,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : true} } }").getSegmentingRule());
+        assertEquals(SegmentingRule.LANGUAGE_DEFAULT,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").getSegmentingRule());
+    }
 
+    @Test
+    public void testNfkc() {
+        assertEquals("a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : false} } }").getWord());
+        assertEquals("\u00e5", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : true} } }").getWord());
+        assertEquals("No NKFC by default", "a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"] } } ").getWord());
+    }
 
+    @Test
+    public void testImplicitTransforms() {
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : false} } }").isFromQuery());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : true} } }").isFromQuery());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"] } }").isFromQuery());
+    }
 
+    @Test
+    public void testConnectivity() {
+        QueryTree parsed = parseWhere("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }");
+        assertEquals("AND title:madonna title:saint title:angel", parsed.toString());
+
+        AndItem root = (AndItem)parsed.getRoot();
+        WordItem first = (WordItem)root.getItem(0);
+        WordItem second = (WordItem)root.getItem(1);
+        WordItem third = (WordItem)root.getItem(2);
+        assertTrue(first.getConnectedItem() == third);
+        assertEquals(first.getConnectivity(), 7.0d, 1E-6);
+        assertNull(second.getConnectedItem());
+
+        assertParseFail("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }",
+                new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " +
+                "which does not exist in the query."));
+    }
 
+    @Test
+    public void testAnnotatedPhrase() {
+        QueryTree parsed = parseWhere("{ \"contains\": [\"baz\", { \"phrase\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"label\": \"hello world\" } } }] }");
+        assertEquals("baz:\"a b\"", parsed.toString());
+        PhraseItem phrase = (PhraseItem)parsed.getRoot();
+        assertEquals("hello world", phrase.getLabel());
+    }
 
+    @Test
+    public void testRange() {
+        QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": 1, \"<=\": 8 }] }");
+        assertEquals("baz:[1;8]", parsed.toString());
+    }
 
+    @Test
+    public void testNegativeRange() {
+        QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": -8, \"<=\": -1 }] }");
+        assertEquals("baz:[-8;-1]", parsed.toString());
+    }
 
+    @Test
+    public void testRangeIllegalArguments() {
+        assertParseFail("{ \"range\": [\"baz\", { \">=\": \"cox\", \"<=\": -1 }] }",
+                new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."));
+    }
 
+    @Test
+    public void testNear() {
+        assertParse("{ \"contains\": [\"description\", { \"near\": [\"a\", \"b\"] }] }",
+                "NEAR(2) description:a description:b");
+        assertParse("{ \"contains\": [\"description\", { \"near\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }",
+                "NEAR(100) description:a description:b");
+    }
 
+    @Test
+    public void testOrderedNear() {
+        assertParse("{ \"contains\": [\"description\", { \"onear\": [\"a\", \"b\"] }] }",
+                "ONEAR(2) description:a description:b");
+        assertParse("{ \"contains\": [\"description\", { \"onear\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }",
+                "ONEAR(100) description:a description:b");
+    }
 
+    @Test
+    public void testWand() {
+        assertParse("{ \"wand\": [\"description\", { \"a\": 1, \"b\": 2 }] }",
+                "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}");
+        assertParse("{ \"wand\": { \"children\": [\"description\", { \"a\": 1, \"b\": 2 }], \"attributes\": { \"scoreThreshold\": 13.3, \"targetNumHits\": 7, \"thresholdBoostFactor\": 2.3 } } }",
+                "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}");
+    }
 
-    /** Other methods */
-    private WordItem getRootWord(String yqlQuery) {
-        Item root = parseWhere(yqlQuery).getRoot();
-        assertTrue(root instanceof WordItem);
-        return (WordItem)root;
+    @Test
+    public void testNumericWand() {
+        String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}";
+        assertParse("{ \"wand\" : [\"description\", [[11,1], [37,2]] ]}", numWand);
+        assertParseFail("{ \"wand\" : [\"description\", 12] }",
+                new IllegalArgumentException("Expected ARRAY or OBJECT, got LONG."));
+    }
+
+    @Test
+    public void testWeightedSet() {
+        assertParse("{ \"weightedSet\" : [\"description\", {\"a\":1, \"b\":2} ]}",
+                "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}");
+        assertParseFail("{ \"weightedSet\" : [\"description\", {\"a\":\"g\", \"b\":2} ]}",
+                new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."));
+        assertParseFail("{ \"weightedSet\" : [\"description\" ]}",
+                new IllegalArgumentException("Expected 2 arguments, got 1."));
+    }
+
+    @Test
+    public void testDotProduct() {
+        assertParse("{ \"dotProduct\" : [\"description\", {\"a\":1, \"b\":2} ]}",
+                "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}");
+        assertParse("{ \"dotProduct\" : [\"description\", {\"a\":2} ]}",
+                "DOTPRODUCT description{[2]:\"a\"}");
+    }
+
+    @Test
+    public void testPredicate() {
+        assertParse("{ \"predicate\" : [\"predicate_field\", {\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23} ]}",
+                "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23");
+        assertParse("{ \"predicate\" : [\"predicate_field\", 0, \"void\" ]}",
+                "PREDICATE_QUERY_ITEM ");
+    }
+
+    @Test
+    public void testRank() {
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }",
+                "RANK a:A b:B");
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] } ] }",
+                "RANK a:A b:B c:C");
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"or\": [{ \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] }] }] }",
+                "RANK a:A (OR b:B c:C)");
+    }
+
+    @Test
+    public void testWeakAnd() {
+        assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }",
+                "WAND(100) a:A b:B");
+        assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetNumHits\": 37} }}",
+                "WAND(37) a:A b:B");
+
+        QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}");
+        assertEquals("WAND(100) a:A b:B", tree.toString());
+        assertEquals(WeakAndItem.class, tree.getRoot().getClass());
+        assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold());
+    }
+
+    @Test
+    public void testEquiv() {
+        assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"A\",\"B\"]}]}",
+                "EQUIV fieldName:A fieldName:B");
+
+        assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"phrase\" : [ \"new\",\"york\" ] } ] } ] }",
+                "EQUIV fieldName:ny fieldName:\"new york\"");
+
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\"] } ] }",
+                new IllegalArgumentException("Expected 2 or more arguments, got 1."));
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"nalle\" : [ \"void\" ] } ] } ] }",
+                new IllegalArgumentException("Expected operator phrase, got nalle."));
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\", 42]}]}",
+                new IllegalArgumentException("Word item word can not be empty"));
+    }
+
+    @Test
+    public void testAffixItems() {
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true} } }",
+                SuffixItem.class);
+
+
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"prefix\": true} } }",
+                PrefixItem.class);
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"substring\": true} } }",
+                SubstringItem.class);
+        assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"prefix\" : true} } }",
+                new IllegalArgumentException("Only one of prefix, substring and suffix can be set."));
+        assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"substring\" : true} } }",
+                new IllegalArgumentException("Only one of prefix, substring and suffix can be set."));
+    }
+
+    @Test
+    public void testLongNumberInSimpleExpression() {
+        assertParse("{ \"range\" : [ \"price\", { \"=\" : 8589934592 }]}",
+                "price:8589934592");
+    }
+
+    @Test
+    public void testNegativeLongNumberInSimpleExpression() {
+        assertParse("{ \"range\" : [ \"price\", { \"=\" : -8589934592 }]}",
+                "price:-8589934592");
+    }
+
+    @Test
+    public void testNegativeHitLimit() {
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": -38 } } }",
+                "foo:[0;1;-38]");
+    }
+
+    @Test
+    public void testRangeSearchHitPopulationOrdering() {
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": true} } }", "foo:[0;1;38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": false} } }", "foo:[0;1;-38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": true} } }", "foo:[0;1;-38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": false} } }", "foo:[0;1;38]");
+
+        boolean gotExceptionFromParse = false;
+        try {
+            parseWhere("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38, \"ascending\": true, \"descending\": false} } }");
+        } catch (IllegalArgumentException e) {
+            assertTrue("Expected information about abuse of settings.",
+                    e.getMessage().contains("both ascending and descending ordering set"));
+            gotExceptionFromParse = true;
+        }
+        assertTrue(gotExceptionFromParse);
+    }
+
+    // NB: Uses operator-keys to set bounds, not annotations
+    @Test
+    public void testOpenIntervals() {
+        assertParse("{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<=\" : 500.0 }] } }" +
+                        "select * from sources * where range(title, 0.0, 500.0);",
+                "title:[0.0;500.0]");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<\" : 500.0 }] } }",
+                "title:<0.0;500.0>");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<=\" : 500.0 }]  } }",
+                "title:<0.0;500.0]");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<\" : 500.0 }] } }",
+                "title:[0.0;500.0>");
+    }
+
+    @Test
+    public void testRegexp() {
+        QueryTree x = parseWhere("{ \"matches\" : [\"foo\", \"a b\"]}");
+        Item root = x.getRoot();
+        assertSame(RegExpItem.class, root.getClass());
+        assertEquals("a b", ((RegExpItem) root).stringValue());
+    }
+
+    @Test
+    public void testWordAlternatives() {
+        QueryTree x = parseWhere("{\"contains\" : [\"foo\", {\"alternatives\" : [{\"trees\": 1.0, \"tree\": 0.7}]}]}");
+        Item root = x.getRoot();
+        assertSame(WordAlternativesItem.class, root.getClass());
+        WordAlternativesItem alternatives = (WordAlternativesItem) root;
+        checkWordAlternativesContent(alternatives);
     }
 
 
@@ -372,10 +646,26 @@ public class SelectParserTestCase {
     /** Assert-methods */
     private void assertParse(String where, String expectedQueryTree) {
         String queryTree = parseWhere(where).toString();
-        System.out.println(queryTree);
         assertEquals(expectedQueryTree, queryTree);
     }
 
+    private void assertParseFail(String where, Throwable expectedException) {
+        try {
+            parseWhere(where).toString();
+        } catch (Throwable t) {
+            assertEquals(expectedException.getClass(), t.getClass());
+            assertEquals(expectedException.getMessage(), t.getMessage());
+            return;
+        }
+        fail("Parse succeeded: " + where);
+    }
+
+    private void assertRootClass(String where, Class<? extends Item> expectedRootClass) {
+        assertEquals(expectedRootClass, parseWhere(where).getRoot().getClass());
+    }
+
+
+
 
     /** Parse-methods*/
 
@@ -401,4 +691,33 @@ public class SelectParserTestCase {
 
 
 
+    /** Other methods */
+    private WordItem getRootWord(String yqlQuery) {
+        Item root = parseWhere(yqlQuery).getRoot();
+        assertTrue(root instanceof WordItem);
+        return (WordItem)root;
+    }
+
+    private void checkWordAlternativesContent(WordAlternativesItem alternatives) {
+        boolean seenTree = false;
+        boolean seenForest = false;
+        final String forest = "trees";
+        final String tree = "tree";
+        assertEquals(2, alternatives.getAlternatives().size());
+        for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) {
+            if (tree.equals(alternative.word)) {
+                assertFalse("Duplicate term introduced", seenTree);
+                seenTree = true;
+                assertEquals(.7d, alternative.exactness, 1e-15d);
+            } else if (forest.equals(alternative.word)) {
+                assertFalse("Duplicate term introduced", seenForest);
+                seenForest = true;
+                assertEquals(1.0d, alternative.exactness, 1e-15d);
+            } else {
+                fail("Unexpected term: " + alternative.word);
+            }
+        }
+    }
+
+
 }