From df878ed3d499559d06545884bae94d1368696dc4 Mon Sep 17 00:00:00 2001 From: Henrik Date: Thu, 2 Aug 2018 15:35:46 +0200 Subject: SelectParser now handles ''all'' testcases from YQLParserTestCase. Class may need some cleaning and removal of unuses code --- .../java/com/yahoo/search/query/SelectParser.java | 337 +++++++++++------ .../com/yahoo/select/SelectParserTestCase.java | 397 +++++++++++++++++++-- 2 files changed, 588 insertions(+), 146 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 559e0ab034e..779223754f4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -5,6 +5,7 @@ import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; +import com.yahoo.data.access.simple.Value; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; @@ -45,6 +46,7 @@ import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.Parser; import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.yql.YqlParser; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -69,11 +71,8 @@ public class SelectParser implements Parser { Parsable query; private final IndexFacts indexFacts; private final Map identifiedItems = LazyMap.newHashMap(); + private final List connectedItems = new ArrayList<>(); private final Normalizer normalizer; - private final Segmenter segmenter; - private final Detector detector; - private final String localSegmenterBackend; - private final Version localSegmenterVersion; private final ParserEnvironment environment; private IndexFacts.Session indexFactsSession; @@ -83,15 +82,14 @@ public class SelectParser implements Parser { private static final String DESCENDING_HITS_ORDER = "descending"; private static final String ASCENDING_HITS_ORDER = "ascending"; - - private static final Integer DEFAULT_HITS = 10; - private static final Integer DEFAULT_OFFSET = 0; private static final Integer DEFAULT_TARGET_NUM_HITS = 10; - static final String ORIGIN_LENGTH = "length"; - static final String ORIGIN_OFFSET = "offset"; - static final String ORIGIN = "origin"; - static final String ORIGIN_ORIGINAL = "original"; - + private static final String ORIGIN_LENGTH = "length"; + private static final String ORIGIN_OFFSET = "offset"; + private static final String ORIGIN = "origin"; + private static final String ORIGIN_ORIGINAL = "original"; + private static final String CONNECTION_ID = "id"; + private static final String CONNECTION_WEIGHT = "weight"; + private static final String CONNECTIVITY = "connectivity"; private static final String ANNOTATIONS = "annotations"; private static final String NFKC = "nfkc"; private static final String USER_INPUT_LANGUAGE = "language"; @@ -127,30 +125,40 @@ public class SelectParser implements Parser { private static final String WEAK_AND = "weakAnd"; private static final String WEIGHTED_SET = "weightedSet"; private static final String WEIGHT = "weight"; + private static final String AND = "and"; + private static final String AND_NOT = "and_not"; + private static final String OR = "or"; + private static final String EQ = "equals"; + private static final String RANGE = "range"; + private static final String CONTAINS = "contains"; + private static final String MATCHES = "matches"; + private static final String CALL = "call"; + private static final List FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); /**************************************/ + + public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); normalizer = environment.getLinguistics().getNormalizer(); - segmenter = environment.getLinguistics().getSegmenter(); - detector = environment.getLinguistics().getDetector(); - this.environment = environment; - Tuple2 version = environment.getLinguistics().getVersion(Linguistics.Component.SEGMENTER); - localSegmenterBackend = version.first; - localSegmenterVersion = version.second; + this.environment = environment; } + @Override public QueryTree parse(Parsable query) { indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict()); + connectedItems.clear(); + identifiedItems.clear(); this.query = query; return buildTree(); } + public QueryTree buildTree() { Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getBytes()).get(); if (inspector.field("error_message").valid()){ @@ -158,26 +166,16 @@ public class SelectParser implements Parser { } Item root = walkJson(inspector); + connectItems(); QueryTree newTree = new QueryTree(root); return newTree; } - private static final String AND = "and"; - private static final String AND_NOT = "and_not"; - private static final String OR = "or"; - private static final String EQ = "equals"; - private static final String RANGE = "range"; - private static final String CONTAINS = "contains"; - private static final String MATCHES = "matches"; - private static final String CALL = "call"; - private static final List FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); public Item walkJson(Inspector inspector){ final Item[] item = {null}; inspector.traverse((ObjectTraverser) (key, value) -> { - - String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; switch (type) { @@ -208,8 +206,6 @@ public class SelectParser implements Parser { default: throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT); } - - }); return item[0]; } @@ -236,7 +232,6 @@ public class SelectParser implements Parser { } - private void addItemsFromInspector(CompositeItem item, Inspector inspector){ if (inspector.type() == ARRAY){ inspector.traverse((ArrayTraverser) (index, new_value) -> { @@ -253,6 +248,7 @@ public class SelectParser implements Parser { } } + private Inspector getChildren(Inspector inspector){ HashMap children = new HashMap<>(); if (inspector.type() == ARRAY){ @@ -262,6 +258,9 @@ public class SelectParser implements Parser { if (inspector.field("children").valid()){ return inspector.field("children"); } + if (inspector.field(1).valid()){ + return inspector.field(1); + } } return null; } @@ -284,6 +283,7 @@ public class SelectParser implements Parser { return children; } + private Inspector getAnnotations(Inspector inspector){ if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ return inspector.field("attributes"); @@ -291,6 +291,7 @@ public class SelectParser implements Parser { return null; } + private HashMap getAnnotationMapFromAnnotationInspector(Inspector annotation){ HashMap attributes = new HashMap<>(); if (annotation.type() == OBJECT){ @@ -301,6 +302,7 @@ public class SelectParser implements Parser { return attributes; } + private HashMap getAnnotationMap(Inspector inspector){ HashMap attributes = new HashMap<>(); if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ @@ -311,14 +313,50 @@ public class SelectParser implements Parser { return attributes; } + private T getAnnotation(String annotationName, HashMap annotations, Class expectedClass, T defaultValue) { return (annotations.get(annotationName) == null) ? defaultValue : expectedClass.cast(annotations.get(annotationName).asString()); } + + private Boolean getBoolAnnotation(String annotationName, HashMap annotations, Boolean defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return annotation.asBool(); + } + } + return defaultValue; + } + + + private Integer getIntegerAnnotation(String annotationName, HashMap annotations, Integer defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return (int)annotation.asLong(); + } + } + return defaultValue; + } + + + private Double getDoubleAnnotation(String annotationName, HashMap annotations, Double defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return annotation.asDouble(); + } + } + return defaultValue; + } + + private Inspector getAnnotationAsInspectorOrNull(String annotationName, HashMap annotations) { return annotations.get(annotationName); } + @NonNull private CompositeItem buildAnd(String key, Inspector value) { AndItem andItem = new AndItem(); @@ -327,6 +365,7 @@ public class SelectParser implements Parser { return andItem; } + @NonNull private CompositeItem buildNotAnd(String key, Inspector value) { NotItem notItem = new NotItem(); @@ -336,7 +375,6 @@ public class SelectParser implements Parser { } - @NonNull private CompositeItem buildOr(String key, Inspector value) { OrItem orItem = new OrItem(); @@ -344,6 +382,7 @@ public class SelectParser implements Parser { return orItem; } + @NonNull private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); @@ -353,10 +392,10 @@ public class SelectParser implements Parser { if (annotations != null){ annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { if (TARGET_NUM_HITS.equals(annotation_name)){ - weakAnd.setN((Integer.class.cast(annotation_value.asDouble()))); + weakAnd.setN((int)(annotation_value.asDouble())); } if (SCORE_THRESHOLD.equals(annotation_name)){ - weakAnd.setScoreThreshold((Integer.class.cast(annotation_value.asDouble()))); + weakAnd.setScoreThreshold((int)(annotation_value.asDouble())); } }); } @@ -364,11 +403,28 @@ public class SelectParser implements Parser { return weakAnd; } + @NonNull private T leafStyleSettings(Inspector annotations, @NonNull T out) { { if (annotations != null) { + Inspector itemConnectivity= getAnnotationAsInspectorOrNull(CONNECTIVITY, getAnnotationMapFromAnnotationInspector(annotations)); + if (itemConnectivity != null) { + Integer[] id = {null}; + Double[] weight = {null}; + itemConnectivity.traverse((ObjectTraverser) (key, value) -> { + switch (key){ + case CONNECTION_ID: + id[0] = (int) value.asLong(); + case CONNECTION_WEIGHT: + weight[0] = value.asDouble(); + } + }); + connectedItems.add(new ConnectedItem(out, id[0], weight[0])); + } + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (SIGNIFICANCE.equals(annotation_name)) { if (annotation_value != null) { out.setSignificance(annotation_value.asDouble()); @@ -376,8 +432,8 @@ public class SelectParser implements Parser { } if (UNIQUE_ID.equals(annotation_name)) { if (annotation_value != null) { - out.setUniqueID(Integer.class.cast(annotation_name)); - identifiedItems.put(Integer.class.cast(annotation_name), out); + out.setUniqueID((int)annotation_value.asLong()); + identifiedItems.put((int)annotation_value.asLong(), out); } } }); @@ -435,7 +491,7 @@ public class SelectParser implements Parser { annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { if (HIT_LIMIT.equals(annotation_name)) { if (annotation_value != null) { - hitLimit[0] = Integer.class.cast(annotation_value); + hitLimit[0] = (int)(annotation_value.asDouble()); } } }); @@ -445,10 +501,10 @@ public class SelectParser implements Parser { if (hitLimit[0] != null) { annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { if (ASCENDING_HITS_ORDER.equals(annotation_name)) { - ascending[0] = Boolean.class.cast(annotation_value); + ascending[0] = annotation_value.asBool(); } if (DESCENDING_HITS_ORDER.equals(annotation_name)) { - descending[0] = Boolean.class.cast(annotation_value); + descending[0] = annotation_value.asBool(); } }); @@ -463,7 +519,6 @@ public class SelectParser implements Parser { } - @NonNull private Item buildRange(String key, Inspector value) { HashMap children = getChildrenMap(value); @@ -484,16 +539,19 @@ public class SelectParser implements Parser { final Number[] bounds = {null, null}; final String[] operators = {null, null}; boundInspector.traverse((ObjectTraverser) (operator, bound) -> { + if (bound.type() == STRING) { + throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."); + } if (operator.equals("=")) { - bounds[0] = Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); operators[0] = operator; equals[0] = true; } if (operator.equals(">=") || operator.equals(">")){ - bounds[0] = Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); operators[0] = operator; } else if (operator.equals("<=") || operator.equals("<")){ - bounds[1] = Number.class.cast(bound.asLong()); + bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); operators[1] = operator; } @@ -531,32 +589,25 @@ public class SelectParser implements Parser { } + @NonNull private IntItem buildLessThanOrEquals(String field, String bound) { return new IntItem("[;" + bound + "]", field); } + @NonNull private IntItem buildGreaterThan(String field, String bound) { return new IntItem(">" + bound, field); } + @NonNull private IntItem buildLessThan(String field, String bound) { return new IntItem("<" + bound, field); } - /* - @NonNull - private IntItem buildEquals(OperatorNode ast) { - IntItem number = new IntItem(fetchConditionWord(ast), fetchConditionIndex(ast)); - if (isIndexOnLeftHandSide(ast)) { - return leafStyleSettings(ast.getArgument(1, OperatorNode.class), number); - } else { - return leafStyleSettings(ast.getArgument(0, OperatorNode.class), number); - } - }*/ @NonNull private IntItem instantiateRangeItem(Number lowerBound, Number upperBound, String field, boolean bounds_left_open, boolean bounds_right_open) { @@ -595,22 +646,24 @@ public class SelectParser implements Parser { HashMap children = getChildrenMap(value); Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); - Integer target_num_hits= getAnnotation(TARGET_NUM_HITS, annotations, Integer.class, DEFAULT_TARGET_NUM_HITS); + Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); WandItem out = new WandItem(children.get(0).asString(), target_num_hits); - Double scoreThreshold = getAnnotation(SCORE_THRESHOLD, annotations, Double.class, null); + Double scoreThreshold = getDoubleAnnotation(SCORE_THRESHOLD, annotations, null); + if (scoreThreshold != null) { out.setScoreThreshold(scoreThreshold); } - Double thresholdBoostFactor = getAnnotation(THRESHOLD_BOOST_FACTOR, annotations, Double.class, null); + Double thresholdBoostFactor = getDoubleAnnotation(THRESHOLD_BOOST_FACTOR, annotations, null); if (thresholdBoostFactor != null) { out.setThresholdBoostFactor(thresholdBoostFactor); } return fillWeightedSet(value, children, out); } + @NonNull private WeightedSetItem fillWeightedSet(Inspector value, HashMap children, @NonNull WeightedSetItem out) { addItems(children, out); @@ -632,17 +685,24 @@ public class SelectParser implements Parser { } } + private static void addStringItems(HashMap children, WeightedSetItem out) { //{"a":1, "b":2} - children.get(1).traverse((ObjectTraverser) (key, value) -> out.addToken(key, Integer.class.cast(value.asLong()))); + children.get(1).traverse((ObjectTraverser) (key, value) -> { + if (value.type() == STRING){ + throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."); + } + out.addToken(key, (int)value.asLong()); + }); } + private static void addLongItems(HashMap children, WeightedSetItem out) { //[[11,1], [37,2]] children.get(1).traverse((ArrayTraverser) (index, pair) -> { List pairValues = new ArrayList<>(); pair.traverse((ArrayTraverser) (pairIndex, pairValue) -> { - pairValues.add(Integer.class.cast(pairValue.asLong())); + pairValues.add((int)pairValue.asLong()); }); Preconditions.checkArgument(pairValues.size() == 2, "Expected item and weight, got %s.", pairValues); @@ -661,6 +721,7 @@ public class SelectParser implements Parser { return leafStyleSettings(getAnnotations(value), regExp); } + @NonNull private Item buildWeightedSet(String key, Inspector value) { HashMap children = getChildrenMap(value); @@ -669,6 +730,7 @@ public class SelectParser implements Parser { return fillWeightedSet(value, children, new WeightedSetItem(field)); } + @NonNull private Item buildDotProduct(String key, Inspector value) { HashMap children = getChildrenMap(value); @@ -677,6 +739,7 @@ public class SelectParser implements Parser { return fillWeightedSet(value, children, new DotProductItem(field)); } + @NonNull private Item buildPredicate(String key, Inspector value) { HashMap children = getChildrenMap(value); @@ -691,10 +754,17 @@ public class SelectParser implements Parser { List argumentList = valueListFromInspector(getChildren(value)); // Adding attributes - argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addFeature(attrKey, attrValue.asString())); + argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> { + if (attrValue.type() == ARRAY){ + List attributes = valueListFromInspector(attrValue); + attributes.forEach( (attribute) -> item.addFeature(attrKey, attribute.asString())); + } else { + item.addFeature(attrKey, attrValue.asString()); + } + }); // Adding range attributes - argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, Integer.class.cast(attrValue.asDouble()))); + argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, (int)attrValue.asDouble())); return leafStyleSettings(getAnnotations(value), item); } @@ -707,6 +777,7 @@ public class SelectParser implements Parser { return rankItem; } + @NonNull private Item buildTermSearch(String key, Inspector value) { HashMap children = getChildrenMap(value); @@ -714,15 +785,34 @@ public class SelectParser implements Parser { return instantiateLeafItem(field, key, value); } + + + private String getInspectorKey(Inspector inspector){ + String[] actualKey = {""}; + if (inspector.type() == OBJECT){ + inspector.traverse((ObjectTraverser) (key, value) -> { + actualKey[0] = key; + + }); + } + return actualKey[0]; + } + + @NonNull private Item instantiateLeafItem(String field, String key, Inspector value) { - if (CALL.contains(key)) { + List possibleLeafFunction = valueListFromInspector(value); + String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : ""; + if (FUNCTION_CALLS.contains(key)) { return instantiateCompositeLeaf(field, key, value); + } else if(!possibleLeafFunctionName.equals("")){ + return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName)); } else { return instantiateWordItem(field, key, value); } } + @NonNull private Item instantiateCompositeLeaf(String field, String key, Inspector value) { switch (key) { @@ -750,20 +840,21 @@ public class SelectParser implements Parser { return instantiateWordItem(field, wordData, key, value, false, decideParsingLanguage(value, wordData)); } + @NonNull private Item instantiateWordItem(String field, String rawWord, String key, Inspector value, boolean exactMatch, Language language) { String wordData = rawWord; HashMap annotations = getAnnotationMap(value); - if (getAnnotation(NFKC, annotations, Boolean.class, Boolean.FALSE)) { + if (getBoolAnnotation(NFKC, annotations, Boolean.FALSE)) { // NOTE: If this is set to FALSE (default), we will still NFKC normalize text data // during tokenization/segmentation, as that is always turned on also on the indexing side. wordData = normalizer.normalize(wordData); } - boolean fromQuery = getAnnotation(IMPLICIT_TRANSFORMS, annotations, Boolean.class, Boolean.TRUE); - boolean prefixMatch = getAnnotation(PREFIX, annotations, Boolean.class, Boolean.FALSE); - boolean suffixMatch = getAnnotation(SUFFIX, annotations, Boolean.class, Boolean.FALSE); - boolean substrMatch = getAnnotation(SUBSTRING,annotations, Boolean.class, Boolean.FALSE); + boolean fromQuery = getBoolAnnotation(IMPLICIT_TRANSFORMS, annotations, Boolean.TRUE); + boolean prefixMatch = getBoolAnnotation(PREFIX, annotations, Boolean.FALSE); + boolean suffixMatch = getBoolAnnotation(SUFFIX, annotations, Boolean.FALSE); + boolean substrMatch = getBoolAnnotation(SUBSTRING,annotations, Boolean.FALSE); Preconditions.checkArgument((prefixMatch ? 1 : 0) + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, @@ -786,7 +877,7 @@ public class SelectParser implements Parser { if (wordItem instanceof WordItem) { prepareWord(field, value, (WordItem) wordItem); } - if (language != Language.ENGLISH) // mark the language used, unless it's the default + if (language != Language.ENGLISH) ((Item)wordItem).setLanguage(language); return (Item) leafStyleSettings(getAnnotations(value), wordItem); @@ -811,6 +902,7 @@ public class SelectParser implements Parser { wordStyleSettings(value, wordItem); } + private void wordStyleSettings(Inspector value, WordItem out) { HashMap annotations = getAnnotationMap(value); @@ -818,33 +910,36 @@ public class SelectParser implements Parser { if (origin != null) { out.setOrigin(origin); } + if (annotations != null){ + Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null)); + if (usePositionData != null) { + out.setPositionData(usePositionData); + } + Boolean stem = getBoolAnnotation(STEM, annotations, null); + if (stem != null) { + out.setStemmed(!stem); + } - Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null)); - if (usePositionData != null) { - out.setPositionData(usePositionData); - } - Boolean stem = Boolean.getBoolean(getAnnotation(STEM, annotations, String.class, null)); - if (stem != null) { - out.setStemmed(!stem); - } - Boolean normalizeCase = Boolean.getBoolean(getAnnotation(NORMALIZE_CASE, annotations, String.class, null)); - if (normalizeCase != null) { - out.setLowercased(!normalizeCase); - } - Boolean accentDrop = Boolean.getBoolean(getAnnotation(ACCENT_DROP, annotations, String.class, null)); - if (accentDrop != null) { - out.setNormalizable(accentDrop); - } - Boolean andSegmenting = Boolean.getBoolean(getAnnotation(AND_SEGMENTING, annotations, String.class, null)); - if (andSegmenting != null) { - if (andSegmenting) { - out.setSegmentingRule(SegmentingRule.BOOLEAN_AND); - } else { - out.setSegmentingRule(SegmentingRule.PHRASE); + Boolean normalizeCase = getBoolAnnotation(NORMALIZE_CASE, annotations, null); + if (normalizeCase != null) { + out.setLowercased(!normalizeCase); + } + Boolean accentDrop = getBoolAnnotation(ACCENT_DROP, annotations, null); + if (accentDrop != null) { + out.setNormalizable(accentDrop); + } + Boolean andSegmenting = getBoolAnnotation(AND_SEGMENTING, annotations, null); + if (andSegmenting != null) { + if (andSegmenting) { + out.setSegmentingRule(SegmentingRule.BOOLEAN_AND); + } else { + out.setSegmentingRule(SegmentingRule.PHRASE); + } } } } + private Substring getOrigin(Inspector annotations) { if (annotations != null) { Inspector origin = getAnnotationAsInspectorOrNull(ORIGIN, getAnnotationMapFromAnnotationInspector(annotations)); @@ -875,6 +970,7 @@ public class SelectParser implements Parser { return null; } + @NonNull private Item instantiateSameElementItem(String field, String key, Inspector value) { assertHasOperator(key, SAME_ELEMENT); @@ -888,6 +984,7 @@ public class SelectParser implements Parser { return sameElement; } + @NonNull private Item instantiatePhraseItem(String field, String key, Inspector value) { assertHasOperator(key, PHRASE); @@ -897,15 +994,15 @@ public class SelectParser implements Parser { phrase.setIndexName(field); HashMap children = getChildrenMap(value); - for (Inspector word : children.values()){ - phrase.addItem(instantiateWordItem(word.asString(), key, value)); - - } + for (Inspector word : children.values()) + if (word.type() == STRING) phrase.addItem(new WordItem(word.asString())); + else if (word.type() == OBJECT && word.field(PHRASE).valid()) { + phrase.addItem(instantiatePhraseItem(field, key, getChildren(word))); + } return leafStyleSettings(getAnnotations(value), phrase); } - @NonNull private Item instantiateNearItem(String field, String key, Inspector value) { assertHasOperator(key, NEAR); @@ -916,17 +1013,18 @@ public class SelectParser implements Parser { HashMap children = getChildrenMap(value); for (Inspector word : children.values()){ - near.addItem(instantiateWordItem(word.asString(), key, value)); + near.addItem(new WordItem(word.asString(), field)); } - Integer distance = getAnnotation(DISTANCE, getAnnotationMap(value), Integer.class, null); + Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); if (distance != null) { - near.setDistance(distance); + near.setDistance((int)distance); } return near; } + @NonNull private Item instantiateONearItem(String field, String key, Inspector value) { assertHasOperator(key, ONEAR); @@ -936,10 +1034,10 @@ public class SelectParser implements Parser { HashMap children = getChildrenMap(value); for (Inspector word : children.values()){ - onear.addItem(instantiateWordItem(word.asString(), key, value)); + onear.addItem(new WordItem(word.asString(), field)); } - Integer distance = getAnnotation(DISTANCE, getAnnotationMap(value), Integer.class, null); + Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); if (distance != null) { onear.setDistance(distance); } @@ -958,12 +1056,12 @@ public class SelectParser implements Parser { for (Inspector word : children.values()){ if (word.type() == STRING || word.type() == LONG || word.type() == DOUBLE){ - equiv.addItem(instantiateWordItem(word.asString(), key, value)); + equiv.addItem(new WordItem(word.asString(), field)); } if (word.type() == OBJECT){ word.traverse((ObjectTraverser) (key2, value2) -> { assertHasOperator(key2, PHRASE); - equiv.addItem(instantiatePhraseItem(word.asString(), key, value2)); + equiv.addItem(instantiatePhraseItem(field, key2, value2)); }); } } @@ -971,6 +1069,7 @@ public class SelectParser implements Parser { return leafStyleSettings(getAnnotations(value), equiv); } + private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { HashMap children = getChildrenMap(value); Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); @@ -985,9 +1084,7 @@ public class SelectParser implements Parser { } - - - + // Not in use yet @NonNull private String getIndex(String field) { Preconditions.checkArgument(indexFactsSession.isIndex(field), "Field '%s' does not exist.", field); @@ -996,9 +1093,8 @@ public class SelectParser implements Parser { } - private static void assertHasOperator(String key, String expectedKey) { - Preconditions.checkArgument(key.equals(expectedKey), "Expected operator class %s, got %s.", expectedKey, key); + Preconditions.checkArgument(key.equals(expectedKey), "Expected operator %s, got %s.", expectedKey, key); } @@ -1016,6 +1112,7 @@ public class SelectParser implements Parser { return new IllegalArgumentException(out.toString()); } + private List valueListFromInspector(Inspector inspector){ List inspectorList = new ArrayList<>(); inspector.traverse((ArrayTraverser) (key, value) -> inspectorList.add(value)); @@ -1023,4 +1120,30 @@ public class SelectParser implements Parser { } + private void connectItems() { + for (ConnectedItem entry : connectedItems) { + TaggableItem to = identifiedItems.get(entry.toId); + Preconditions.checkNotNull(to, + "Item '%s' was specified to connect to item with ID %s, which does not " + + "exist in the query.", entry.fromItem, + entry.toId); + entry.fromItem.setConnectivity((Item) to, entry.weight); + } + } + + + private static final class ConnectedItem { + + final double weight; + final int toId; + final TaggableItem fromItem; + + ConnectedItem(TaggableItem fromItem, int toId, double weight) { + this.weight = weight; + this.toId = toId; + this.fromItem = fromItem; + } + } + + } diff --git a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java index 324bd8e4387..78a52acbedf 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java @@ -1,8 +1,18 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.select; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SegmentingRule; import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.Select; @@ -12,13 +22,15 @@ import com.yahoo.search.query.parser.ParserEnvironment; import org.json.JSONException; import org.json.JSONObject; import org.junit.Test; - import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** @@ -34,11 +46,9 @@ public class SelectParserTestCase { @Test public void test_contains() throws Exception { JSONObject json = new JSONObject(); - List contains = Arrays.asList("default", "henrik"); + List contains = Arrays.asList("default", "foo"); json.put("contains", contains); - - System.out.println(json.toString()); - assertParse(json.toString(), "default:henrik"); + assertParse(json.toString(), "default:foo"); } @Test @@ -113,7 +123,6 @@ public class SelectParserTestCase { json_and_not.put("and_not", Arrays.asList(contains_json1, contains_json2)); - System.out.println(json_and_not.toString()); assertParse(json_and_not.toString(), "+title:madonna -title:saint"); } @@ -129,7 +138,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:<500"); } @@ -144,7 +152,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:>500"); } @@ -160,7 +167,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:[;500]"); } @@ -175,7 +181,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:[500;]"); } @@ -190,7 +195,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:500"); } @@ -205,7 +209,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:<-500"); } @@ -220,7 +223,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:>-500"); } @@ -235,7 +237,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:[;-500]"); } @@ -250,7 +251,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:[-500;]"); } @@ -265,7 +265,6 @@ public class SelectParserTestCase { range_json.put("range", range); - System.out.println(range_json.toString()); assertParse(range_json.toString(), "price:-500"); } @@ -273,32 +272,24 @@ public class SelectParserTestCase { @Test public void testAnnotatedLessThan() { String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<\" : -500}], \"attributes\" : {\"filter\" : true} } }"; - System.out.println(jsonString); - assertParse(jsonString, "|price:<-500"); } @Test public void testAnnotatedGreaterThan() { String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">\" : 500}], \"attributes\" : {\"filter\" : true} } }"; - System.out.println(jsonString); - assertParse(jsonString, "|price:>500"); } @Test public void testAnnotatedLessThanOrEqual() { String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<=\" : -500}], \"attributes\" : {\"filter\" : true} } }"; - System.out.println(jsonString); - assertParse(jsonString, "|price:[;-500]"); } @Test public void testAnnotatedGreaterThanOrEqual() { String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">=\" : 500}], \"attributes\" : {\"filter\" : true} } }"; - System.out.println(jsonString); - assertParse(jsonString, "|price:[500;]"); } @@ -306,8 +297,6 @@ public class SelectParserTestCase { @Test public void testAnnotatedEquality() { String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"=\" : -500}], \"attributes\" : {\"filter\" : true} } }"; - System.out.println(jsonString); - assertParse(jsonString, "|price:-500"); } @@ -331,31 +320,316 @@ public class SelectParserTestCase { @Test public void testSameElement() { - //System.out.println("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }"); - // assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }", - //"baz:{f1:a f2:b}"); - //assertParse("select foo from bar where baz contains sameElement(f1 contains \"a\", f2 = 10);", - // "baz:{f1:a f2:10}"); - //assertParse("select foo from bar where baz contains sameElement(key contains \"a\", value.f2 = 10);", - // "baz:{key:a value.f2:10}"); + assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }", + "baz:{f1:a f2:b}"); + + assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, {\"range\":[\"f2\",{\"=\":10}] } ]} ] }", + "baz:{f1:a f2:10}"); + + assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"key\", \"a\"] }, {\"range\":[\"value.f2\",{\"=\":10}] } ]} ] }", + "baz:{key:a value.f2:10}"); + } + + @Test + public void testPhrase() { + assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\"] } ] }", + "baz:\"a b\""); + } + + @Test + public void testNestedPhrase() { + assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\", {\"phrase\" : [ \"c\", \"d\"] }] } ] }", + "baz:\"a b c d\""); + } + + @Test + public void testStemming() { + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : false} } }").isStemmed()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : true} } }").isStemmed()); + assertFalse(getRootWord("{ \"contains\": [\"baz\", \"colors\"] }").isStemmed()); + } + + @Test + public void testRaw() { + Item root = parseWhere("{ \"contains\":[ \"baz\", \"yoni jo dima\" ] }").getRoot(); + assertTrue(root instanceof WordItem); + assertFalse(root instanceof ExactStringItem); + assertEquals("yoni jo dima", ((WordItem)root).getWord()); + + root = parseWhere("{ \"contains\": { \"children\" : [\"baz\", \"yoni jo dima\"], \"attributes\" : {\"grammar\" : \"raw\"} } }").getRoot(); + assertTrue(root instanceof WordItem); + assertFalse(root instanceof ExactStringItem); + assertEquals("yoni jo dima", ((WordItem)root).getWord()); + } + + @Test + public void testAccentDropping() { + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : false} } }").isNormalizable()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : true} } }").isNormalizable()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isNormalizable()); + } + + @Test + public void testCaseNormalization() { + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : false} } }").isLowercased()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : true} } }").isLowercased()); + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isLowercased()); } + @Test + public void testSegmentingRule() { + assertEquals(SegmentingRule.PHRASE, + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : false} } }").getSegmentingRule()); + assertEquals(SegmentingRule.BOOLEAN_AND, + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : true} } }").getSegmentingRule()); + assertEquals(SegmentingRule.LANGUAGE_DEFAULT, + getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").getSegmentingRule()); + } + @Test + public void testNfkc() { + assertEquals("a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : false} } }").getWord()); + assertEquals("\u00e5", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : true} } }").getWord()); + assertEquals("No NKFC by default", "a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"] } } ").getWord()); + } + @Test + public void testImplicitTransforms() { + assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : false} } }").isFromQuery()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : true} } }").isFromQuery()); + assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"] } }").isFromQuery()); + } + @Test + public void testConnectivity() { + QueryTree parsed = parseWhere("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }"); + assertEquals("AND title:madonna title:saint title:angel", parsed.toString()); + + AndItem root = (AndItem)parsed.getRoot(); + WordItem first = (WordItem)root.getItem(0); + WordItem second = (WordItem)root.getItem(1); + WordItem third = (WordItem)root.getItem(2); + assertTrue(first.getConnectedItem() == third); + assertEquals(first.getConnectivity(), 7.0d, 1E-6); + assertNull(second.getConnectedItem()); + + assertParseFail("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " + + "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }", + new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " + + "which does not exist in the query.")); + } + @Test + public void testAnnotatedPhrase() { + QueryTree parsed = parseWhere("{ \"contains\": [\"baz\", { \"phrase\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"label\": \"hello world\" } } }] }"); + assertEquals("baz:\"a b\"", parsed.toString()); + PhraseItem phrase = (PhraseItem)parsed.getRoot(); + assertEquals("hello world", phrase.getLabel()); + } + @Test + public void testRange() { + QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": 1, \"<=\": 8 }] }"); + assertEquals("baz:[1;8]", parsed.toString()); + } + @Test + public void testNegativeRange() { + QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": -8, \"<=\": -1 }] }"); + assertEquals("baz:[-8;-1]", parsed.toString()); + } + @Test + public void testRangeIllegalArguments() { + assertParseFail("{ \"range\": [\"baz\", { \">=\": \"cox\", \"<=\": -1 }] }", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + } + @Test + public void testNear() { + assertParse("{ \"contains\": [\"description\", { \"near\": [\"a\", \"b\"] }] }", + "NEAR(2) description:a description:b"); + assertParse("{ \"contains\": [\"description\", { \"near\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }", + "NEAR(100) description:a description:b"); + } + @Test + public void testOrderedNear() { + assertParse("{ \"contains\": [\"description\", { \"onear\": [\"a\", \"b\"] }] }", + "ONEAR(2) description:a description:b"); + assertParse("{ \"contains\": [\"description\", { \"onear\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }", + "ONEAR(100) description:a description:b"); + } + @Test + public void testWand() { + assertParse("{ \"wand\": [\"description\", { \"a\": 1, \"b\": 2 }] }", + "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}"); + assertParse("{ \"wand\": { \"children\": [\"description\", { \"a\": 1, \"b\": 2 }], \"attributes\": { \"scoreThreshold\": 13.3, \"targetNumHits\": 7, \"thresholdBoostFactor\": 2.3 } } }", + "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}"); + } - /** Other methods */ - private WordItem getRootWord(String yqlQuery) { - Item root = parseWhere(yqlQuery).getRoot(); - assertTrue(root instanceof WordItem); - return (WordItem)root; + @Test + public void testNumericWand() { + String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}"; + assertParse("{ \"wand\" : [\"description\", [[11,1], [37,2]] ]}", numWand); + assertParseFail("{ \"wand\" : [\"description\", 12] }", + new IllegalArgumentException("Expected ARRAY or OBJECT, got LONG.")); + } + + @Test + public void testWeightedSet() { + assertParse("{ \"weightedSet\" : [\"description\", {\"a\":1, \"b\":2} ]}", + "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}"); + assertParseFail("{ \"weightedSet\" : [\"description\", {\"a\":\"g\", \"b\":2} ]}", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + assertParseFail("{ \"weightedSet\" : [\"description\" ]}", + new IllegalArgumentException("Expected 2 arguments, got 1.")); + } + + @Test + public void testDotProduct() { + assertParse("{ \"dotProduct\" : [\"description\", {\"a\":1, \"b\":2} ]}", + "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}"); + assertParse("{ \"dotProduct\" : [\"description\", {\"a\":2} ]}", + "DOTPRODUCT description{[2]:\"a\"}"); + } + + @Test + public void testPredicate() { + assertParse("{ \"predicate\" : [\"predicate_field\", {\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23} ]}", + "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); + assertParse("{ \"predicate\" : [\"predicate_field\", 0, \"void\" ]}", + "PREDICATE_QUERY_ITEM "); + } + + @Test + public void testRank() { + assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", + "RANK a:A b:B"); + assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] } ] }", + "RANK a:A b:B c:C"); + assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"or\": [{ \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] }] }] }", + "RANK a:A (OR b:B c:C)"); + } + + @Test + public void testWeakAnd() { + assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", + "WAND(100) a:A b:B"); + assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetNumHits\": 37} }}", + "WAND(37) a:A b:B"); + + QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}"); + assertEquals("WAND(100) a:A b:B", tree.toString()); + assertEquals(WeakAndItem.class, tree.getRoot().getClass()); + assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); + } + + @Test + public void testEquiv() { + assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"A\",\"B\"]}]}", + "EQUIV fieldName:A fieldName:B"); + + assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"phrase\" : [ \"new\",\"york\" ] } ] } ] }", + "EQUIV fieldName:ny fieldName:\"new york\""); + + assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\"] } ] }", + new IllegalArgumentException("Expected 2 or more arguments, got 1.")); + assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"nalle\" : [ \"void\" ] } ] } ] }", + new IllegalArgumentException("Expected operator phrase, got nalle.")); + assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\", 42]}]}", + new IllegalArgumentException("Word item word can not be empty")); + } + + @Test + public void testAffixItems() { + assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true} } }", + SuffixItem.class); + + + assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"prefix\": true} } }", + PrefixItem.class); + assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"substring\": true} } }", + SubstringItem.class); + assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"prefix\" : true} } }", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"substring\" : true} } }", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + } + + @Test + public void testLongNumberInSimpleExpression() { + assertParse("{ \"range\" : [ \"price\", { \"=\" : 8589934592 }]}", + "price:8589934592"); + } + + @Test + public void testNegativeLongNumberInSimpleExpression() { + assertParse("{ \"range\" : [ \"price\", { \"=\" : -8589934592 }]}", + "price:-8589934592"); + } + + @Test + public void testNegativeHitLimit() { + assertParse( + "{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": -38 } } }", + "foo:[0;1;-38]"); + } + + @Test + public void testRangeSearchHitPopulationOrdering() { + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": true} } }", "foo:[0;1;38]"); + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": false} } }", "foo:[0;1;-38]"); + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": true} } }", "foo:[0;1;-38]"); + assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": false} } }", "foo:[0;1;38]"); + + boolean gotExceptionFromParse = false; + try { + parseWhere("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38, \"ascending\": true, \"descending\": false} } }"); + } catch (IllegalArgumentException e) { + assertTrue("Expected information about abuse of settings.", + e.getMessage().contains("both ascending and descending ordering set")); + gotExceptionFromParse = true; + } + assertTrue(gotExceptionFromParse); + } + + // NB: Uses operator-keys to set bounds, not annotations + @Test + public void testOpenIntervals() { + assertParse("{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<=\" : 500.0 }] } }" + + "select * from sources * where range(title, 0.0, 500.0);", + "title:[0.0;500.0]"); + assertParse( + "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<\" : 500.0 }] } }", + "title:<0.0;500.0>"); + assertParse( + "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<=\" : 500.0 }] } }", + "title:<0.0;500.0]"); + assertParse( + "{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<\" : 500.0 }] } }", + "title:[0.0;500.0>"); + } + + @Test + public void testRegexp() { + QueryTree x = parseWhere("{ \"matches\" : [\"foo\", \"a b\"]}"); + Item root = x.getRoot(); + assertSame(RegExpItem.class, root.getClass()); + assertEquals("a b", ((RegExpItem) root).stringValue()); + } + + @Test + public void testWordAlternatives() { + QueryTree x = parseWhere("{\"contains\" : [\"foo\", {\"alternatives\" : [{\"trees\": 1.0, \"tree\": 0.7}]}]}"); + Item root = x.getRoot(); + assertSame(WordAlternativesItem.class, root.getClass()); + WordAlternativesItem alternatives = (WordAlternativesItem) root; + checkWordAlternativesContent(alternatives); } @@ -372,10 +646,26 @@ public class SelectParserTestCase { /** Assert-methods */ private void assertParse(String where, String expectedQueryTree) { String queryTree = parseWhere(where).toString(); - System.out.println(queryTree); assertEquals(expectedQueryTree, queryTree); } + private void assertParseFail(String where, Throwable expectedException) { + try { + parseWhere(where).toString(); + } catch (Throwable t) { + assertEquals(expectedException.getClass(), t.getClass()); + assertEquals(expectedException.getMessage(), t.getMessage()); + return; + } + fail("Parse succeeded: " + where); + } + + private void assertRootClass(String where, Class expectedRootClass) { + assertEquals(expectedRootClass, parseWhere(where).getRoot().getClass()); + } + + + /** Parse-methods*/ @@ -401,4 +691,33 @@ public class SelectParserTestCase { + /** Other methods */ + private WordItem getRootWord(String yqlQuery) { + Item root = parseWhere(yqlQuery).getRoot(); + assertTrue(root instanceof WordItem); + return (WordItem)root; + } + + private void checkWordAlternativesContent(WordAlternativesItem alternatives) { + boolean seenTree = false; + boolean seenForest = false; + final String forest = "trees"; + final String tree = "tree"; + assertEquals(2, alternatives.getAlternatives().size()); + for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) { + if (tree.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenTree); + seenTree = true; + assertEquals(.7d, alternative.exactness, 1e-15d); + } else if (forest.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenForest); + seenForest = true; + assertEquals(1.0d, alternative.exactness, 1e-15d); + } else { + fail("Unexpected term: " + alternative.word); + } + } + } + + } -- cgit v1.2.3