diff options
author | Arne Juul <arnej@verizonmedia.com> | 2020-06-19 09:36:51 +0000 |
---|---|---|
committer | Arne Juul <arnej@verizonmedia.com> | 2020-06-19 09:36:51 +0000 |
commit | ae1297268fa447f7fb7f2f548d4f32c5327ec187 (patch) | |
tree | db10da7ae3848d24d9be728175b36b9596d1e19b /container-search/src/main/java/com | |
parent | b70147cbdc3139ae9ea43b8341b79b3e1cbbdd1b (diff) |
update SelectParser and add targetHits
* targetHits is now the preferred annotation replacing
targetNumHits (for wand,weakand,nearestneighbor), the
old name still works as an alias
* note: targetNumHits is still produced when serializing to YQL
* debugging/trace output will print the "targetHits" form
* add nearestNeighborItem support to SelectParser
* implement disclose() in nearestNeighborItem
Diffstat (limited to 'container-search/src/main/java/com')
4 files changed, 79 insertions, 10 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java index 52ef6c40a6a..be3ae913476 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.query; import com.google.common.annotations.Beta; import com.yahoo.compress.IntegerCompressor; +import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; @@ -83,7 +84,17 @@ public class NearestNeighborItem extends SimpleTaggableItem { buffer.append(",queryTensorName=").append(queryTensorName); buffer.append(",hnsw.exploreAdditionalHits=").append(hnswExploreAdditionalHits); buffer.append(",approximate=").append(approximate); - buffer.append(",targetNumHits=").append(targetNumHits).append("}"); + buffer.append(",targetHits=").append(targetNumHits).append("}"); + } + + @Override + public void disclose(Discloser discloser) { + super.disclose(discloser); + discloser.addProperty("field", field); + discloser.addProperty("queryTensorName", queryTensorName); + discloser.addProperty("hnsw.exploreAdditionalHits", hnswExploreAdditionalHits); + discloser.addProperty("approximate", approximate); + discloser.addProperty("targetHits", targetNumHits); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 775dca7c444..42e1bf46902 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -16,6 +16,7 @@ import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.OrItem; @@ -93,14 +94,17 @@ public class SelectParser implements Parser { private static final String ACCENT_DROP = "accentDrop"; private static final String ALTERNATIVES = "alternatives"; private static final String AND_SEGMENTING = "andSegmenting"; + private static final String APPROXIMATE = "approximate"; private static final String DISTANCE = "distance"; private static final String DOT_PRODUCT = "dotProduct"; private static final String EQUIV = "equiv"; private static final String FILTER = "filter"; private static final String HIT_LIMIT = "hitLimit"; + private static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; private static final String LABEL = "label"; private static final String NEAR = "near"; + private static final String NEAREST_NEIGHBOR = "nearestNeighbor"; private static final String NORMALIZE_CASE = "normalizeCase"; private static final String ONEAR = "onear"; private static final String PHRASE = "phrase"; @@ -114,6 +118,7 @@ public class SelectParser implements Parser { private static final String STEM = "stem"; private static final String SUBSTRING = "substring"; private static final String SUFFIX = "suffix"; + private static final String TARGET_HITS = "targetHits"; private static final String TARGET_NUM_HITS = "targetNumHits"; private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; private static final String UNIQUE_ID = "id"; @@ -130,7 +135,7 @@ public class SelectParser implements Parser { private static final String CONTAINS = "contains"; private static final String MATCHES = "matches"; private static final String CALL = "call"; - private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); + private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); @@ -259,6 +264,8 @@ public class SelectParser implements Parser { return buildWeightedSet(key, value); case DOT_PRODUCT: return buildDotProduct(key, value); + case NEAREST_NEIGHBOR: + return buildNearestNeighbor(key, value); case PREDICATE: return buildPredicate(key, value); case RANK: @@ -266,7 +273,7 @@ public class SelectParser implements Parser { case WEAK_AND: return buildWeakAnd(key, value); default: - throw newUnexpectedArgumentException(key, DOT_PRODUCT, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); } } @@ -403,6 +410,38 @@ public class SelectParser implements Parser { return orItem; } + private Item buildNearestNeighbor(String key, Inspector value) { + + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + String property = children.get(0).asString(); + NearestNeighborItem item = new NearestNeighborItem(field, property); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (TARGET_HITS.equals(annotation_name)){ + item.setTargetNumHits((int)(annotation_value.asDouble())); + } + if (TARGET_NUM_HITS.equals(annotation_name)){ + item.setTargetNumHits((int)(annotation_value.asDouble())); + } + if (HNSW_EXPLORE_ADDITIONAL_HITS.equals(annotation_name)) { + int hnswExploreAdditionalHits = (int)(annotation_value.asDouble()); + item.setHnswExploreAdditionalHits(hnswExploreAdditionalHits); + } + if (APPROXIMATE.equals(annotation_name)) { + boolean allowApproximate = annotation_value.asBool(); + item.setAllowApproximate(allowApproximate); + } + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); addItemsFromInspector(weakAnd, value); @@ -410,6 +449,9 @@ public class SelectParser implements Parser { if (annotations != null){ annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (TARGET_HITS.equals(annotation_name)){ + weakAnd.setN((int)(annotation_value.asDouble())); + } if (TARGET_NUM_HITS.equals(annotation_name)){ weakAnd.setN((int)(annotation_value.asDouble())); } @@ -662,7 +704,10 @@ public class SelectParser implements Parser { HashMap<Integer, Inspector> children = childMap(value); Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); - Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + Integer target_num_hits= getIntegerAnnotation(TARGET_HITS, annotations, null); + if (target_num_hits == null) { + target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + } WandItem out = new WandItem(children.get(0).asString(), target_num_hits); diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java index 76b8c1ef8a2..aca2998cba3 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java @@ -97,7 +97,7 @@ public class ValidateNearestNeighborSearcher extends Searcher { /** Returns an error message if this is invalid, or null if it is valid */ private String validate(NearestNeighborItem item) { if (item.getTargetNumHits() < 1) - return item + " has invalid targetNumHits " + item.getTargetNumHits() + ": Must be >= 1"; + return item + " has invalid targetHits " + item.getTargetNumHits() + ": Must be >= 1"; String queryFeatureName = "query(" + item.getQueryTensorName() + ")"; Optional<Tensor> queryTensor = query.getRanking().getFeatures().getTensor(queryFeatureName); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index f4560806dd2..7d17fe4f09d 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -173,6 +173,7 @@ public class YqlParser implements Parser { static final String STEM = "stem"; static final String SUBSTRING = "substring"; static final String SUFFIX = "suffix"; + static final String TARGET_HITS = "targetHits"; static final String TARGET_NUM_HITS = "targetNumHits"; static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; static final String UNIQUE_ID = "id"; @@ -418,8 +419,12 @@ public class YqlParser implements Parser { String field = fetchFieldRead(args.get(0)); String property = fetchFieldRead(args.get(1)); NearestNeighborItem item = new NearestNeighborItem(field, property); - Integer targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, Integer.class, null, "desired minimum hits to produce"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, + Integer.class, null, "desired minimum hits to produce"); + } if (targetNumHits != null) { item.setTargetNumHits(targetNumHits); } @@ -504,9 +509,13 @@ public class YqlParser implements Parser { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); - WandItem out = new WandItem(getIndex(args.get(0)), getAnnotation(ast, - TARGET_NUM_HITS, Integer.class, DEFAULT_TARGET_NUM_HITS, - "desired number of hits to accumulate in wand")); + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, + Integer.class, null, "desired number of hits to accumulate in wand"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, + Integer.class, DEFAULT_TARGET_NUM_HITS, "desired number of hits to accumulate in wand"); + } + WandItem out = new WandItem(getIndex(args.get(0)), targetNumHits); Double scoreThreshold = getAnnotation(ast, SCORE_THRESHOLD, Double.class, null, "min score for hit inclusion"); if (scoreThreshold != null) { @@ -1028,8 +1037,12 @@ public class YqlParser implements Parser { private CompositeItem buildWeakAnd(OperatorNode<ExpressionOperator> spec) { WeakAndItem weakAnd = new WeakAndItem(); - Integer targetNumHits = getAnnotation(spec, TARGET_NUM_HITS, + Integer targetNumHits = getAnnotation(spec, TARGET_HITS, Integer.class, null, "desired minimum hits to produce"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(spec, TARGET_NUM_HITS, + Integer.class, null, "desired minimum hits to produce"); + } if (targetNumHits != null) { weakAnd.setN(targetNumHits); } |