diff options
Diffstat (limited to 'container-search')
12 files changed, 434 insertions, 127 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index fcbe23aeb61..5faa5ec322d 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -527,6 +527,31 @@ ], "fields": [] }, + "com.yahoo.prelude.query.GeoLocationItem": { + "superClass": "com.yahoo.prelude.query.TermItem", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.prelude.Location)", + "public void <init>(com.yahoo.prelude.Location, java.lang.String)", + "public com.yahoo.prelude.Location getLocation()", + "public java.lang.String getRawWord()", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public java.lang.String getName()", + "public java.lang.String stringValue()", + "public void setValue(java.lang.String)", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public java.lang.String getIndexedString()", + "protected void encodeThis(java.nio.ByteBuffer)", + "public int getNumWords()", + "public boolean isStemmed()", + "public boolean isWords()" + ], + "fields": [] + }, "com.yahoo.prelude.query.HasIndexItem": { "superClass": "java.lang.Object", "interfaces": [], @@ -697,7 +722,7 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType REGEXP", "public static final enum com.yahoo.prelude.query.Item$ItemType WORD_ALTERNATIVES", "public static final enum com.yahoo.prelude.query.Item$ItemType NEAREST_NEIGHBOR", - "public static final enum com.yahoo.prelude.query.Item$ItemType LOCATION_TERM", + "public static final enum com.yahoo.prelude.query.Item$ItemType GEO_LOCATION_TERM", "public final int code" ] }, diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java index 908bf835e3c..3d3eed3b3df 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Location.java +++ b/container-search/src/main/java/com/yahoo/prelude/Location.java @@ -9,7 +9,7 @@ import java.util.StringTokenizer; /** * Location data for a geographical query. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen * @author arnej27959 */ public class Location { @@ -127,7 +127,7 @@ public class Location { throw new IllegalArgumentException("n/s location must be in range [-90,+90]"); } if (radius_in_degrees < 0) { - pr = 512 * 1024 * 1024; + pr = -1; } x = px; y = py; @@ -142,7 +142,7 @@ public class Location { throw new IllegalArgumentException("can only set geo circle once"); } if (radius_in_units < 0) { - throw new IllegalArgumentException("radius must be positive"); + radius_in_units = -1; } x = px; y = py; @@ -248,6 +248,13 @@ public class Location { } public String toString() { + return render(false); + } + public String backendString() { + return render(true); + } + + private String render(boolean forBackend) { StringBuilder ser = new StringBuilder(); if (attribute != null) { ser.append(attribute).append(':'); @@ -271,7 +278,7 @@ public class Location { if (dimensions == 2) { ser.append(",").append(y); } - ser.append(",").append(r). + ser.append(",").append(forBackend ? backendRadius() : r). append(",").append(tableId). append(",").append(s). append(",").append(replace); @@ -358,11 +365,16 @@ public class Location { /** * Obtain circle radius (in degrees). + * Note that "no radius" or "infinite radius" is represented as -1. * May only be called when isGeoCircle() returns true. **/ public double degRadius() { checkGeoCircle(); - return 0.000001 * r; + return (r < 0) ? -1.0 : (0.000001 * r); + } + + private int backendRadius() { + return (r < 0) ? (512 * 1024 * 1024) : r; } /** @@ -370,7 +382,7 @@ public class Location { * For internal use. */ public int encode(ByteBuffer buffer) { - byte[] loc = Utf8.toBytes(toString()); + byte[] loc = Utf8.toBytes(backendString()); buffer.put(loc); return loc.length; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java new file mode 100644 index 00000000000..8202c8fb279 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java @@ -0,0 +1,119 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.prelude.query; + +import com.google.common.annotations.Beta; +import com.yahoo.prelude.Location; +import java.nio.ByteBuffer; + +/** + * This represents a geo-location in the query tree. + * Used for closeness(fieldname) and distance(fieldname) rank features. + * @author arnej + */ +@Beta +public class GeoLocationItem extends TermItem { + + private Location location; + + /** + * Construct from a Location, which must be geo circle with an attribute set. + **/ + public GeoLocationItem(Location location) { + this(location, location.getAttribute()); + if (! location.hasAttribute()) { + throw new IllegalArgumentException("missing attribute on location: "+location); + } + } + + /** + * Construct from a Location and a field name. + * The Location must be a geo circle. + * If the Location has an attribute set, it must match the field name. + **/ + public GeoLocationItem(Location location, String fieldName) { + super(fieldName, false); + if (location.hasAttribute() && ! location.getAttribute().equals(fieldName)) { + throw new IllegalArgumentException("inconsistent attribute on location: "+location.getAttribute()+" versus fieldName: "+fieldName); + } + if (! location.isGeoCircle()) { + throw new IllegalArgumentException("GeoLocationItem only supports Geo Circles, got: "+location); + } + if (location.hasBoundingBox()) { + throw new IllegalArgumentException("GeoLocationItem does not support bounding box yet, got: "+location); + } + this.location = new Location(location.toString()); + this.location.setAttribute(null); // keep this in (superclass) indexName only + setNormalizable(false); + } + + public Location getLocation() { + return location; + } + + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public ItemType getItemType() { + return ItemType.GEO_LOCATION_TERM; + } + + @Override + public String getName() { + return "GEO_LOCATION"; + } + + @Override + public String stringValue() { + return location.toString(); + } + + @Override + public void setValue(String value) { + throw new UnsupportedOperationException("Cannot setValue("+value+") on "+getName()); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(super.hashCode(), location); + } + + @Override + public boolean equals(Object object) { + if ( ! super.equals(object)) return false; + GeoLocationItem other = (GeoLocationItem) object; // Ensured by superclass + if ( ! location.equals(other.location)) return false; + return true; + } + + @Override + public String getIndexedString() { + return location.toString(); + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); // takes care of index bytes + // TODO: use a better format for encoding the location on the wire. + putString(location.backendString(), buffer); + } + + @Override + public int getNumWords() { + return 1; + } + + @Override + public boolean isStemmed() { + return true; + } + + @Override + public boolean isWords() { + return false; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index bd368864e9a..c4978b2a378 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -61,7 +61,7 @@ public abstract class Item implements Cloneable { REGEXP(24), WORD_ALTERNATIVES(25), NEAREST_NEIGHBOR(26), - LOCATION_TERM(27); + GEO_LOCATION_TERM(27); public final int code; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java index 9b661368972..0e8759f740e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java @@ -137,7 +137,7 @@ public class RpcFillInvoker extends FillInvoker { root.setString("ranking", rankProfile); } if (location != null) { - root.setString("location", location.toString()); + root.setString("location", location.backendString()); } Cursor gids = root.setArray("gids"); for (FastHit hit : hits) { diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 9910eb9532d..0d9acea7643 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -3,9 +3,12 @@ package com.yahoo.search.query; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.process.Normalizer; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.CompositeItem; @@ -15,6 +18,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -47,6 +51,7 @@ import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -61,6 +66,65 @@ import static com.yahoo.slime.Type.LONG; import static com.yahoo.slime.Type.OBJECT; import static com.yahoo.slime.Type.STRING; +import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; +import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; +import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; +import static com.yahoo.search.yql.YqlParser.ANNOTATIONS; +import static com.yahoo.search.yql.YqlParser.APPROXIMATE; +import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.BOUNDS; +import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN; +import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; +import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; +import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; +import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.DISTANCE; +import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; +import static com.yahoo.search.yql.YqlParser.END_ANCHOR; +import static com.yahoo.search.yql.YqlParser.EQUIV; +import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; +import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; +import static com.yahoo.search.yql.YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; +import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; +import static com.yahoo.search.yql.YqlParser.LABEL; +import static com.yahoo.search.yql.YqlParser.NEAR; +import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; +import static com.yahoo.search.yql.YqlParser.NFKC; +import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; +import static com.yahoo.search.yql.YqlParser.ONEAR; +import static com.yahoo.search.yql.YqlParser.ORIGIN; +import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; +import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; +import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; +import static com.yahoo.search.yql.YqlParser.PHRASE; +import static com.yahoo.search.yql.YqlParser.PREDICATE; +import static com.yahoo.search.yql.YqlParser.PREFIX; +import static com.yahoo.search.yql.YqlParser.RANGE; +import static com.yahoo.search.yql.YqlParser.RANK; +import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; +import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; +import static com.yahoo.search.yql.YqlParser.START_ANCHOR; +import static com.yahoo.search.yql.YqlParser.STEM; +import static com.yahoo.search.yql.YqlParser.SUBSTRING; +import static com.yahoo.search.yql.YqlParser.SUFFIX; +import static com.yahoo.search.yql.YqlParser.TARGET_HITS; +import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; +import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; +import static com.yahoo.search.yql.YqlParser.URI; +import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; +import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE; +import static com.yahoo.search.yql.YqlParser.WAND; +import static com.yahoo.search.yql.YqlParser.WEAK_AND; +import static com.yahoo.search.yql.YqlParser.WEIGHT; +import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; + /** * The Select query language. * @@ -70,6 +134,14 @@ import static com.yahoo.slime.Type.STRING; */ public class SelectParser implements Parser { + private static final String AND = "and"; + private static final String AND_NOT = "and_not"; + private static final String CALL = "call"; + private static final String CONTAINS = "contains"; + private static final String EQ = "equals"; + private static final String MATCHES = "matches"; + private static final String OR = "or"; + Parsable query; private final IndexFacts indexFacts; private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap(); @@ -77,65 +149,7 @@ public class SelectParser implements Parser { private final Normalizer normalizer; private IndexFacts.Session indexFactsSession; - // YQL parameters and functions - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; - private static final String ORIGIN_LENGTH = "length"; - private static final String ORIGIN_OFFSET = "offset"; - private static final String ORIGIN = "origin"; - private static final String ORIGIN_ORIGINAL = "original"; - private static final String CONNECTION_ID = "id"; - private static final String CONNECTION_WEIGHT = "weight"; - private static final String CONNECTIVITY = "connectivity"; - private static final String ANNOTATIONS = "annotations"; - private static final String NFKC = "nfkc"; - private static final String USER_INPUT_LANGUAGE = "language"; - private static final String ACCENT_DROP = "accentDrop"; - private static final String ALTERNATIVES = "alternatives"; - private static final String AND_SEGMENTING = "andSegmenting"; - private static final String APPROXIMATE = "approximate"; - private static final String DISTANCE = "distance"; - private static final String DOT_PRODUCT = "dotProduct"; - private static final String EQUIV = "equiv"; - private static final String FILTER = "filter"; - private static final String HIT_LIMIT = "hitLimit"; - private static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; - private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - private static final String LABEL = "label"; - private static final String NEAR = "near"; - private static final String NEAREST_NEIGHBOR = "nearestNeighbor"; - private static final String NORMALIZE_CASE = "normalizeCase"; - private static final String ONEAR = "onear"; - private static final String PHRASE = "phrase"; - private static final String PREDICATE = "predicate"; - private static final String PREFIX = "prefix"; - private static final String RANKED = "ranked"; - private static final String RANK = "rank"; - private static final String SAME_ELEMENT = "sameElement"; - private static final String SCORE_THRESHOLD = "scoreThreshold"; - private static final String SIGNIFICANCE = "significance"; - private static final String STEM = "stem"; - private static final String SUBSTRING = "substring"; - private static final String SUFFIX = "suffix"; - private static final String TARGET_HITS = "targetHits"; - private static final String TARGET_NUM_HITS = "targetNumHits"; - private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - private static final String UNIQUE_ID = "id"; - private static final String USE_POSITION_DATA = "usePositionData"; - private static final String WAND = "wand"; - private static final String WEAK_AND = "weakAnd"; - private static final String WEIGHTED_SET = "weightedSet"; - private static final String WEIGHT = "weight"; - private static final String AND = "and"; - private static final String AND_NOT = "and_not"; - private static final String OR = "or"; - private static final String EQ = "equals"; - private static final String RANGE = "range"; - private static final String CONTAINS = "contains"; - private static final String MATCHES = "matches"; - private static final String CALL = "call"; - private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); + private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); @@ -153,7 +167,7 @@ public class SelectParser implements Parser { } private QueryTree buildTree() { - Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get(); + Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get(); if (inspector.field("error_message").valid()) { throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); @@ -213,7 +227,7 @@ public class SelectParser implements Parser { /** Translates a list of grouping requests on JSON form to a list in the grouping language form */ private List<String> toGroupingRequests(String groupingJson) { - Inspector inspector = SlimeUtils.jsonToSlime(groupingJson.getBytes()).get(); + Inspector inspector = SlimeUtils.jsonToSlime(groupingJson).get(); if (inspector.field("error_message").valid()) { throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); @@ -264,6 +278,8 @@ public class SelectParser implements Parser { return buildWeightedSet(key, value); case DOT_PRODUCT: return buildDotProduct(key, value); + case GEO_LOCATION: + return buildGeoLocation(key, value); case NEAREST_NEIGHBOR: return buildNearestNeighbor(key, value); case PREDICATE: @@ -410,6 +426,47 @@ public class SelectParser implements Parser { return orItem; } + private Item buildGeoLocation(String key, Inspector value) { + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 4, "Expected 4 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + var arg1 = children.get(1); + var arg2 = children.get(2); + var arg3 = children.get(3); + var loc = new Location(); + if (arg3.type() != Type.STRING) { + throw new IllegalArgumentException("Invalid geoLocation radius type "+arg3.type()+" for "+arg3); + } + double radius = DistanceParser.parse(arg3.asString()); + if (arg1.type() == Type.STRING && arg2.type() == Type.STRING) { + var c1input = children.get(1).asString(); + var c2input = children.get(2).asString(); + var coord_1 = ParsedDegree.fromString(c1input, true, false); + var coord_2 = ParsedDegree.fromString(c2input, false, true); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+c1input+"' and '"+c2input+"'"); + } + } else if (arg1.type() == Type.DOUBLE && arg2.type() == Type.DOUBLE) { + loc.setGeoCircle(arg1.asDouble(), arg2.asDouble(), radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinate types "+arg1.type()+" and "+arg2.type()); + } + var item = new GeoLocationItem(loc, field); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + private Item buildNearestNeighbor(String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index dd52b9e19b8..22328fb026e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -16,6 +16,7 @@ import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; import static com.yahoo.search.yql.YqlParser.END_ANCHOR; import static com.yahoo.search.yql.YqlParser.EQUIV; import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; import static com.yahoo.search.yql.YqlParser.LABEL; @@ -72,6 +73,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; @@ -689,6 +691,26 @@ public class VespaSerializer { } + private static class GeoLocationSerializer extends Serializer<GeoLocationItem> { + @Override + void onExit(StringBuilder destination, GeoLocationItem item) { } + @Override + boolean serialize(StringBuilder destination, GeoLocationItem item) { + String annotations = leafAnnotations(item); + if (annotations.length() > 0) { + destination.append("([{").append(annotations).append("}]"); + } + destination.append(GEO_LOCATION).append('('); + destination.append(item.getIndexName()).append(", "); + var loc = item.getLocation(); + destination.append(loc.degNS()).append(", "); + destination.append(loc.degEW()).append(", "); + destination.append('"').append(loc.degRadius()).append(" deg").append('"'); + destination.append(')'); + return false; + } + } + private static class NearestNeighborSerializer extends Serializer<NearestNeighborItem> { @Override @@ -1163,6 +1185,7 @@ public class VespaSerializer { dispatchBuilder.put(EquivItem.class, new EquivSerializer()); dispatchBuilder.put(ExactStringItem.class, new WordSerializer()); dispatchBuilder.put(IntItem.class, new NumberSerializer()); + dispatchBuilder.put(GeoLocationItem.class, new GeoLocationSerializer()); dispatchBuilder.put(BoolItem.class, new BoolSerializer()); dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(NearItem.class, new NearSerializer()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 7d17fe4f09d..6a464a1503b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -19,11 +19,14 @@ import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; import com.yahoo.collections.LazySet; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.BoolItem; @@ -34,6 +37,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -94,8 +98,8 @@ import com.yahoo.search.query.parser.ParserFactory; */ public class YqlParser implements Parser { - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; + public static final String DESCENDING_HITS_ORDER = "descending"; + public static final String ASCENDING_HITS_ORDER = "ascending"; private enum SegmentWhen { NEVER, POSSIBLY, ALWAYS; @@ -107,12 +111,12 @@ public class YqlParser implements Parser { private static final Integer DEFAULT_HITS = 10; private static final Integer DEFAULT_OFFSET = 0; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; + public static final Integer DEFAULT_TARGET_NUM_HITS = 10; private static final String ACCENT_DROP_DESCRIPTION = "setting for whether to remove accents if field implies it"; - private static final String ANNOTATIONS = "annotations"; + public static final String ANNOTATIONS = "annotations"; private static final String FILTER_DESCRIPTION = "term filter setting"; private static final String IMPLICIT_TRANSFORMS_DESCRIPTION = "setting for whether built-in query transformers should touch the term"; - private static final String NFKC = "nfkc"; + public static final String NFKC = "nfkc"; private static final String NORMALIZE_CASE_DESCRIPTION = "setting for whether to do case normalization if field implies it"; private static final String ORIGIN_DESCRIPTION = "string origin for a term"; private static final String RANKED_DESCRIPTION = "setting for whether to use term for ranking"; @@ -121,7 +125,7 @@ public class YqlParser implements Parser { private static final String USER_INPUT_ALLOW_EMPTY = "allowEmpty"; private static final String USER_INPUT_DEFAULT_INDEX = "defaultIndex"; private static final String USER_INPUT_GRAMMAR = "grammar"; - private static final String USER_INPUT_LANGUAGE = "language"; + public static final String USER_INPUT_LANGUAGE = "language"; private static final String USER_INPUT_RAW = "raw"; private static final String USER_INPUT_SEGMENT = "segment"; private static final String USER_INPUT = "userInput"; @@ -134,55 +138,56 @@ public class YqlParser implements Parser { public static final String SORTING_LOCALE = "locale"; public static final String SORTING_STRENGTH = "strength"; - static final String ACCENT_DROP = "accentDrop"; - static final String ALTERNATIVES = "alternatives"; - static final String AND_SEGMENTING = "andSegmenting"; - static final String APPROXIMATE = "approximate"; - static final String BOUNDS = "bounds"; - static final String BOUNDS_LEFT_OPEN = "leftOpen"; - static final String BOUNDS_OPEN = "open"; - static final String BOUNDS_RIGHT_OPEN = "rightOpen"; - static final String CONNECTION_ID = "id"; - static final String CONNECTION_WEIGHT = "weight"; - static final String CONNECTIVITY = "connectivity"; - static final String DISTANCE = "distance"; - static final String DOT_PRODUCT = "dotProduct"; - static final String EQUIV = "equiv"; - static final String FILTER = "filter"; - static final String HIT_LIMIT = "hitLimit"; - static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; - static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - static final String LABEL = "label"; - static final String NEAR = "near"; - static final String NEAREST_NEIGHBOR = "nearestNeighbor"; - static final String NORMALIZE_CASE = "normalizeCase"; - static final String ONEAR = "onear"; - static final String ORIGIN_LENGTH = "length"; - static final String ORIGIN_OFFSET = "offset"; - static final String ORIGIN = "origin"; - static final String ORIGIN_ORIGINAL = "original"; - static final String PHRASE = "phrase"; - static final String PREDICATE = "predicate"; - static final String PREFIX = "prefix"; - static final String RANGE = "range"; - static final String RANKED = "ranked"; - static final String RANK = "rank"; - static final String SAME_ELEMENT = "sameElement"; - static final String SCORE_THRESHOLD = "scoreThreshold"; - static final String SIGNIFICANCE = "significance"; - static final String STEM = "stem"; - static final String SUBSTRING = "substring"; - static final String SUFFIX = "suffix"; - static final String TARGET_HITS = "targetHits"; - static final String TARGET_NUM_HITS = "targetNumHits"; - static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - static final String UNIQUE_ID = "id"; - static final String USE_POSITION_DATA = "usePositionData"; - static final String WAND = "wand"; - static final String WEAK_AND = "weakAnd"; - static final String WEIGHTED_SET = "weightedSet"; - static final String WEIGHT = "weight"; - static final String URI = "uri"; + public static final String ACCENT_DROP = "accentDrop"; + public static final String ALTERNATIVES = "alternatives"; + public static final String AND_SEGMENTING = "andSegmenting"; + public static final String APPROXIMATE = "approximate"; + public static final String BOUNDS = "bounds"; + public static final String BOUNDS_LEFT_OPEN = "leftOpen"; + public static final String BOUNDS_OPEN = "open"; + public static final String BOUNDS_RIGHT_OPEN = "rightOpen"; + public static final String CONNECTION_ID = "id"; + public static final String CONNECTION_WEIGHT = "weight"; + public static final String CONNECTIVITY = "connectivity"; + public static final String DISTANCE = "distance"; + public static final String DOT_PRODUCT = "dotProduct"; + public static final String EQUIV = "equiv"; + public static final String FILTER = "filter"; + public static final String GEO_LOCATION = "geoLocation"; + public static final String HIT_LIMIT = "hitLimit"; + public static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; + public static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; + public static final String LABEL = "label"; + public static final String NEAR = "near"; + public static final String NEAREST_NEIGHBOR = "nearestNeighbor"; + public static final String NORMALIZE_CASE = "normalizeCase"; + public static final String ONEAR = "onear"; + public static final String ORIGIN_LENGTH = "length"; + public static final String ORIGIN_OFFSET = "offset"; + public static final String ORIGIN = "origin"; + public static final String ORIGIN_ORIGINAL = "original"; + public static final String PHRASE = "phrase"; + public static final String PREDICATE = "predicate"; + public static final String PREFIX = "prefix"; + public static final String RANGE = "range"; + public static final String RANKED = "ranked"; + public static final String RANK = "rank"; + public static final String SAME_ELEMENT = "sameElement"; + public static final String SCORE_THRESHOLD = "scoreThreshold"; + public static final String SIGNIFICANCE = "significance"; + public static final String STEM = "stem"; + public static final String SUBSTRING = "substring"; + public static final String SUFFIX = "suffix"; + public static final String TARGET_HITS = "targetHits"; + public static final String TARGET_NUM_HITS = "targetNumHits"; + public static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; + public static final String UNIQUE_ID = "id"; + public static final String USE_POSITION_DATA = "usePositionData"; + public static final String WAND = "wand"; + public static final String WEAK_AND = "weakAnd"; + public static final String WEIGHTED_SET = "weightedSet"; + public static final String WEIGHT = "weight"; + public static final String URI = "uri"; private final IndexFacts indexFacts; private final List<ConnectedItem> connectedItems = new ArrayList<>(); @@ -372,6 +377,8 @@ public class YqlParser implements Parser { return buildWeightedSet(ast); case DOT_PRODUCT: return buildDotProduct(ast); + case GEO_LOCATION: + return buildGeoLocation(ast); case NEAREST_NEIGHBOR: return buildNearestNeighbor(ast); case PREDICATE: @@ -413,6 +420,29 @@ public class YqlParser implements Parser { return fillWeightedSet(ast, args.get(1), new DotProductItem(getIndex(args.get(0)))); } + private Item buildGeoLocation(OperatorNode<ExpressionOperator> ast) { + List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); + Preconditions.checkArgument(args.size() == 4, "Expected 4 arguments, got %s.", args.size()); + String field = fetchFieldRead(args.get(0)); + var coord_1 = ParsedDegree.fromString(fetchFieldRead(args.get(1)), true, false); + var coord_2 = ParsedDegree.fromString(fetchFieldRead(args.get(2)), false, true); + double radius = DistanceParser.parse(fetchFieldRead(args.get(3))); + var loc = new Location(); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+coord_1+"' and '"+coord_2+"'"); + } + var item = new GeoLocationItem(loc, field); + String label = getAnnotation(ast, LABEL, String.class, null, "item label"); + if (label != null) { + item.setLabel(label); + } + return item; + } + private Item buildNearestNeighbor(OperatorNode<ExpressionOperator> ast) { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); @@ -438,7 +468,7 @@ public class YqlParser implements Parser { item.setAllowApproximate(allowApproximate); String label = getAnnotation(ast, LABEL, String.class, null, "item label"); if (label != null) { - item.setLabel(label); + item.setLabel(label); } return item; } @@ -902,6 +932,8 @@ public class YqlParser implements Parser { private static String fetchFieldRead(OperatorNode<ExpressionOperator> ast) { switch (ast.getOperator()) { + case LITERAL: + return ast.getArgument(0).toString(); case READ_FIELD: return ast.getArgument(1); case PROPREF: diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java index aa3fa53119e..aa48e8494f2 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java @@ -123,7 +123,8 @@ public class PosSearcherTestCase { q.properties().set("pos.ll", "N0;E0"); q.properties().set("pos.radius", "-1"); doSearch(searcher, q, 0, 10); - assertEquals("(2,0,0,536870912,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + assertEquals("(2,0,0,-1,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + assertEquals("(2,0,0,536870912,0,1,0,4294967295)", q.getRanking().getLocation().backendString()); } /** diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index d770b08d31a..f8e930fa19d 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -119,6 +119,14 @@ public class VespaSerializerTestCase { } @Test + public void testGeoLocation() { + parseAndConfirm("geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\")"); + parseAndConfirm("geoLocation(headquarters, 37.41638, -122.024683, \"180.0 deg\")"); + parseAndConfirm("geoLocation(home, -17.0, 42.0, \"0.0 deg\")"); + parseAndConfirm("geoLocation(workplace, -12.0, -34.0, \"-1.0 deg\")"); + } + + @Test public void testNear() { parseAndConfirm("title contains near(\"a\", \"b\")"); parseAndConfirm("title contains ([{\"distance\": 50}]near(\"a\", \"b\"))"); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index a151244525a..62a9e27cd96 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -548,6 +548,24 @@ public class YqlParserTestCase { } @Test + public void testGeoLocation() { + assertParse("select foo from bar where geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\");", + "GEO_LOCATION workplace:(2,10433033,63418417,500000,0,1,0,1921876103)"); + assertParse("select foo from bar where geoLocation(headquarters, \"37.416383\", \"-122.024683\", \"100 miles\");", + "GEO_LOCATION headquarters:(2,-122024683,37416383,1450561,0,1,0,3411238761)"); + assertParse("select foo from bar where geoLocation(home, \"E10.433033\", \"N63.418417\", \"5km\");", + "GEO_LOCATION home:(2,10433033,63418417,45066,0,1,0,1921876103)"); + + assertParseFail("select foo from bar where geoLocation(qux, 1, 2);", + new IllegalArgumentException("Expected 4 arguments, got 3.")); + assertParseFail("select foo from bar where geoLocation(qux, 2.0, \"N5.0\", \"0.5 deg\");", + new IllegalArgumentException( + "Invalid geoLocation coordinates 'Latitude: 2.0 degrees' and 'Latitude: 5.0 degrees'")); + assertParse("select foo from bar where geoLocation(workplace, -12, -34, \"-77 d\");", + "GEO_LOCATION workplace:(2,-34000000,-12000000,-1,0,1,0,4201111954)"); + } + + @Test public void testNearestNeighbor() { assertParse("select foo from bar where nearestNeighbor(semantic_embedding, my_vector);", "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=0}"); diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 4691ef42e55..f297fd69f24 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -522,6 +522,18 @@ public class SelectTestCase { } @Test + public void testGeoLocation() { + assertParse("{ \"geoLocation\": [ \"workplace\", 63.418417, 10.433033, \"0.5 deg\" ] }", + "GEO_LOCATION workplace:(2,10433033,63418417,500000,0,1,0,1921876103)"); + assertParse("{ \"geoLocation\": [ \"headquarters\", \"37.416383\", \"-122.024683\", \"100 miles\" ] }", + "GEO_LOCATION headquarters:(2,-122024683,37416383,1450561,0,1,0,3411238761)"); + assertParse("{ \"geoLocation\": [ \"home\", \"E10.433033\", \"N63.418417\", \"5km\" ] }", + "GEO_LOCATION home:(2,10433033,63418417,45066,0,1,0,1921876103)"); + assertParse("{ \"geoLocation\": [ \"workplace\", -12.0, -34.0, \"-77 deg\" ] }", + "GEO_LOCATION workplace:(2,-34000000,-12000000,-1,0,1,0,4201111954)"); + } + + @Test public void testNearestNeighbor() { assertParse("{ \"nearestNeighbor\": [ \"f1field\", \"q2prop\" ] }", "NEAREST_NEIGHBOR {field=f1field,queryTensorName=q2prop,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=0}"); |