From 911f614fbab51de142cbea19f2c1aed9160c132d Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 8 Jul 2020 06:18:16 +0000 Subject: minimal YQL support --- container-search/abi-spec.json | 48 ++++----- .../src/main/java/com/yahoo/prelude/Location.java | 6 +- .../com/yahoo/prelude/query/GeoLocationItem.java | 108 +++++++++++++++++++++ .../java/com/yahoo/prelude/query/LocationItem.java | 105 -------------------- .../java/com/yahoo/search/query/SelectParser.java | 24 ++++- .../java/com/yahoo/search/yql/VespaSerializer.java | 20 ++++ .../main/java/com/yahoo/search/yql/YqlParser.java | 20 +++- 7 files changed, 199 insertions(+), 132 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java delete mode 100644 container-search/src/main/java/com/yahoo/prelude/query/LocationItem.java (limited to 'container-search') diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 8728e525f88..98cc7c5165d 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -527,6 +527,30 @@ ], "fields": [] }, + "com.yahoo.prelude.query.GeoLocationItem": { + "superClass": "com.yahoo.prelude.query.TermItem", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void (com.yahoo.prelude.Location)", + "public void (com.yahoo.prelude.Location, java.lang.String)", + "public java.lang.String getRawWord()", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public java.lang.String getName()", + "public java.lang.String stringValue()", + "public void setValue(java.lang.String)", + "public int hashCode()", + "public boolean equals(java.lang.Object)", + "public java.lang.String getIndexedString()", + "protected void encodeThis(java.nio.ByteBuffer)", + "public int getNumWords()", + "public boolean isStemmed()", + "public boolean isWords()" + ], + "fields": [] + }, "com.yahoo.prelude.query.HasIndexItem": { "superClass": "java.lang.Object", "interfaces": [], @@ -805,30 +829,6 @@ "public static final com.yahoo.prelude.query.Limit POSITIVE_INFINITY" ] }, - "com.yahoo.prelude.query.LocationItem": { - "superClass": "com.yahoo.prelude.query.TermItem", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void (com.yahoo.prelude.Location)", - "public void (com.yahoo.prelude.Location, java.lang.String)", - "public java.lang.String getRawWord()", - "public com.yahoo.prelude.query.Item$ItemType getItemType()", - "public java.lang.String getName()", - "public java.lang.String stringValue()", - "public void setValue(java.lang.String)", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public java.lang.String getIndexedString()", - "protected void encodeThis(java.nio.ByteBuffer)", - "public int getNumWords()", - "public boolean isStemmed()", - "public boolean isWords()" - ], - "fields": [] - }, "com.yahoo.prelude.query.MarkerWordItem": { "superClass": "com.yahoo.prelude.query.WordItem", "interfaces": [], diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java index 908bf835e3c..dc98284df22 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Location.java +++ b/container-search/src/main/java/com/yahoo/prelude/Location.java @@ -9,7 +9,7 @@ import java.util.StringTokenizer; /** * Location data for a geographical query. * - * @author Steinar Knutsen + * @author Steinar Knutsen * @author arnej27959 */ public class Location { @@ -247,6 +247,10 @@ public class Location { } } + public Location clone() { + return new Location(toString()); + } + public String toString() { StringBuilder ser = new StringBuilder(); if (attribute != null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java new file mode 100644 index 00000000000..8007bb823f2 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java @@ -0,0 +1,108 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import com.yahoo.prelude.Location; +import java.nio.ByteBuffer; + + +/** + * This represents a geo-location for matching. + * Note that this won't produce summary fields. + * + * @author arnej + */ +public class GeoLocationItem extends TermItem { + + private Location location; + + /** + */ + public GeoLocationItem(Location location) { + super(location.getAttribute(), false); + this.location = location.clone(); + if (! location.hasAttribute()) { + throw new IllegalArgumentException("missing attribute on location: "+location); + } + location.setAttribute(null); + setNormalizable(false); + } + + /** + */ + public GeoLocationItem(Location location, String indexName) { + super(indexName, false); + this.location = location.clone(); + if (location.hasAttribute() && ! location.getAttribute().equals(indexName)) { + throw new IllegalArgumentException("inconsistent attribute on location: "+location+" versus indexName: "+indexName); + } + location.setAttribute(null); + setNormalizable(false); + } + + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public ItemType getItemType() { + return ItemType.LOCATION_TERM; + } + + @Override + public String getName() { + return "LOCATION"; + } + + @Override + public String stringValue() { + location.setAttribute(null); + return location.toString(); + } + + @Override + public void setValue(String value) { + throw new UnsupportedOperationException("Cannot setValue("+value+") on "+getName()); + } + + @Override + public int hashCode() { + return super.hashCode() + 199 * location.hashCode(); + } + + @Override + public boolean equals(Object object) { + if ( ! super.equals(object)) return false; + GeoLocationItem other = (GeoLocationItem) object; // Ensured by superclass + if ( ! location.equals(other.location)) return false; + return true; + } + + @Override + public String getIndexedString() { + location.setAttribute(null); + return location.toString(); + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); // takes care of index bytes + putString(getIndexedString(), buffer); + } + + @Override + public int getNumWords() { + return 1; + } + + @Override + public boolean isStemmed() { + return true; + } + + @Override + public boolean isWords() { + return false; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/LocationItem.java b/container-search/src/main/java/com/yahoo/prelude/query/LocationItem.java deleted file mode 100644 index cf45712088b..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/query/LocationItem.java +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.query; - -import com.yahoo.prelude.Location; -import java.nio.ByteBuffer; - - -/** - * This represents a geo-location for matching. - * Note that this won't produce summary fields. - * - * @author arnej - */ -public class LocationItem extends TermItem { - - private Location location; - - /** - */ - public LocationItem(Location location) { - super(location.getAttribute(), false); - this.location = location; - if (! location.hasAttribute()) { - throw new IllegalArgumentException("missing attribute on location: "+location); - } - setNormalizable(false); - } - - /** - */ - public LocationItem(Location location, String indexName) { - super(indexName, false); - this.location = location; - if (location.hasAttribute() && ! location.getAttribute().equals(indexName)) { - throw new IllegalArgumentException("inconsistent attribute on location: "+location+" versus indexName: "+indexName); - } - this.location.setAttribute(indexName); - setNormalizable(false); - } - - @Override - public String getRawWord() { - return stringValue(); - } - - @Override - public ItemType getItemType() { - return ItemType.LOCATION_TERM; - } - - @Override - public String getName() { - return "LOCATION"; - } - - @Override - public String stringValue() { - return location.toString(); - } - - @Override - public void setValue(String value) { - throw new UnsupportedOperationException("Cannot setValue("+value+") on "+getName()); - } - - @Override - public int hashCode() { - return super.hashCode() + 199 * location.hashCode(); - } - - @Override - public boolean equals(Object object) { - if ( ! super.equals(object)) return false; - LocationItem other = (LocationItem) object; // Ensured by superclass - if ( ! location.equals(other.location)) return false; - return true; - } - - @Override - public String getIndexedString() { - return location.toString(); - } - - @Override - protected void encodeThis(ByteBuffer buffer) { - super.encodeThis(buffer); // takes care of index bytes - putString(getIndexedString(), buffer); - } - - @Override - public int getNumWords() { - return 1; - } - - @Override - public boolean isStemmed() { - return true; - } - - @Override - public boolean isWords() { - return false; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 9910eb9532d..4ae2e4ceddd 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -6,6 +6,7 @@ import com.yahoo.collections.LazyMap; import com.yahoo.language.Language; import com.yahoo.language.process.Normalizer; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.CompositeItem; @@ -15,6 +16,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -99,6 +101,7 @@ public class SelectParser implements Parser { private static final String DOT_PRODUCT = "dotProduct"; private static final String EQUIV = "equiv"; private static final String FILTER = "filter"; + private static final String GEO_LOCATION = "geoLocation"; private static final String HIT_LIMIT = "hitLimit"; private static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; @@ -135,7 +138,7 @@ public class SelectParser implements Parser { private static final String CONTAINS = "contains"; private static final String MATCHES = "matches"; private static final String CALL = "call"; - private static final List FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); + private static final List FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); @@ -264,6 +267,8 @@ public class SelectParser implements Parser { return buildWeightedSet(key, value); case DOT_PRODUCT: return buildDotProduct(key, value); + case GEO_LOCATION: + return buildGeoLocation(key, value); case NEAREST_NEIGHBOR: return buildNearestNeighbor(key, value); case PREDICATE: @@ -410,6 +415,23 @@ public class SelectParser implements Parser { return orItem; } + private Item buildGeoLocation(String key, Inspector value) { + HashMap children = childMap(value); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + String location = children.get(1).asString(); + GeoLocationItem item = new GeoLocationItem(new Location(location), field); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + private Item buildNearestNeighbor(String key, Inspector value) { HashMap children = childMap(value); diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index dd52b9e19b8..77250ecd439 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -16,6 +16,7 @@ import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; import static com.yahoo.search.yql.YqlParser.END_ANCHOR; import static com.yahoo.search.yql.YqlParser.EQUIV; import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; import static com.yahoo.search.yql.YqlParser.LABEL; @@ -72,6 +73,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; @@ -689,6 +691,23 @@ public class VespaSerializer { } + private static class GeoLocationSerializer extends Serializer { + @Override + void onExit(StringBuilder destination, GeoLocationItem item) { } + @Override + boolean serialize(StringBuilder destination, GeoLocationItem item) { + String annotations = leafAnnotations(item); + if (annotations.length() > 0) { + destination.append("([{").append(annotations).append("}]"); + } + destination.append(GEO_LOCATION).append('('); + destination.append(item.getIndexName()).append(", ").append('"'); + escape(item.getIndexedString(), destination); + destination.append('"').append(')'); + return false; + } + } + private static class NearestNeighborSerializer extends Serializer { @Override @@ -1163,6 +1182,7 @@ public class VespaSerializer { dispatchBuilder.put(EquivItem.class, new EquivSerializer()); dispatchBuilder.put(ExactStringItem.class, new WordSerializer()); dispatchBuilder.put(IntItem.class, new NumberSerializer()); + dispatchBuilder.put(GeoLocationItem.class, new GeoLocationSerializer()); dispatchBuilder.put(BoolItem.class, new BoolSerializer()); dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(NearItem.class, new NearSerializer()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 7d17fe4f09d..a0bb4efb70b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -24,6 +24,7 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.BoolItem; @@ -34,6 +35,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -149,6 +151,7 @@ public class YqlParser implements Parser { static final String DOT_PRODUCT = "dotProduct"; static final String EQUIV = "equiv"; static final String FILTER = "filter"; + static final String GEO_LOCATION = "geoLocation"; static final String HIT_LIMIT = "hitLimit"; static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; @@ -372,6 +375,8 @@ public class YqlParser implements Parser { return buildWeightedSet(ast); case DOT_PRODUCT: return buildDotProduct(ast); + case GEO_LOCATION: + return buildGeoLocation(ast); case NEAREST_NEIGHBOR: return buildNearestNeighbor(ast); case PREDICATE: @@ -413,6 +418,19 @@ public class YqlParser implements Parser { return fillWeightedSet(ast, args.get(1), new DotProductItem(getIndex(args.get(0)))); } + private Item buildGeoLocation(OperatorNode ast) { + List> args = ast.getArgument(1); + Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); + String field = fetchFieldRead(args.get(0)); + String location = fetchFieldRead(args.get(1)); + GeoLocationItem item = new GeoLocationItem(new Location(location), field); + String label = getAnnotation(ast, LABEL, String.class, null, "item label"); + if (label != null) { + item.setLabel(label); + } + return item; + } + private Item buildNearestNeighbor(OperatorNode ast) { List> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); @@ -438,7 +456,7 @@ public class YqlParser implements Parser { item.setAllowApproximate(allowApproximate); String label = getAnnotation(ast, LABEL, String.class, null, "item label"); if (label != null) { - item.setLabel(label); + item.setLabel(label); } return item; } -- cgit v1.2.3