diff options
Diffstat (limited to 'container-search/src')
186 files changed, 3690 insertions, 2359 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 65d5879b004..0dfbf6470ad 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude; - import com.yahoo.language.process.StemMode; import java.util.ArrayList; @@ -10,7 +9,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; - /** * Information about configured settings of a field or field collection (an actual index or not) in a search definition. * There are two types of settings: @@ -26,6 +24,7 @@ import java.util.Set; public class Index { public static class Attribute { + private boolean tokenizedContent = false; public final String name; @@ -64,6 +63,7 @@ public class Index { private boolean normalize = false; private boolean literalBoost = false; private boolean numerical = false; + private boolean predicate = false; private long predicateUpperBound = Long.MAX_VALUE; private long predicateLowerBound = Long.MIN_VALUE; @@ -73,8 +73,8 @@ public class Index { private boolean isNGram = false; private int gramSize = 2; - /** Whether implicit phrases should lead to a phrase item or an and item */ - private boolean phraseSegmenting = true; + /** Whether implicit phrases should lead to a phrase item or an and item. */ + private Boolean phraseSegmenting = false; /** The string terminating an exact token in this index, or null to use the default (space) */ private String exactTerminator = null; @@ -182,6 +182,8 @@ public class Index { setLiteralBoost(true); } else if (commandString.equals("numerical")) { setNumerical(true); + } else if (commandString.equals("predicate")) { + setPredicate(true); } else if (commandString.startsWith("predicate-bounds ")) { setPredicateBounds(commandString.substring(17)); } else if (commandString.equals("phrase-segmenting")) { @@ -207,20 +209,12 @@ public class Index { } } - /** - * Whether terms in this field are lower cased when indexing. - * - * @param lowercase true if terms are lowercased - */ + /** Sets whether terms in this field are lowercased when indexing. */ public void setLowercase(boolean lowercase) { this.lowercase = lowercase; } - /** - * Whether terms in this field are lower cased when indexing. - * - * @return true if terms are lowercased - */ + /** Returns whether terms in this field are lowercased when indexing. */ public boolean isLowercase() { return lowercase; } @@ -313,6 +307,10 @@ public class Index { public boolean isNumerical() { return numerical; } + public void setPredicate(boolean isPredicate) { this.predicate = isPredicate; } + + public boolean isPredicate() { return predicate; } + public long getPredicateUpperBound() { return predicateUpperBound; } public long getPredicateLowerBound() { return predicateLowerBound; } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 76eef33d6c0..aa3d6a2c0f8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -48,7 +48,6 @@ public class IndexFacts { static final String unionName = "unionOfAllKnown"; /** A search definition which contains the union of all settings. */ - @SuppressWarnings("deprecation") private SearchDefinition unionSearchDefinition = new SearchDefinition(unionName); private boolean frozen; diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index 062a514056b..00935392683 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -109,7 +109,6 @@ public final class IndexModel { return searchDefinitions; } - @SuppressWarnings("deprecation") private SearchDefinition unionOf(Collection<SearchDefinition> searchDefinitions) { SearchDefinition union = new SearchDefinition(IndexFacts.unionName); diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java index 37284bd6bcc..3d3eed3b3df 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Location.java +++ b/container-search/src/main/java/com/yahoo/prelude/Location.java @@ -9,7 +9,7 @@ import java.util.StringTokenizer; /** * Location data for a geographical query. * - * @author <a href="mailto:[email protected]">Steinar Knutsen</a> + * @author Steinar Knutsen * @author arnej27959 */ public class Location { @@ -126,8 +126,8 @@ public class Location { if (ns < -90.1 || ns > +90.1) { throw new IllegalArgumentException("n/s location must be in range [-90,+90]"); } - if (radius_in_degrees < 0 || radius_in_degrees > 180.0) { - throw new IllegalArgumentException("radius must be in range [0,180] degrees, approximately upto 20000km"); + if (radius_in_degrees < 0) { + pr = -1; } x = px; y = py; @@ -142,7 +142,7 @@ public class Location { throw new IllegalArgumentException("can only set geo circle once"); } if (radius_in_units < 0) { - throw new IllegalArgumentException("radius must be positive"); + radius_in_units = -1; } x = px; y = py; @@ -248,6 +248,13 @@ public class Location { } public String toString() { + return render(false); + } + public String backendString() { + return render(true); + } + + private String render(boolean forBackend) { StringBuilder ser = new StringBuilder(); if (attribute != null) { ser.append(attribute).append(':'); @@ -271,7 +278,7 @@ public class Location { if (dimensions == 2) { ser.append(",").append(y); } - ser.append(",").append(r). + ser.append(",").append(forBackend ? backendRadius() : r). append(",").append(tableId). append(",").append(s). append(",").append(replace); @@ -358,11 +365,16 @@ public class Location { /** * Obtain circle radius (in degrees). + * Note that "no radius" or "infinite radius" is represented as -1. * May only be called when isGeoCircle() returns true. **/ public double degRadius() { checkGeoCircle(); - return 0.000001 * r; + return (r < 0) ? -1.0 : (0.000001 * r); + } + + private int backendRadius() { + return (r < 0) ? (512 * 1024 * 1024) : r; } /** @@ -370,7 +382,7 @@ public class Location { * For internal use. */ public int encode(ByteBuffer buffer) { - byte[] loc = Utf8.toBytes(toString()); + byte[] loc = Utf8.toBytes(backendString()); buffer.put(loc); return loc.length; } diff --git a/container-search/src/main/java/com/yahoo/prelude/Ping.java b/container-search/src/main/java/com/yahoo/prelude/Ping.java index dd14e150d95..1d5d4c92827 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Ping.java +++ b/container-search/src/main/java/com/yahoo/prelude/Ping.java @@ -4,7 +4,7 @@ package com.yahoo.prelude; /** * A ping, typically to ask whether backend is alive. * - * @author <a href="mailto:[email protected]">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class Ping { diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index a60fba9a4f7..1e5513f1274 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -4,82 +4,102 @@ package com.yahoo.prelude; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.statistics.ElapsedTime; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; /** * An answer from Ping. * - * @author Steinar Knutsen + * @author bratseth */ public class Pong { - private String pingInfo=""; - private final List<ErrorMessage> errors = new ArrayList<>(1); - private ElapsedTime elapsed = new ElapsedTime(); + private final ElapsedTime elapsed = new ElapsedTime(); private final Optional<Long> activeDocuments; + private final boolean isBlockingWrites; + private final Optional<ErrorMessage> error; public Pong() { - this.activeDocuments = Optional.empty(); + this(Optional.empty(), false, Optional.empty()); } public Pong(ErrorMessage error) { - errors.add(error); - this.activeDocuments = Optional.empty(); + this(Optional.empty(), false, Optional.of(error)); } public Pong(long activeDocuments) { - this.activeDocuments = Optional.of(activeDocuments); + this(Optional.of(activeDocuments), false, Optional.empty()); } - public void addError(ErrorMessage error) { - errors.add(error); + public Pong(long activeDocuments, boolean isBlockingWrites) { + this(Optional.of(activeDocuments), isBlockingWrites, Optional.empty()); } - public ErrorMessage getError(int i) { - return errors.get(i); + private Pong(Optional<Long> activeDocuments, boolean isBlockingWrites, Optional<ErrorMessage> error) { + this.activeDocuments = activeDocuments; + this.isBlockingWrites = isBlockingWrites; + this.error = error; } - /** Returns the number of active documents in the backend responding in this Pong, if available */ - public Optional<Long> activeDocuments() { - return activeDocuments; + /** + * @deprecated do not use. Additional errors are ignored. + */ + @Deprecated + public void addError(ErrorMessage error) { } + + /** + * @deprecated use error() instead + */ + @Deprecated + public ErrorMessage getError(int i) { + if (i > 1) throw new IllegalArgumentException("No error at position " + i); + if (i == 0 && error.isEmpty()) throw new IllegalArgumentException("No error at position " + i); + return error.get(); } - /** Returns the number of nodes which responded to this Pong, if available */ + public Optional<ErrorMessage> error() { return error; } + + /** Returns the number of active documents in the backend responding in this Pong, if available */ + public Optional<Long> activeDocuments() { return activeDocuments; } + + /** Returns true if the pinged node is currently blocking write operations due to being full */ + public boolean isBlockingWrites() { return isBlockingWrites; } + + /** + * Returns Optional.empty() + * + * @return empty + * @deprecated do not use. There is always one pong per node. + */ + @Deprecated public Optional<Integer> activeNodes() { return Optional.empty(); } + /** + * Returns a list containing 0 or 1 errors + * + * @deprecated use error() instead + */ + @Deprecated public List<ErrorMessage> getErrors() { - return Collections.unmodifiableList(errors); + return error.stream().collect(Collectors.toList()); } /** Returns whether there is an error or not */ - public boolean badResponse() { - return ! errors.isEmpty(); - } + public boolean badResponse() { return error.isPresent(); } - public ElapsedTime getElapsedTime() { - return elapsed; - } + public ElapsedTime getElapsedTime() { return elapsed; } /** Returns a string which included the ping info (if any) and any errors added to this */ @Override public String toString() { - StringBuilder m = new StringBuilder("Result of pinging"); - if (pingInfo.length() > 0) { - m.append(" using "); - m.append(pingInfo); - } - if (errors.size() > 0) - m.append(" "); - for (int i = 0; i < errors.size(); i++) { - m.append(errors.get(i).toString()); - if ( i <errors.size()-1) - m.append(", "); - } + StringBuilder m = new StringBuilder("Ping result"); + activeDocuments.ifPresent(docCount -> m.append(" active docs: ").append(docCount)); + if (isBlockingWrites) + m.append(" blocking writes: true"); + error.ifPresent(e -> m.append(" error: ").append(error)); return m.toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 329a9caaf91..ef892585d21 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -2,7 +2,7 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -58,7 +58,7 @@ public abstract class DocsumField { fieldFactory.put("xmlstring", XMLField.class); fieldFactory.put("tensor", TensorField.class); } catch (Exception e) { - log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e); + log.log(Level.SEVERE, "Could not initialize docsum decoding properly.", e); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 338add37213..56dfc700ca7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -190,7 +190,7 @@ public class FastHit extends Hit { /** * Returns values for the features listed in - * <a href="https://docs.vespa.ai/documentation/reference/search-definitions-reference.html#summary-features">summary-features</a> + * <a href="https://docs.vespa.ai/documentation/reference/schema-reference.html#summary-features">summary-features</a> * in the rank profile specified in the query producing this. */ public FeatureData features() { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index c04553ae2f5..14604d61c0a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -85,7 +85,7 @@ public class FastSearcher extends VespaBackEndSearcher { public Result doSearch2(Query query, Execution execution) { if (dispatcher.searchCluster().groupSize() == 1) forceSinglePassGrouping(query); - try(SearchInvoker invoker = getSearchInvoker(query)) { + try (SearchInvoker invoker = getSearchInvoker(query)) { Result result = invoker.search(query, execution); injectSource(result.hits()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java index bac227ac3e3..55e16804602 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java @@ -31,14 +31,17 @@ public class AndSegmentItem extends SegmentItem implements BlockItem { } } + @Override public ItemType getItemType() { return ItemType.AND; } + @Override public String getName() { return "SAND"; } + @Override public String getIndexName() { if (getItemCount() == 0) { return ""; @@ -54,4 +57,5 @@ public class AndSegmentItem extends SegmentItem implements BlockItem { i.next().setWeight(w); } } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java index 13673144a0a..d0ffcd2d0e0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java @@ -3,10 +3,9 @@ package com.yahoo.prelude.query; /** - * An interface used for anything which represents a single block - * of query input. + * An interface used for anything which represents a single block of query input. * - * @author <a href="mailto:[email protected]">Steinar Knutsen</a> + * @author Steinar Knutsen */ public interface BlockItem extends HasIndexItem { @@ -39,4 +38,5 @@ public interface BlockItem extends HasIndexItem { * is necessary to change operator? */ SegmentingRule getSegmentingRule(); + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java new file mode 100644 index 00000000000..8202c8fb279 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/GeoLocationItem.java @@ -0,0 +1,119 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.prelude.query; + +import com.google.common.annotations.Beta; +import com.yahoo.prelude.Location; +import java.nio.ByteBuffer; + +/** + * This represents a geo-location in the query tree. + * Used for closeness(fieldname) and distance(fieldname) rank features. + * @author arnej + */ +@Beta +public class GeoLocationItem extends TermItem { + + private Location location; + + /** + * Construct from a Location, which must be geo circle with an attribute set. + **/ + public GeoLocationItem(Location location) { + this(location, location.getAttribute()); + if (! location.hasAttribute()) { + throw new IllegalArgumentException("missing attribute on location: "+location); + } + } + + /** + * Construct from a Location and a field name. + * The Location must be a geo circle. + * If the Location has an attribute set, it must match the field name. + **/ + public GeoLocationItem(Location location, String fieldName) { + super(fieldName, false); + if (location.hasAttribute() && ! location.getAttribute().equals(fieldName)) { + throw new IllegalArgumentException("inconsistent attribute on location: "+location.getAttribute()+" versus fieldName: "+fieldName); + } + if (! location.isGeoCircle()) { + throw new IllegalArgumentException("GeoLocationItem only supports Geo Circles, got: "+location); + } + if (location.hasBoundingBox()) { + throw new IllegalArgumentException("GeoLocationItem does not support bounding box yet, got: "+location); + } + this.location = new Location(location.toString()); + this.location.setAttribute(null); // keep this in (superclass) indexName only + setNormalizable(false); + } + + public Location getLocation() { + return location; + } + + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public ItemType getItemType() { + return ItemType.GEO_LOCATION_TERM; + } + + @Override + public String getName() { + return "GEO_LOCATION"; + } + + @Override + public String stringValue() { + return location.toString(); + } + + @Override + public void setValue(String value) { + throw new UnsupportedOperationException("Cannot setValue("+value+") on "+getName()); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(super.hashCode(), location); + } + + @Override + public boolean equals(Object object) { + if ( ! super.equals(object)) return false; + GeoLocationItem other = (GeoLocationItem) object; // Ensured by superclass + if ( ! location.equals(other.location)) return false; + return true; + } + + @Override + public String getIndexedString() { + return location.toString(); + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); // takes care of index bytes + // TODO: use a better format for encoding the location on the wire. + putString(location.backendString(), buffer); + } + + @Override + public int getNumWords() { + return 1; + } + + @Override + public boolean isStemmed() { + return true; + } + + @Override + public boolean isWords() { + return false; + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java index a06009e642a..300d40d4366 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java @@ -79,4 +79,5 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements super.disclose(discloser); discloser.addProperty("index", index); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index ea65bc7d7d2..c4978b2a378 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -32,7 +32,7 @@ public abstract class Item implements Cloneable { /** * The definitions in Item.ItemType must match the ones in - * searchlib/src/searchlib/parsequery/parse.h + * searchlib/src/vespa/searchlib/parsequery/parse.h */ public static enum ItemType { OR(0), @@ -42,7 +42,7 @@ public abstract class Item implements Cloneable { WORD(4), INT(5), PHRASE(6), - PAREN(7), + PAREN(7), // TODO not used - remove on Vespa 8 PREFIX(8), SUBSTRING(9), NEAR(11), @@ -60,7 +60,8 @@ public abstract class Item implements Cloneable { PREDICATE_QUERY(23), REGEXP(24), WORD_ALTERNATIVES(25), - NEAREST_NEIGHBOR(26); + NEAREST_NEIGHBOR(26), + GEO_LOCATION_TERM(27); public final int code; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java index 153606e6d99..56554e14d01 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearItem.java @@ -8,7 +8,7 @@ import java.nio.ByteBuffer; /** - * <p>A set of terms which must be near each other to match.</p> + * A set of terms which must be near each other to match. * * @author bratseth * @author havardpe @@ -18,7 +18,7 @@ public class NearItem extends CompositeItem { protected int distance; /** The default distance used if none is specified: 2 */ - public static final int defaultDistance=2; + public static final int defaultDistance = 2; /** Creates a NEAR item with distance 2 */ public NearItem() { @@ -26,20 +26,17 @@ public class NearItem extends CompositeItem { } /** - * Creates a <i>near</i> item with a limit to the distance - * between the words. + * Creates a <i>near</i> item with a limit to the distance between the words. * - * @param distance the number of word position which may separate - * the words for this near item to match + * @param distance the maximum position difference between the words which should be counted as a match */ public NearItem(int distance) { setDistance(distance); } public void setDistance(int distance) { - if (distance < 0) { - throw new IllegalArgumentException("Can not use negative distance '" + distance + "'."); - } + if (distance < 0) + throw new IllegalArgumentException("Can not use negative distance " + distance); this.distance = distance; } @@ -47,14 +44,17 @@ public class NearItem extends CompositeItem { return distance; } + @Override public ItemType getItemType() { return ItemType.NEAR; } + @Override public String getName() { return "NEAR"; } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); IntegerCompressor.putCompressedPositiveNumber(distance, buffer); @@ -67,6 +67,7 @@ public class NearItem extends CompositeItem { } /** Appends the heading of this string - <code>[getName()]([limit]) </code> */ + @Override protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); @@ -75,6 +76,7 @@ public class NearItem extends CompositeItem { buffer.append(" "); } + @Override public int hashCode() { return super.hashCode() + 23* distance; } @@ -83,10 +85,11 @@ public class NearItem extends CompositeItem { * Returns whether this item is of the same class and * contains the same state as the given item */ + @Override public boolean equals(Object object) { if (!super.equals(object)) return false; NearItem other = (NearItem) object; // Ensured by superclass - if (this.distance !=other.distance) return false; + if (this.distance != other.distance) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java index 35b87ec0190..be3ae913476 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.query; import com.google.common.annotations.Beta; import com.yahoo.compress.IntegerCompressor; +import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; @@ -20,6 +21,8 @@ import java.nio.ByteBuffer; public class NearestNeighborItem extends SimpleTaggableItem { private int targetNumHits = 0; + private int hnswExploreAdditionalHits = 0; + private boolean approximate = true; private String field; private String queryTensorName; @@ -34,12 +37,24 @@ public class NearestNeighborItem extends SimpleTaggableItem { /** Returns the field name */ public String getIndexName() { return field; } + /** Returns the number of extra hits to explore in HNSW algorithm */ + public int getHnswExploreAdditionalHits() { return hnswExploreAdditionalHits; } + + /** Returns whether approximation is allowed */ + public boolean getAllowApproximate() { return approximate; } + /** Returns the name of the query tensor */ public String getQueryTensorName() { return queryTensorName; } /** Set the K number of hits to produce */ public void setTargetNumHits(int target) { this.targetNumHits = target; } + /** Set the number of extra hits to explore in HNSW algorithm */ + public void setHnswExploreAdditionalHits(int num) { this.hnswExploreAdditionalHits = num; } + + /** Set whether approximation is allowed */ + public void setAllowApproximate(boolean value) { this.approximate = value; } + @Override public void setIndexName(String index) { this.field = index; } @@ -58,6 +73,8 @@ public class NearestNeighborItem extends SimpleTaggableItem { putString(field, buffer); putString(queryTensorName, buffer); IntegerCompressor.putCompressedPositiveNumber(targetNumHits, buffer); + IntegerCompressor.putCompressedPositiveNumber((approximate ? 1 : 0), buffer); + IntegerCompressor.putCompressedPositiveNumber(hnswExploreAdditionalHits, buffer); return 1; // number of encoded stack dump items } @@ -65,6 +82,19 @@ public class NearestNeighborItem extends SimpleTaggableItem { protected void appendBodyString(StringBuilder buffer) { buffer.append("{field=").append(field); buffer.append(",queryTensorName=").append(queryTensorName); - buffer.append(",targetNumHits=").append(targetNumHits).append("}"); + buffer.append(",hnsw.exploreAdditionalHits=").append(hnswExploreAdditionalHits); + buffer.append(",approximate=").append(approximate); + buffer.append(",targetHits=").append(targetNumHits).append("}"); } + + @Override + public void disclose(Discloser discloser) { + super.disclose(discloser); + discloser.addProperty("field", field); + discloser.addProperty("queryTensorName", queryTensorName); + discloser.addProperty("hnsw.exploreAdditionalHits", hnswExploreAdditionalHits); + discloser.addProperty("approximate", approximate); + discloser.addProperty("targetHits", targetNumHits); + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java b/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java index 84e93d5de8f..88982195af6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ONearItem.java @@ -25,10 +25,12 @@ public class ONearItem extends NearItem { super(distance); } + @Override public ItemType getItemType() { return ItemType.ONEAR; } + @Override public String getName() { return "ONEAR"; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java index 542f1393852..9b34fd7d62b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java @@ -19,16 +19,13 @@ public class PhraseSegmentItem extends IndexedSegmentItem { /** Whether this was explicitly written as a phrase using quotes by the user */ private boolean explicit = false; - /** - * Creates a phrase containing the same words and state (as pertinent) as - * the given SegmentAndItem. - */ - public PhraseSegmentItem(AndSegmentItem segAnd) { - super(segAnd.getRawWord(), segAnd.stringValue(), segAnd.isFromQuery(), segAnd.isStemmed(), segAnd.getOrigin()); - if (segAnd.getItemCount() > 0) { - WordItem w = (WordItem) segAnd.getItem(0); + /** Creates a phrase containing the same words and state (as pertinent) as the given SegmentAndItem. */ + public PhraseSegmentItem(AndSegmentItem andSegment) { + super(andSegment.getRawWord(), andSegment.stringValue(), andSegment.isFromQuery(), andSegment.isStemmed(), andSegment.getOrigin()); + if (andSegment.getItemCount() > 0) { + WordItem w = (WordItem) andSegment.getItem(0); setIndexName(w.getIndexName()); - for (Iterator<Item> i = segAnd.getItemIterator(); i.hasNext();) { + for (Iterator<Item> i = andSegment.getItemIterator(); i.hasNext();) { WordItem word = (WordItem) i.next(); addWordItem(word); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java index 97c68ee3da8..2c017410109 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordAlternativesItem.java @@ -12,8 +12,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.compress.IntegerCompressor; /** - * A set words with differing exactness scores to be used for literal boost - * ranking. + * A set of words with differing exactness scores to be used for literal boost ranking. * * @author Steinar Knutsen */ @@ -145,17 +144,14 @@ public class WordAlternativesItem extends TermItem { } /** - * Return an immutable snapshot of the contained terms. This list will not - * reflect later changes to the item. + * Return an immutable snapshot of the contained terms. This list will not reflect later changes to the item. * - * @return an immutable list of word alternatives and their respective - * scores + * @return an immutable list of word alternatives and their respective scores */ public List<Alternative> getAlternatives() { return alternatives; } - @Override public void encodeThis(ByteBuffer target) { super.encodeThis(target); @@ -172,17 +168,14 @@ public class WordAlternativesItem extends TermItem { * equal or higher exactness score. If the term string is present with a * lower exactness score, the new, higher score will take precedence. * - * @param term - * one of several string interpretations of the input word - * @param exactness - * how close the term string matches what the user input + * @param term one of several string interpretations of the input word + * @param exactness how close the term string matches what the user input */ public void addTerm(String term, double exactness) { // do note, Item is Cloneable, and overwriting the reference is what // saves us from overriding the method - if (alternatives.stream().anyMatch((a) -> a.word.equals(term) && a.exactness >= exactness )) { - return; - } + if (alternatives.stream().anyMatch((a) -> a.word.equals(term) && a.exactness >= exactness )) return; + List<Alternative> newTerms = new ArrayList<>(alternatives.size() + 1); newTerms.addAll(alternatives); newTerms.add(new Alternative(term, exactness)); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index cd8579be7f0..902be7e15dd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -326,6 +326,7 @@ public abstract class AbstractParser implements CustomParser { * * @param indexName the index name which preceeded this token, or null if none * @param token the token to segment + * @param quoted whether this segment is within quoted text * @return the resulting item */ // TODO: The segmenting stuff is a mess now, this will fix it: @@ -341,7 +342,7 @@ public abstract class AbstractParser implements CustomParser { // This can be solved by making the segment method language independent by // always producing a query item containing the token text and resolve it to a WordItem or // SegmentItem after parsing and language detection. - protected Item segment(String indexName, Token token) { + protected Item segment(String indexName, Token token, boolean quoted) { String normalizedToken = normalize(token.toString()); if (token.isSpecial()) { @@ -361,12 +362,13 @@ public abstract class AbstractParser implements CustomParser { if (segments.size() == 0) { return null; } + if (segments.size() == 1) { return new WordItem(segments.get(0), "", true, token.substring); } CompositeItem composite; - if (indexFacts.getIndex(indexName).getPhraseSegmenting()) { + if (indexFacts.getIndex(indexName).getPhraseSegmenting() || quoted) { composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring); } else { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index d9b969757c2..49bdba2c90f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -30,6 +30,7 @@ public class AllParser extends SimpleParser { super(environment); } + @Override protected Item parseItems() { int position = tokens.getPosition(); try { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index dd836e9c8e1..b714a1d8b34 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -35,21 +35,12 @@ public class AnyParser extends SimpleParser { return anyItems(true); } - Item parseFilter(String filter, Language queryLanguage, Set<String> searchDefinitions) { - return parseFilter(filter, queryLanguage, environment.getIndexFacts().newSession(searchDefinitions, Collections.emptySet())); - } - Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) { - Item filterRoot; - setState(queryLanguage, indexFacts); tokenize(filter, null, indexFacts, queryLanguage); - filterRoot = anyItems(true); - - if (filterRoot == null) { - return null; - } + Item filterRoot = anyItems(true); + if (filterRoot == null) return null; markAllTermsAsFilters(filterRoot); return filterRoot; @@ -61,18 +52,10 @@ public class AnyParser extends SimpleParser { try { tokens.skipMultiple(PLUS); + if ( ! tokens.skipMultiple(MINUS)) return null; + if (tokens.currentIsNoIgnore(SPACE)) return null; - if (!tokens.skipMultiple(MINUS)) { - return null; - } - - if (tokens.currentIsNoIgnore(SPACE)) { - return null; - } - - if (item == null) { - item = indexableItem(); - } + item = indexableItem(); if (item == null) { item = compositeItem(); @@ -88,13 +71,13 @@ public class AnyParser extends SimpleParser { } } } - if (item!=null) + if (item != null) item.setProtected(true); + return item; } finally { - if (item == null) { + if (item == null) tokens.setPosition(position); - } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java index 6d4401aca04..12f63276269 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java @@ -23,8 +23,7 @@ public class PhraseParser extends AbstractParser { /** * Ignores everything but words and numbers * - * @return a phrase item if several words/numbers was found, - * a word item if only one was found + * @return a phrase item if several words/numbers was found, a word item if only one was found */ private Item forcedPhrase() { Item firstWord = null; @@ -38,7 +37,7 @@ public class PhraseParser extends AbstractParser { } // Note, this depends on segment never creating AndItems when quoted // (the second argument) is true. - Item newWord = segment(null, token); + Item newWord = segment(null, token, true); if (firstWord == null) { // First pass firstWord = newWord; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 9ddfea6dffb..0686a4bdb43 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -50,32 +50,28 @@ abstract class SimpleParser extends StructuredParser { private Item anyItemsBody(boolean topLevel) { Item topLevelItem = null; NotItem not = null; - Item item; + Item item = null; do { - item = null; - - if (item == null) { - item = positiveItem(); - if (item != null) { - if (not == null) { - not = new NotItem(); - not.addPositiveItem(item); - topLevelItem = combineItems(topLevelItem, not); - } else { - not.addPositiveItem(item); - } + item = positiveItem(); + if (item != null) { + if (not == null) { + not = new NotItem(); + not.addPositiveItem(item); + topLevelItem = combineItems(topLevelItem, not); + } else { + not.addPositiveItem(item); } } if (item == null) { item = negativeItem(); if (item != null) { - if (not == null && item != null) { + if (not == null) { not = new NotItem(); not.addNegativeItem(item); topLevelItem = combineItems(topLevelItem, not); - } else if (item != null) { + } else { not.addNegativeItem(item); } } @@ -97,9 +93,8 @@ abstract class SimpleParser extends StructuredParser { if (item != null) { if (topLevelItem == null) { topLevelItem = item; - } else if (needNewTopLevel(topLevelItem, item)) { + } else if (needNewORTopLevel(topLevelItem, item)) { CompositeItem newTop = new OrItem(); - newTop.addItem(topLevelItem); newTop.addItem(item); topLevelItem = newTop; @@ -144,21 +139,13 @@ abstract class SimpleParser extends StructuredParser { } } - - /** Says whether we need a new top level item given the new item */ - private boolean needNewTopLevel(Item topLevelItem, Item item) { - if (item == null) { - return false; - } - if (topLevelItem instanceof TermItem) { - return true; - } - if (topLevelItem instanceof PhraseItem) { - return true; - } - if (topLevelItem instanceof BlockItem) { - return true; - } + /** Says whether we need a new top level OR item given the new item */ + private boolean needNewORTopLevel(Item topLevelItem, Item item) { + if (item == null) return false; + if (topLevelItem instanceof TermItem) return true; + if (topLevelItem instanceof PhraseItem) return true; + if (topLevelItem instanceof BlockItem) return true; + if ( topLevelItem instanceof AndItem) return true; return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java index c206ff7567e..95a374316de 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.query.Substring; import java.util.*; @@ -66,7 +66,7 @@ public class SpecialTokens { // TODO put along with the global toLowerCase String asHigh = token.toUpperCase(Locale.ENGLISH); if (asLow.length() != token.length() || asHigh.length() != token.length()) { - log.log(LogLevel.ERROR, "Special token '" + token + "' has case sensitive length. Ignoring the token." + log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token." + " Please report this message in a bug to the Vespa team."); return false; } else { @@ -159,7 +159,7 @@ public class SpecialTokens { @Override public int hashCode() { return token.hashCode(); } - public Token toToken(int start,String rawSource) { + public Token toToken(int start, String rawSource) { return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe? } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index ee4c0d4d9f0..76ea7fb11a8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -406,13 +406,18 @@ abstract class StructuredParser extends AbstractParser { } } - /** Words for phrases also permits numerals as words */ - private Item phraseWord(String indexName, boolean insidePhrase) { + /** + * Words for phrases also permits numerals as words + * + * @param quoted whether we are consuming text within quoted + * @param insidePhrase whether we are consuming additional items for an existing phrase + */ + private Item phraseWord(String indexName, boolean quoted, boolean insidePhrase) { int position = tokens.getPosition(); Item item = null; try { - item = word(indexName); + item = word(indexName, quoted); if (item == null && tokens.currentIs(NUMBER)) { Token t = tokens.next(); @@ -434,17 +439,19 @@ abstract class StructuredParser extends AbstractParser { /** * Returns a WordItem if this is a non CJK query, - * a WordItem or PhraseSegmentItem if this is a CJK query, + * a WordItem or SegmentItem if this is a CJK query, * null if the current item is not a word + * + * @param quoted whether this token is inside quotes */ - private Item word(String indexName) { + private Item word(String indexName, boolean quoted) { int position = tokens.getPosition(); Item item = null; try { - if (!tokens.currentIs(WORD) - && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) - && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { + if ( ! tokens.currentIs(WORD) + && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS) + && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) { return null; } Token word = tokens.next(); @@ -452,7 +459,7 @@ abstract class StructuredParser extends AbstractParser { if (submodes.url) { item = new WordItem(word, true); } else { - item = segment(indexName, word); + item = segment(indexName, word, quoted); } if (submodes.url || submodes.site) { @@ -520,7 +527,7 @@ abstract class StructuredParser extends AbstractParser { /** Returns a word, a phrase, or another composite */ private Item phraseBody(String indexName) { boolean quoted = false; - PhraseItem phrase = null; + CompositeItem composite = null; Item firstWord = null; boolean starAfterFirst = false; boolean starBeforeFirst; @@ -539,7 +546,7 @@ abstract class StructuredParser extends AbstractParser { quoted = !quoted; } - Item word = phraseWord(indexName, (firstWord != null) || (phrase != null)); + Item word = phraseWord(indexName, quoted, (firstWord != null) || (composite != null)); if (word == null) { if (tokens.skipMultiple(QUOTE)) { @@ -555,34 +562,39 @@ abstract class StructuredParser extends AbstractParser { ((PhraseSegmentItem) word).setExplicit(true); } - if (phrase != null) { - phrase.addItem(word); + if (composite != null) { + composite.addItem(word); + connectLastTermsIn(composite); } else if (firstWord != null) { if (submodes.site || submodes.url) { UriItem uriItem = new UriItem(); if (submodes.site) uriItem.setEndAnchorDefault(true); - phrase = uriItem; + composite = uriItem; } else { - phrase = new PhraseItem(); + if (quoted || indexFacts.getIndex(indexName).getPhraseSegmenting()) + composite = new PhraseItem(); + else + composite = new AndItem(); } - if (quoted || submodes.site || submodes.url) { - phrase.setExplicit(true); + if ( (quoted || submodes.site || submodes.url) && composite instanceof PhraseItem) { + ((PhraseItem)composite).setExplicit(true); } if (addStartOfHostMarker) { - phrase.addItem(MarkerWordItem.createStartOfHost()); + composite.addItem(MarkerWordItem.createStartOfHost()); } if (firstWord instanceof IntItem) { IntItem asInt = (IntItem) firstWord; firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), true, asInt.getOrigin()); } - phrase.addItem(firstWord); - phrase.addItem(word); + composite.addItem(firstWord); + composite.addItem(word); + connectLastTermsIn(composite); } else if (word instanceof PhraseItem) { - phrase = (PhraseItem) word; + composite = (PhraseItem)word; } else { firstWord = word; starAfterFirst = tokens.skipNoIgnore(STAR); @@ -609,29 +621,29 @@ abstract class StructuredParser extends AbstractParser { braceLevelURL = 0; - if (phrase != null) { + if (composite != null) { if (addEndMarking()) { - phrase.addItem(MarkerWordItem.createEndOfHost()); + composite.addItem(MarkerWordItem.createEndOfHost()); } - return phrase; + return composite; } else if (firstWord != null && submodes.site) { if (starAfterFirst && !addStartOfHostMarker) { return firstWord; } else { - phrase = new PhraseItem(); + composite = new PhraseItem(); + ((PhraseItem)composite).setExplicit(true); if (addStartOfHostMarker) { - phrase.addItem(MarkerWordItem.createStartOfHost()); + composite.addItem(MarkerWordItem.createStartOfHost()); } if (firstWord instanceof IntItem) { IntItem asInt = (IntItem) firstWord; firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), true, asInt.getOrigin()); } - phrase.addItem(firstWord); + composite.addItem(firstWord); if (!starAfterFirst) { - phrase.addItem(MarkerWordItem.createEndOfHost()); + composite.addItem(MarkerWordItem.createEndOfHost()); } - phrase.setExplicit(true); - return phrase; + return composite; } } else { if (firstWord != null && firstWord instanceof TermItem && (starAfterFirst || starBeforeFirst)) { @@ -651,6 +663,15 @@ abstract class StructuredParser extends AbstractParser { } } + private void connectLastTermsIn(CompositeItem composite) { + int items = composite.items().size(); + if (items < 2) return; + Item nextToLast = composite.items().get(items - 2); + Item last = composite.items().get(items - 1); + if ( ! (nextToLast instanceof TermItem)) return; + ((TermItem)nextToLast).setConnectivity(last, 1); + } + private boolean addStartMarking() { if (submodes.explicitAnchoring() && tokens.currentIs(HAT)) { tokens.skip(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 61f09e2f7b7..5e243e52057 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -108,8 +108,7 @@ public final class Tokenizer { if (i >= source.length()) break; int c = source.codePointAt(i); - if (characterClasses.isLetterOrDigit(c) - || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) { + if (characterClasses.isLetterOrDigit(c) || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) { i = consumeWordOrNumber(i, currentIndex); } else if (Character.isWhitespace(c)) { addToken(SPACE, " ", i, i + 1); @@ -187,7 +186,6 @@ public final class Tokenizer { return true; } - @SuppressWarnings({"deprecation"}) private Index determineCurrentIndex(Index defaultIndex, IndexFacts.Session indexFacts) { int backtrack = tokens.size(); int tokencnt = 0; @@ -328,7 +326,6 @@ public final class Tokenizer { wantEndQuote = true; actualStart = curPos+1; } else if (wantEndQuote && looksLikeExactEnd(curPos+1)) { - // System.err.println("seen quoted token from "+actualStart+" to "+curPos); seenSome = true; wantEndQuote = false; isQuoted = true; @@ -435,7 +432,7 @@ public final class Tokenizer { if (suffStar) { addToken(STAR, "*", starPos, starPos + 1); } - tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source))); // XXX: Unsafe? + tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source))); // skip terminating quote if (isQuoted) { @@ -451,17 +448,17 @@ public final class Tokenizer { break; end++; } - tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source))); // XXX: Unsafe start? - if (end>=source.length()) + tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source))); + if (end >= source.length()) return end; else - return end+terminator.length(); // Don't create a token for the terminator + return end + terminator.length(); // Don't create a token for the terminator } private boolean terminatorStartsAt(int start,String terminator) { - int terminatorPosition=0; - while ((terminatorPosition+start)<source.length()) { - if (source.charAt(start+terminatorPosition)!=terminator.charAt(terminatorPosition)) + int terminatorPosition = 0; + while ((terminatorPosition + start) < source.length()) { + if (source.charAt(start+terminatorPosition) != terminator.charAt(terminatorPosition)) return false; terminatorPosition++; if (terminatorPosition >= terminator.length()) @@ -481,8 +478,8 @@ public final class Tokenizer { while (tokenEnd < source.length()) { if (substringSpecialTokens) { - substringSpecialToken=getSpecialToken(tokenEnd); - if (substringSpecialToken!=null) break; + substringSpecialToken = getSpecialToken(tokenEnd); + if (substringSpecialToken != null) break; } int c = source.codePointAt(tokenEnd); @@ -506,7 +503,7 @@ public final class Tokenizer { // underscoresOnly = false; quotesOnly = false; } else if (c == '\'') { - if (!acceptApostropheAsWordCharacter(currentIndex)) { + if ( ! acceptApostropheAsWordCharacter(currentIndex)) { break; } // Otherwise consume apostrophes... @@ -530,15 +527,15 @@ public final class Tokenizer { } } - if (substringSpecialToken==null) + if (substringSpecialToken == null) return --tokenEnd; // TODO: test the logic around tokenEnd with friends - addToken(substringSpecialToken.toToken(tokenEnd,source)); - return --tokenEnd+substringSpecialToken.token().length(); + addToken(substringSpecialToken.toToken(tokenEnd, source)); + return --tokenEnd + substringSpecialToken.token().length(); } private void addToken(Token.Kind kind, String word, int start, int end) { - addToken(new Token(kind, word, false, new Substring(start, end, source))); // XXX: Unsafe? + addToken(new Token(kind, word, false, new Substring(start, end, source))); } private void addToken(Token token) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java index ae8c289a5b0..785477d6df7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/CJKSearcher.java @@ -56,26 +56,31 @@ public class CJKSearcher extends Searcher { AndItem replacement = new AndItem(); for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext();) { Item item = i.next(); - if (item instanceof WordItem) replacement.addItem(item); - else if (item instanceof PhraseSegmentItem) { + if (item instanceof WordItem) + replacement.addItem(item); + else if (item instanceof PhraseSegmentItem) replacement.addItem(new AndSegmentItem((PhraseSegmentItem) item)); - } - else replacement.addItem(item); // should never run, but hey... just convert and hope it's OK :) + else + replacement.addItem(item); // should never get here } return replacement; - } else if (root instanceof PhraseSegmentItem) { + } + else if (root instanceof PhraseSegmentItem) { PhraseSegmentItem asSegment = (PhraseSegmentItem) root; - if (asSegment.isExplicit() || hasOverlappingTokens(asSegment)) return root; - else return new AndSegmentItem(asSegment); - } else if (root instanceof SegmentItem) { + if (asSegment.isExplicit() || hasOverlappingTokens(asSegment)) + return root; + else + return new AndSegmentItem(asSegment); + } + else if (root instanceof SegmentItem) { return root; // avoid descending into AndSegmentItems and similar - } else if (root instanceof CompositeItem) { + } + else if (root instanceof CompositeItem) { for (ListIterator<Item> i = ((CompositeItem) root).getItemIterator(); i.hasNext();) { Item item = i.next(); Item transformedItem = transform(item); - if (item != transformedItem) { + if (item != transformedItem) i.set(transformedItem); - } } return root; } @@ -96,8 +101,7 @@ public class CJKSearcher extends Searcher { * We have overlapping tokens (see * com.yahoo.prelude.querytransform.test.CJKSearcherTestCase * .testCjkQueryWithOverlappingTokens and ParseTestCase for an explanation) - * if the sum of length of tokens is greater than the lenght of the original - * word + * if the sum of length of tokens is greater than the length of the original word */ private boolean hasOverlappingTokens(PhraseSegmentItem segments) { int segmentsLength=0; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java index fdd6ad47a98..ce13045b518 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java @@ -111,25 +111,17 @@ public class NormalizingSearcher extends Searcher { } private void normalizeAlternatives(Language language, Session indexFacts, WordAlternativesItem block) { - if (!block.isNormalizable()) { - return; - } - { - Index index = indexFacts.getIndex(block.getIndexName()); - if (index.isAttribute()) { - return; - } - if (!index.getNormalize()) { - return; - } - } + if ( ! block.isNormalizable()) return; + + Index index = indexFacts.getIndex(block.getIndexName()); + if (index.isAttribute()) return; + if ( ! index.getNormalize()) return; List<Alternative> terms = block.getAlternatives(); for (Alternative term : terms) { String accentDropped = linguistics.getTransformer().accentDrop(term.word, language); - if (!term.word.equals(accentDropped) && accentDropped.length() > 0) { + if ( ! term.word.equals(accentDropped) && accentDropped.length() > 0) block.addTerm(accentDropped, term.exactness * .7d); - } } } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java index 84c793a6df1..5a936d42ccc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java @@ -4,6 +4,8 @@ package com.yahoo.prelude.querytransform; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.HasIndexItem; +import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NotItem; @@ -169,7 +171,9 @@ public class QueryRewrite { removeOtherNonrankedChildren(item, i); recall = Recall.RECALLS_EVERYTHING; } else if ((item instanceof AndItem) || (item instanceof NearItem)) { - item.removeItem(i); + if ( ! isRanked(item.getItem(i))) { + item.removeItem(i); + } } else if (item instanceof RankItem) { // empty } else { @@ -200,6 +204,20 @@ public class QueryRewrite { parent.removeItem(i); } } + + private static boolean isRanked(Item item) { + if (item instanceof CompositeItem) { + for (Item child : ((CompositeItem)item).items()) + if (isRanked(child)) return true; + return false; + } + else if (item instanceof HasIndexItem && Hit.SDDOCNAME_FIELD.equals(((HasIndexItem)item).getIndexName())) { + return false; // No point in ranking by sddocname + } + else { + return item.isRanked(); + } + } private static Item collapseSingleComposites(Item item) { if (!(item instanceof CompositeItem)) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 655fbf6acc3..318912eab04 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -9,7 +9,7 @@ import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.StemList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; @@ -98,6 +98,7 @@ public class StemmingSearcher extends Searcher { context.language = language; context.indexFacts = indexFacts; context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot()); + q.trace("Stemming with language="+language, 3); return scan(q.getModel().getQueryTree().getRoot(), context); } @@ -167,7 +168,7 @@ public class StemmingSearcher extends Searcher { if (i instanceof TermItem) { return ((TermItem) i).getOrigin(); // this should always be the case } else { - getLogger().log(LogLevel.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + + getLogger().log(Level.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + i.getClass().getName() + ", expected TermItem."); } } @@ -183,20 +184,29 @@ public class StemmingSearcher extends Searcher { Substring substring = getOffsets(current); if (segments.size() == 1) { + getLogger().log(Level.FINE, () -> "Stem '"+current.stringValue()+"' mode "+index.getStemMode() + +" and language '"+context.language+"' -> '"+segments.get(0)+"'"); TaggableItem w = singleWordSegment(current, segments.get(0), index, substring, context.insidePhrase); setMetaData(current, context.reverseConnectivity, w); return (Item) w; + } else if (getLogger().isLoggable(Level.FINE)) { + var buf = new StringBuilder(); + buf.append("Stem '").append(current.stringValue()); + buf.append("' mode ").append(index.getStemMode()); + buf.append(" and language '").append(context.language).append("' ->"); + for (StemList segment : segments) { + buf.append(" '").append(segment).append("'"); + } + getLogger().log(Level.FINE, buf.toString()); } - if (context.isCJK) { - composite = chooseCompositeForCJK(current, - ((Item) current).getParent(), - indexName); - } else { - composite = phraseSegment(current, indexName); - } + if (context.isCJK) + composite = chooseCompositeForCJK(current, ((Item) current).getParent(), indexName); + else + composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { + getLogger().log(Level.FINE, () -> "Stem to multiple segments '"+segment+"'"); TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); if (composite instanceof AndSegmentItem) { @@ -331,39 +341,34 @@ public class StemmingSearcher extends Searcher { } } + private CompositeItem chooseComposite(BlockItem current, CompositeItem parent, String indexName) { + if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) + return createPhraseSegment(current, indexName); + else + return createAndSegment(current); + + } + private CompositeItem chooseCompositeForCJK(BlockItem current, CompositeItem parent, String indexName) { - CompositeItem composite; - if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) { - if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) { - composite = phraseSegment(current, indexName); - } else - composite = createAndSegment(current); - } else { - switch (current.getSegmentingRule()) { - case PHRASE: - composite = phraseSegment(current, indexName); - break; - case BOOLEAN_AND: - composite = createAndSegment(current); - break; + if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) + return chooseComposite(current, parent, indexName); + + switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only + case PHRASE: return createPhraseSegment(current, indexName); + case BOOLEAN_AND: return createAndSegment(current); default: - throw new IllegalArgumentException( - "Unknown segmenting rule: " - + current.getSegmentingRule() - + ". This is a bug in Vespa, as the implementation has gotten out of sync." - + " Please create a ticket as soon as possible."); - } + throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() + + ". This is a bug in Vespa, as the implementation has gotten out of sync." + + " Please create a ticket as soon as possible."); } - return composite; } private AndSegmentItem createAndSegment(BlockItem current) { return new AndSegmentItem(current.stringValue(), true, true); } - private CompositeItem phraseSegment(BlockItem current, String indexName) { - CompositeItem composite; - composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true); + private CompositeItem createPhraseSegment(BlockItem current, String indexName) { + CompositeItem composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true); composite.setIndexName(indexName); return composite; } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 43717ecf6cd..37561d3a0f5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -147,7 +147,9 @@ public class PosSearcher extends Searcher { String radius = query.properties().getString(posRadius); int radiusUnits; if (radius == null) { - radiusUnits = 5000; + double radiuskm = 50.0; + double radiusdegrees = radiuskm * km2deg; + radiusUnits = (int)(radiusdegrees * 1000000); } else if (radius.endsWith("km")) { double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2)); double radiusdegrees = radiuskm * km2deg; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java index 9b6f5926b61..a8b3c76fe00 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.searcher; -import java.util.Optional; import com.yahoo.component.chain.dependencies.After; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.SimpleIndexedItem; import com.yahoo.prelude.query.ToolBox; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -15,7 +15,8 @@ import com.yahoo.search.querytransform.BooleanSearcher; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import java.util.Collection; +import java.util.ArrayList; +import java.util.List; /** * Checks that predicate queries don't use values outside the defined upper/lower bounds. @@ -27,26 +28,26 @@ public class ValidatePredicateSearcher extends Searcher { @Override public Result search(Query query, Execution execution) { - Optional<ErrorMessage> e = validate(query, execution.context().getIndexFacts().newSession(query)); - if (e.isPresent()) { + List<ErrorMessage> errorMessages = validate(query, execution.context().getIndexFacts().newSession(query)); + if (!errorMessages.isEmpty()) { Result r = new Result(query); - r.hits().addError(e.get()); + errorMessages.forEach(msg -> r.hits().addError(msg)); return r; } return execution.search(query); } - private Optional<ErrorMessage> validate(Query query, IndexFacts.Session indexFacts) { + private List<ErrorMessage> validate(Query query, IndexFacts.Session indexFacts) { ValidatePredicateVisitor visitor = new ValidatePredicateVisitor(indexFacts); ToolBox.visit(visitor, query.getModel().getQueryTree().getRoot()); - return visitor.errorMessage; + return visitor.errorMessages; } private static class ValidatePredicateVisitor extends ToolBox.QueryVisitor { private final IndexFacts.Session indexFacts; - public Optional<ErrorMessage> errorMessage = Optional.empty(); + final List<ErrorMessage> errorMessages = new ArrayList<>(); public ValidatePredicateVisitor(IndexFacts.Session indexFacts) { this.indexFacts = indexFacts; @@ -57,22 +58,37 @@ public class ValidatePredicateSearcher extends Searcher { if (item instanceof PredicateQueryItem) { visit((PredicateQueryItem) item); } + if (item instanceof SimpleIndexedItem) { + visit((SimpleIndexedItem) item); + } return true; } private void visit(PredicateQueryItem item) { - Index index = getIndexFromUnionOfDocumentTypes(item); + Index index = getIndexFromUnionOfDocumentTypes(item.getIndexName()); + if (!index.isPredicate()) { + errorMessages.add(ErrorMessage.createIllegalQuery(String.format("Index '%s' is not a predicate attribute.", index.getName()))); + } for (PredicateQueryItem.RangeEntry entry : item.getRangeFeatures()) { long value = entry.getValue(); if (value < index.getPredicateLowerBound() || value > index.getPredicateUpperBound()) { - errorMessage = Optional.of(ErrorMessage.createIllegalQuery( - String.format("%s=%d outside configured predicate bounds.", entry.getKey(), value))); + errorMessages.add( + ErrorMessage.createIllegalQuery(String.format("%s=%d outside configured predicate bounds.", entry.getKey(), value))); } } } - private Index getIndexFromUnionOfDocumentTypes(PredicateQueryItem item) { - return indexFacts.getIndex(item.getIndexName()); + private void visit(SimpleIndexedItem item) { + String indexName = item.getIndexName(); + Index index = getIndexFromUnionOfDocumentTypes(indexName); + if (index.isPredicate()) { + errorMessages.add( + ErrorMessage.createIllegalQuery(String.format("Index '%s' is predicate attribute and can only be used in conjunction with a predicate query operator.", indexName))); + } + } + + private Index getIndexFromUnionOfDocumentTypes(String indexName) { + return indexFacts.getIndex(indexName); } @Override diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java index bbdb3b796a2..82148cf54e6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java @@ -119,6 +119,9 @@ public class ValidateSortingSearcher extends Searcher { String name = f.getFieldName(); if ("[rank]".equals(name) || "[docid]".equals(name)) { // built-in constants - ok + } else if ("[relevance]".equals(name)) { + // built-in constant '[relevance]' must map to '[rank]' + f.getSorter().setName("[rank]"); } else if ("[relevancy]".equals(name)) { // built-in constant '[relevancy]' must map to '[rank]' f.getSorter().setName("[rank]"); diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 77cfebd07e5..1867da0317b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -5,7 +5,7 @@ import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.protect.Error; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.MetricSettings; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.processing.request.CompoundName; @@ -248,7 +248,7 @@ public class StatisticsSearcher extends Searcher { if (latency >= 0) { addLatency(latency, metricContext); } else { - getLogger().log(LogLevel.WARNING, + getLogger().log(Level.WARNING, "Apparently negative latency measure, start: " + start + ", end: " + end + ", for query: " + query.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 395d8853603..9e3f6d20e36 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -7,7 +7,7 @@ import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.QueryException; @@ -288,7 +288,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(""); } - /** * Construct a query from a string formatted in the http style, e.g <code>?query=test&offset=10&hits=13</code> * The query must be uri encoded. @@ -297,7 +296,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { this(query, null); } - /** * Creates a query from a request * @@ -665,7 +663,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // getQueryTree isn't exception safe try { queryTree = model.getQueryTree().toString(); - } catch (Exception e) { + } catch (Exception | StackOverflowError e) { queryTree = "[Could not parse user input: " + model.getQueryString() + "]"; } return "query '" + queryTree + "'"; @@ -677,7 +675,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // getQueryTree isn't exception safe try { queryTree = model.getQueryTree().toString(); - } catch (Exception e) { + } catch (Exception | StackOverflowError e) { queryTree = "Could not parse user input: " + model.getQueryString(); } return "query=[" + queryTree + "]" + " offset=" + getOffset() + " hits=" + getHits() + "]"; @@ -728,7 +726,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (includeQuery) message += ": [" + queryTreeText() + "]"; - log.log(LogLevel.DEBUG,message); + log.log(Level.FINE,message); // Pass 0 as traceLevel as the trace level check is already done above, // and it is not propagated to trace until execution has started diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index 4080b09f40b..ab48d5797b2 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -89,7 +89,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone * with a result. It should <b>always</b> be called when adding * hits from a result, but there is no constraints on the order of the calls. */ - @SuppressWarnings("deprecation") public void mergeWith(Result result) { totalHitCount += result.getTotalHitCount(); deepHitCount += result.getDeepHitCount(); diff --git a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java index dd01d895963..0d491d2f0c1 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java @@ -79,6 +79,8 @@ public abstract class BaseNodeMonitor<T> { */ public abstract void responded(); + /** @deprecated Not used */ + @Deprecated // TODO: Remove on Vespa 8 public boolean isIdle() { return (now()-respondedAt) >= configuration.getIdleLimit(); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index d4b6279be89..15cf4995b77 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -38,6 +38,9 @@ public class ClusterMonitor<T> { /** A map from Node to corresponding MonitoredNode */ private final Map<T, TrafficNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); + /** @deprecated It is not advised to start the monitoring thread in the constructor. + * Use ClusterMonitor(NodeManager manager, false) and explicit start(). */ + @Deprecated public ClusterMonitor(NodeManager<T> manager) { this(manager, true); } @@ -50,6 +53,12 @@ public class ClusterMonitor<T> { } } + public void start() { + if ( ! monitorThread.isAlive()) { + monitorThread.start(); + } + } + /** Returns the configuration of this cluster monitor */ public MonitorConfiguration getConfiguration() { return configuration; } @@ -92,7 +101,7 @@ public class ClusterMonitor<T> { Boolean wasWorking = monitor.isKnownWorking(); monitor.responded(); if (wasWorking != monitor.isKnownWorking()) - nodeManager.working(monitor.getNode()); + nodeManager.working(node); } /** @@ -101,7 +110,7 @@ public class ClusterMonitor<T> { public void ping(Executor executor) { for (Iterator<BaseNodeMonitor<T>> i = nodeMonitorIterator(); i.hasNext() && !closed.get(); ) { BaseNodeMonitor<T> monitor= i.next(); - nodeManager.ping(monitor.getNode(), executor); // Cause call to failed or responded + nodeManager.ping(this, monitor.getNode(), executor); // Cause call to failed or responded } if (closed.get()) return; // Do nothing to change state if close has started. nodeManager.pingIterationCompleted(); diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 06cbf9a9706..db79c610dc7 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.cluster; import com.yahoo.component.ComponentId; import com.yahoo.container.protect.Error; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.yolean.Exceptions; @@ -58,7 +58,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod this(id, connections, hasher, internal, true); } - public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) { + protected ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) { super(id); this.hasher = hasher; this.monitor = new ClusterMonitor<>(this, startPingThread); @@ -70,8 +70,8 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod /** Pinging a node, called from ClusterMonitor */ @Override - public final void ping(T p, Executor executor) { - log(LogLevel.FINE, "Sending ping to: ", p); + public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) { + log(Level.FINE, "Sending ping to: ", p); Pinger pinger = new Pinger(p); FutureTask<Pong> future = new FutureTask<>(pinger); @@ -80,7 +80,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod Throwable logThrowable = null; try { - pong = future.get(monitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); + pong = future.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { pong = new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + p)); logThrowable = e; @@ -96,11 +96,11 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod future.cancel(true); if (pong.badResponse()) { - monitor.failed(p, pong.getError(0)); - log(LogLevel.FINE, "Failed ping - ", pong); + clusterMonitor.failed(p, pong.error().get()); + log(Level.FINE, "Failed ping - ", pong); } else { - monitor.responded(p); - log(LogLevel.FINE, "Answered ping - ", p); + clusterMonitor.responded(p); + log(Level.FINE, "Answered ping - ", p); } if (logThrowable != null) { @@ -173,7 +173,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (result.hits().getError().getCode() == Error.TIMEOUT.code) return result; // Retry is unlikely to help - log(LogLevel.FINER, "No result, checking for timeout."); + log(Level.FINER, "No result, checking for timeout."); tries++; connection = nodes.select(code, tries); } while (tries < nodes.getNodeCount()); @@ -211,7 +211,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod try { result = search(query, execution, connection); } catch (RuntimeException e) { //TODO: Exceptions should not be used to signal backend communication errors - log(LogLevel.WARNING, "An exception occurred while invoking backend searcher.", e); + log(Level.WARNING, "An exception occurred while invoking backend searcher.", e); result = new Result(query, ErrorMessage.createBackendCommunicationError("Failed calling " + connection + " in " + this + " for " + query + ": " + Exceptions.toMessageString(e))); @@ -220,14 +220,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (result == null) result = new Result(query, ErrorMessage.createBackendCommunicationError("No result returned in " + this + " from " + connection + " for " + query)); - - if (result.hits().getError() != null) { - log(LogLevel.FINE, "FAILED: ", query); - } else if ( ! result.isCached()) { - log(LogLevel.FINE, "WORKING: ", query); - } else { - log(LogLevel.FINE, "CACHE HIT: ", query); - } return result; } @@ -263,13 +255,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling " + result + " from " + connection + ": " + Exceptions.toMessageString(e))); } - if (result.hits().getError() != null) { - log(LogLevel.FINE, "FAILED: ", result.getQuery()); - } else if ( ! result.isCached()) { - log(LogLevel.FINE, "WORKING: ", result.getQuery()); - } else { - log(LogLevel.FINE, "CACHE HIT: " + result.getQuery()); - } } /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java index 46752b0bedb..6c83e1c64e3 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java @@ -11,6 +11,7 @@ package com.yahoo.search.cluster; public class Hasher<T> { public static class NodeFactor<T> { + private final T node; /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java index 226e0180d2e..a2fb982e3c5 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java @@ -8,85 +8,45 @@ package com.yahoo.search.cluster; */ public class MonitorConfiguration { - /** - * The interval in ms between consecutive checks of the monitored - * nodes - */ + /** The interval in ms between consecutive checks of the monitored nodes */ private long checkInterval=1000; - /** - * The number of times a failed node must respond before getting - * traffic again - */ - private int responseAfterFailLimit=3; + /** The number of milliseconds to attempt to complete a request before giving up */ + private final long requestTimeout = 980; - /** - * The number of ms a node is allowed to stay idle before it is - * pinged - */ - private long idleLimit=3000; + /** The number of milliseconds a node is allowed to fail before we mark it as not working */ + private long failLimit = 5000; - /** - * The number of milliseconds to attempt to complete a request - * before giving up - */ - private long requestTimeout = 5000; + /** Sets the interval between each ping of idle or failing nodes. Default is 1000 ms. */ + public void setCheckInterval(long intervalMs) { this.checkInterval = intervalMs; } - /** - * The number of milliseconds a node is allowed to fail before we - * mark it as not working - */ - private long failLimit=5000; + /** Returns the interval between each ping of idle or failing nodes. Default is 1000 ms. */ + public long getCheckInterval() { return checkInterval; } /** - * The number of times a node is allowed to fail in one hour - * before it is quarantined for an hour + * Sets the number of times a failed node must respond before it is put back in service. Default is 3. + * + * @deprecated will go away in Vespa 8 */ - private int failQuarantineLimit=3; + @Deprecated // TODO: Remove on Vespa 8 + public void setResponseAfterFailLimit(int responseAfterFailLimit) { } /** - * The number of ms to quarantine an unstable node + * Sets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. + * + * @deprecated Will go away in Vespa 8 */ - private long quarantineTime=1000*60*60; - - /** - * Sets the interval between each ping of idle or failing nodes - * Default is 1000ms - */ - public void setCheckInterval(long intervalMs) { - this.checkInterval=intervalMs; - } + @Deprecated // TODO: Remove on Vespa 8 + public void setIdleLimit(int idleLimit) { } /** - * Returns the interval between each ping of idle or failing nodes - * Default is 1000ms - */ - public long getCheckInterval() { - return checkInterval; - } - - /** - * Sets the number of times a failed node must respond before it is put - * back in service. Default is 3. - */ - public void setResponseAfterFailLimit(int responseAfterFailLimit) { - this.responseAfterFailLimit=responseAfterFailLimit; - } - - /** - * Sets the number of ms a node (failing or working) is allowed to - * stay idle before it is pinged. Default is 3000 - */ - public void setIdleLimit(int idleLimit) { - this.idleLimit=idleLimit; - } - - /** - * Gets the number of ms a node (failing or working) - * is allowed to stay idle before it is pinged. Default is 3000 + * Gets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. + * + * @deprecated Will go away in Vespa 8 */ + @Deprecated // TODO: Remove on Vespa 8 public long getIdleLimit() { - return idleLimit; + return 3000; } /** @@ -112,29 +72,26 @@ public class MonitorConfiguration { * in quarantine. Once in quarantine it won't be put back in * productuion before quarantineTime has expired even if it is * working. Default is 3 + * + * @deprecated Will go away in Vespa 8 */ - public void setFailQuarantineLimit(int failQuarantineLimit) { - this.failQuarantineLimit=failQuarantineLimit; - } + @Deprecated // TODO: Remove on Vespa 8 + public void setFailQuarantineLimit(int failQuarantineLimit) { } /** - * The number of ms an unstable node is quarantined. Default is - * 100*60*60 + * The number of ms an unstable node is quarantined. Default is 100*60*60 + * + * @deprecated Will go away in Vespa 8 */ - public void setQuarantineTime(long quarantineTime) { - this.quarantineTime=quarantineTime; - } + @Deprecated // TODO: Remove on Vespa 8 + public void setQuarantineTime(long quarantineTime) { } public String toString() { return "monitor configuration [" + - "checkInterval: " + checkInterval + - " responseAfterFailLimit: " + responseAfterFailLimit + - " idleLimit: " + idleLimit + - " requestTimeout " + requestTimeout + - " feilLimit " + failLimit + - " failQuerantineLimit " + failQuarantineLimit + - " quarantineTime " + quarantineTime + - "]"; + "checkInterval: " + checkInterval + + " requestTimeout " + requestTimeout + + " failLimit " + failLimit + + "]"; } } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index 9b20139e3c5..836c71089c1 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -7,7 +7,7 @@ import java.util.concurrent.Executor; * Must be implemented by a node collection which wants * it's node state monitored by a ClusterMonitor * - * @author bratseth + * @author bratseth */ public interface NodeManager<T> { @@ -19,9 +19,22 @@ public interface NodeManager<T> { /** * Called when a node should be pinged. - * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded + * + * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead. */ - void ping(T node, Executor executor); + @Deprecated // TODO: Remove on Vespa 8 + default void ping(T node, Executor executor) { + throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method."); + } + + /** + * Called when a node should be pinged. + * This *must* lead to either a call to ClusterMonitor.failed or ClusterMonitor.responded + */ + default void ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor) { + ping(node, executor); + } /** Called right after a ping has been issued to each node. This default implementation does nothing. */ default void pingIterationCompleted() {} diff --git a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java index ccf3e863ff3..d17f6bfbaa8 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java @@ -52,8 +52,8 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> { * Called when a response is received from this node. */ public void responded() { - respondedAt=now(); - succeededAt=respondedAt; + respondedAt = now(); + succeededAt = respondedAt; setWorking(true,"Responds correctly"); } @@ -69,20 +69,20 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> { atStartUp = false; if (this.isWorking == working) return; // Old news - if (explanation==null) { - explanation=""; + if (explanation == null) { + explanation = ""; } else { - explanation=": " + explanation; + explanation = ": " + explanation; } if (working) { log.info("Putting " + node + " in service" + explanation); } else { log.warning("Taking " + node + " out of service" + explanation); - failedAt=now(); + failedAt = now(); } - this.isWorking=working; + this.isWorking = working; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 224facd0c5b..626cf087aca 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -5,18 +5,20 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; +import com.yahoo.compress.Compressor; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; @@ -30,6 +32,7 @@ import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.stream.Collectors; /** * A dispatcher communicates with search nodes to perform queries and fill hits. @@ -48,6 +51,7 @@ public class Dispatcher extends AbstractComponent { public static final String DISPATCH = "dispatch"; private static final String INTERNAL = "internal"; private static final String PROTOBUF = "protobuf"; + private static final String TOP_K_PROBABILITY = "topKProbability"; private static final String INTERNAL_METRIC = "dispatch_internal"; @@ -56,8 +60,12 @@ public class Dispatcher extends AbstractComponent { /** If enabled, search queries will use protobuf rpc */ public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF); + /** If set will control computation of how many hits will be fetched from each partition.*/ + public static final CompoundName topKProbability = CompoundName.fromComponents(DISPATCH, TOP_K_PROBABILITY); + /** A model of the search cluster this dispatches to */ private final SearchCluster searchCluster; + private final ClusterMonitor clusterMonitor; private final LoadBalancer loadBalancer; @@ -66,6 +74,8 @@ public class Dispatcher extends AbstractComponent { private final Metric metric; private final Metric.Context metricContext; + private final int maxHitsPerNode; + private static final QueryProfileType argumentType; static { @@ -74,54 +84,77 @@ public class Dispatcher extends AbstractComponent { argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(INTERNAL, FieldType.booleanType)); argumentType.addField(new FieldDescription(PROTOBUF, FieldType.booleanType)); + argumentType.addField(new FieldDescription(TOP_K_PROBABILITY, FieldType.doubleType)); argumentType.freeze(); } public static QueryProfileType getArgumentType() { return argumentType; } @Inject - public Dispatcher(ComponentId clusterId, + public Dispatcher(RpcResourcePool resourcePool, + ComponentId clusterId, DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, VipStatus vipStatus, Metric metric) { - this(new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus), - dispatchConfig, - metric); - } + this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig,clusterInfoConfig.nodeCount(), + vipStatus, new RpcPingFactory(resourcePool)), + dispatchConfig, metric); - private Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) { - this(searchCluster, - dispatchConfig, - new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster), - metric); } - /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(SearchCluster searchCluster, - DispatchConfig dispatchConfig, - RpcInvokerFactory rcpInvokerFactory, - Metric metric) { - this(searchCluster, dispatchConfig, rcpInvokerFactory, rcpInvokerFactory, metric); + private Dispatcher(RpcResourcePool resourcePool, SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) { + this(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, new RpcInvokerFactory(resourcePool, searchCluster), metric); } /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ - protected Dispatcher(SearchCluster searchCluster, + protected Dispatcher(ClusterMonitor clusterMonitor, + SearchCluster searchCluster, DispatchConfig dispatchConfig, InvokerFactory invokerFactory, - PingFactory pingFactory, Metric metric) { if (dispatchConfig.useMultilevelDispatch()) throw new IllegalArgumentException(searchCluster + " is configured with multilevel dispatch, but this is not supported"); this.searchCluster = searchCluster; + this.clusterMonitor = clusterMonitor; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.invokerFactory = invokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); + this.maxHitsPerNode = dispatchConfig.maxHitsPerNode(); + searchCluster.addMonitoring(clusterMonitor); + Thread warmup = new Thread(new Runnable() { + @Override + public void run() { + warmup(dispatchConfig.warmuptime()); + } + }); + warmup.start(); + try { + while ( ! searchCluster.hasInformationAboutAllNodes()) { + Thread.sleep(1); + } + warmup.join(); + } catch (InterruptedException e) {} + + /* + * No we have information from all nodes and a ping iteration has completed. + * Instead of waiting until next ping interval to update coverage and group state, + * we should compute the state ourselves, so that when the dispatcher is ready the state + * of its groups are also known. + */ + searchCluster.pingIterationCompleted(); + } - searchCluster.startClusterMonitoring(pingFactory); + /* + Will run important code in order to trigger JIT compilation and avoid cold start issues. + Currently warms up lz4 compression code. + */ + + private static long warmup(double seconds) { + return new Compressor().warmup(seconds); } /** Returns the search cluster this dispatches to */ @@ -131,8 +164,8 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { - /* The seach cluster must be shutdown first as it uses the invokerfactory. */ - searchCluster.shutDown(); + /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */ + clusterMonitor.shutdown(); invokerFactory.release(); } @@ -161,7 +194,11 @@ public class Dispatcher extends AbstractComponent { if (nodes.isEmpty()) return Optional.empty(); query.trace(false, 2, "Dispatching with search path ", searchPath); - return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), nodes, true); + return invokerFactory.createSearchInvoker(searcher, query, + OptionalInt.empty(), + nodes, + true, + maxHitsPerNode); } catch (InvalidSearchPathException e) { return Optional.of(new SearchErrorInvoker(ErrorMessage.createIllegalQuery(e.getMessage()))); } @@ -172,14 +209,19 @@ public class Dispatcher extends AbstractComponent { if (directNode.isPresent()) { Node node = directNode.get(); query.trace(false, 2, "Dispatching to ", node); - return invokerFactory.createSearchInvoker(searcher, query, OptionalInt.empty(), Arrays.asList(node), true) + return invokerFactory.createSearchInvoker(searcher, + query, + OptionalInt.empty(), + Arrays.asList(node), + true, + maxHitsPerNode) .orElseThrow(() -> new IllegalStateException("Could not dispatch directly to " + node)); } int covered = searchCluster.groupsWithSufficientCoverage(); int groups = searchCluster.orderedGroups().size(); int max = Integer.min(Integer.min(covered + 1, groups), MAX_GROUP_SELECTION_ATTEMPTS); - Set<Integer> rejected = null; + Set<Integer> rejected = rejectGroupBlockingFeed(searchCluster.orderedGroups()); for (int i = 0; i < max; i++) { Optional<Group> groupInCluster = loadBalancer.takeGroup(rejected); if (groupInCluster.isEmpty()) break; // No groups available @@ -190,7 +232,8 @@ public class Dispatcher extends AbstractComponent { query, OptionalInt.of(group.id()), group.nodes(), - acceptIncompleteCoverage); + acceptIncompleteCoverage, + maxHitsPerNode); if (invoker.isPresent()) { query.trace(false, 2, "Dispatching to group ", group.id()); query.getModel().setSearchPath("/" + group.id()); @@ -207,4 +250,21 @@ public class Dispatcher extends AbstractComponent { throw new IllegalStateException("No suitable groups to dispatch query. Rejected: " + rejected); } + /** + * We want to avoid groups blocking feed because their data may be out of date. + * If there is a single group blocking feed, we want to reject it. + * If multiple groups are blocking feed we should use them anyway as we may not have remaining + * capacity otherwise. Same if there are no other groups. + * + * @return a modifiable set containing the single group to reject, or null otherwise + */ + private Set<Integer> rejectGroupBlockingFeed(List<Group> groups) { + if (groups.size() == 1) return null; + List<Group> groupsRejectingFeed = groups.stream().filter(Group::isBlockingWrites).collect(Collectors.toList()); + if (groupsRejectingFeed.size() != 1) return null; + Set<Integer> rejected = new HashSet<>(); + rejected.add(groupsRejectingFeed.get(0).id()); + return rejected; + } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 84e5e7d747f..d8fb7b46440 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -36,6 +36,7 @@ import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT; * @author ollivir */ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseMonitor<SearchInvoker> { + private static final Logger log = Logger.getLogger(InterleavedSearchInvoker.class.getName()); private final Set<SearchInvoker> invokers; @@ -73,23 +74,30 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM * will be adjusted accordingly. */ @Override - protected void sendSearchRequest(Query query) throws IOException { + protected Object sendSearchRequest(Query query, Object unusedContext) throws IOException { this.query = query; invokers.forEach(invoker -> invoker.setMonitor(this)); deadline = currentTime() + query.getTimeLeft(); int originalHits = query.getHits(); int originalOffset = query.getOffset(); - query.setHits(query.getHits() + query.getOffset()); + int neededHits = originalHits + originalOffset; + Double topkProbabilityOverrride = query.properties().getDouble(Dispatcher.topKProbability); + int q = (topkProbabilityOverrride != null) + ? searchCluster.estimateHitsToFetch(neededHits, invokers.size(), topkProbabilityOverrride) + : searchCluster.estimateHitsToFetch(neededHits, invokers.size()); + query.setHits(q); query.setOffset(0); + Object context = null; for (SearchInvoker invoker : invokers) { - invoker.sendSearchRequest(query); + context = invoker.sendSearchRequest(query, context); askedNodes++; } query.setHits(originalHits); query.setOffset(originalOffset); + return null; } @Override @@ -320,4 +328,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM protected LinkedBlockingQueue<SearchInvoker> newQueue() { return new LinkedBlockingQueue<>(); } + + // For testing + Collection<SearchInvoker> invokers() { return invokers; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 6030e989595..03160e6c9c7 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -27,7 +27,10 @@ public abstract class InvokerFactory { this.searchCluster = searchCluster; } - protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node); + protected abstract Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + Query query, + int maxHits, + Node node); public abstract FillInvoker createFillInvoker(VespaBackEndSearcher searcher, Result result); @@ -40,20 +43,21 @@ public abstract class InvokerFactory { * @param nodes pre-selected list of content nodes * @param acceptIncompleteCoverage if some of the nodes are unavailable and this parameter is * false, verify that the remaining set of nodes has sufficient coverage - * @return Optional containing the SearchInvoker or empty if some node in the + * @return the invoker or empty if some node in the * list is invalid and the remaining coverage is not sufficient */ public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher, Query query, OptionalInt groupId, List<Node> nodes, - boolean acceptIncompleteCoverage) { + boolean acceptIncompleteCoverage, + int maxHits) { List<SearchInvoker> invokers = new ArrayList<>(nodes.size()); Set<Integer> failed = null; for (Node node : nodes) { boolean nodeAdded = false; if (node.isWorking() != Boolean.FALSE) { - Optional<SearchInvoker> invoker = createNodeSearchInvoker(searcher, query, node); + Optional<SearchInvoker> invoker = createNodeSearchInvoker(searcher, query, maxHits, node); if (invoker.isPresent()) { invokers.add(invoker.get()); nodeAdded = true; @@ -78,7 +82,7 @@ public abstract class InvokerFactory { if ( ! searchCluster.isPartialGroupCoverageSufficient(groupId, success) && !acceptIncompleteCoverage) { return Optional.empty(); } - if(invokers.size() == 0) { + if (invokers.size() == 0) { return Optional.of(createCoverageErrorInvoker(nodes, failed)); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 86f1999d8b4..8a90557fa3b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -12,15 +12,11 @@ public class LeanHit implements Comparable<LeanHit> { private final int distributionKey; public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) { - this.gid = gid; - this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance; - this.sortData = null; - this.partId = partId; - this.distributionKey = distributionKey; + this(gid, partId, distributionKey, relevance, null); } - public LeanHit(byte [] gid, int partId, int distributionKey, byte [] sortData) { + public LeanHit(byte [] gid, int partId, int distributionKey, double relevance, byte [] sortData) { this.gid = gid; - this.relevance = 0.0; + this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance; this.sortData = sortData; this.partId = partId; this.distributionKey = distributionKey; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 210ab5777d2..05e1ea6e2f9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -134,6 +134,7 @@ public class LoadBalancer { } private static class RoundRobinScheduler implements GroupScheduler { + private int needle = 0; private final List<GroupStatus> scoreboard; @@ -204,6 +205,7 @@ public class LoadBalancer { } static class AdaptiveScheduler implements GroupScheduler { + private final Random random; private final List<GroupStatus> scoreboard; @@ -251,4 +253,5 @@ public class LoadBalancer { return selectGroup(needle, false, rejectedGroups); } } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java index 52a45dc421c..256759360f7 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchErrorInvoker.java @@ -18,6 +18,7 @@ import java.util.Optional; * @author ollivir */ public class SearchErrorInvoker extends SearchInvoker { + private final ErrorMessage message; private Query query; private final Coverage coverage; @@ -34,11 +35,12 @@ public class SearchErrorInvoker extends SearchInvoker { } @Override - protected void sendSearchRequest(Query query) throws IOException { + protected Object sendSearchRequest(Query query, Object context) throws IOException { this.query = query; - if(monitor != null) { + if (monitor != null) { monitor.responseAvailable(this); } + return context; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java index 77b3df7c83a..b33e91189cc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchInvoker.java @@ -18,6 +18,7 @@ import java.util.Optional; * @author ollivir */ public abstract class SearchInvoker extends CloseableInvoker { + private final Optional<Node> node; private ResponseMonitor<SearchInvoker> monitor; @@ -31,14 +32,21 @@ public abstract class SearchInvoker extends CloseableInvoker { * for correct result windowing. */ public Result search(Query query, Execution execution) throws IOException { - sendSearchRequest(query); + sendSearchRequest(query, null); InvokerResult result = getSearchResult(execution); setFinalStatus(result.getResult().hits().getError() == null); result.complete(); return result.getResult(); } - protected abstract void sendSearchRequest(Query query) throws IOException; + /** + * + * @param query the query to send + * @param context a context object that can be used to pass context among different + * invokers, e.g for reuse of preserialized data. + * @return an object that can be passed to the next invocation of sendSearchRequest + */ + protected abstract Object sendSearchRequest(Query query, Object context) throws IOException; protected abstract InvokerResult getSearchResult(Execution execution) throws IOException; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java index 6800a80b78f..1e0153761c9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchPath.java @@ -26,6 +26,7 @@ import java.util.stream.IntStream; * @author ollivir */ public class SearchPath { + /** * Parse the search path and select nodes from the given cluster based on it. * @@ -193,7 +194,7 @@ public class SearchPath { private static Pair<String, String> halveAt(char divider, String string) { int pos = string.indexOf(divider); if (pos >= 0) { - return new Pair<>(string.substring(0, pos), string.substring(pos + 1, string.length())); + return new Pair<>(string.substring(0, pos), string.substring(pos + 1)); } return new Pair<>(string, ""); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java new file mode 100644 index 00000000000..d48e337b0c1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import org.apache.commons.math3.distribution.TDistribution; + +/** + * Use StudentT distribution and estimate how many hits you need from each partition + * to to get the globally top-k documents with the desired probability + * @author baldersheim + */ +public class TopKEstimator { + private final TDistribution studentT; + private final double defaultP; + private final boolean estimate; + private final double skewFactor; + + private static boolean needEstimate(double p) { + return (0.0 < p) && (p < 1.0); + } + TopKEstimator(double freedom, double defaultProbability) { + this(freedom, defaultProbability, 0.0); + } + public TopKEstimator(double freedom, double defaultProbability, double skewFactor) { + this.studentT = new TDistribution(null, freedom); + defaultP = defaultProbability; + estimate = needEstimate(defaultP); + this.skewFactor = skewFactor; + } + double estimateExactK(double k, double n, double p) { + double p_max = (1 + skewFactor)/n; + n = Math.max(1, 1/p_max); + double variance = k * 1/n * (1 - 1/n); + double p_inverse = 1 - (1 - p)/n; + return k/n + studentT.inverseCumulativeProbability(p_inverse) * Math.sqrt(variance); + } + double estimateExactK(double k, double n) { + return estimateExactK(k, n, defaultP); + } + public int estimateK(int k, int n) { + return (estimate && n > 1) + ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, defaultP))) + : k; + } + public int estimateK(int k, int n, double p) { + return (needEstimate(p) && (n > 1)) + ? Math.min(k, (int)Math.ceil(estimateExactK(k, n, p))) + : k; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java index e54e2187818..f4536a7aa4e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/Client.java @@ -13,6 +13,7 @@ import java.util.Optional; * @author bratseth */ interface Client { + /** Creates a connection to a particular node in this */ NodeConnection createConnection(String hostname, int port); @@ -21,6 +22,7 @@ interface Client { } class ResponseOrError<T> { + final Optional<T> response; final Optional<String> error; @@ -85,7 +87,7 @@ interface Client { RpcFillInvoker.GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds); void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, - ResponseReceiver responseReceiver, double timeoutSeconds); + ResponseReceiver responseReceiver, double timeoutSeconds); /** Closes this connection */ void close(); @@ -93,6 +95,7 @@ interface Client { } class ProtobufResponse { + private final byte compression; private final int uncompressedSize; private final byte[] compressedPayload; @@ -114,6 +117,7 @@ interface Client { public byte[] compressedPayload() { return compressedPayload; } + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 58d7035c5e8..51290c245ac 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -38,12 +38,12 @@ public class ProtobufSerialization { private static final int INITIAL_SERIALIZATION_BUFFER_SIZE = 10 * 1024; - static byte[] serializeSearchRequest(Query query, String serverId) { - return convertFromQuery(query, serverId).toByteArray(); + static byte[] serializeSearchRequest(Query query, int hits, String serverId) { + return convertFromQuery(query, hits, serverId).toByteArray(); } - private static SearchProtocol.SearchRequest convertFromQuery(Query query, String serverId) { - var builder = SearchProtocol.SearchRequest.newBuilder().setHits(query.getHits()).setOffset(query.getOffset()) + private static SearchProtocol.SearchRequest convertFromQuery(Query query, int hits, String serverId) { + var builder = SearchProtocol.SearchRequest.newBuilder().setHits(hits).setOffset(query.getOffset()) .setTimeout((int) query.getTimeLeft()); var documentDb = query.getModel().getDocumentDb(); @@ -214,7 +214,7 @@ public class ProtobufSerialization { for (var replyHit : protobuf.getHitsList()) { LeanHit hit = (replyHit.getSortData().isEmpty()) ? new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance()) - : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getSortData().toByteArray()); + : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance(), replyHit.getSortData().toByteArray()); result.getLeanHits().add(hit); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java index 05ce6d50493..52cb2b4c061 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java @@ -22,6 +22,7 @@ import java.util.List; * @author bratseth */ class RpcClient implements Client { + private final Supervisor supervisor; public RpcClient(int transportThreads) { @@ -43,13 +44,14 @@ class RpcClient implements Client { // The current shared connection. This will be recycled when it becomes invalid. // All access to this must be synchronized - private Target target = null; + private Target target; public RpcNodeConnection(String hostname, int port, Supervisor supervisor) { this.supervisor = supervisor; this.hostname = hostname; this.port = port; description = "rpc node connection to " + hostname + ":" + port; + target = supervisor.connect(new Spec(hostname, port)); } @Override @@ -66,7 +68,7 @@ class RpcClient implements Client { @Override public void request(String rpcMethod, CompressionType compression, int uncompressedLength, byte[] compressedPayload, - ResponseReceiver responseReceiver, double timeoutSeconds) { + ResponseReceiver responseReceiver, double timeoutSeconds) { Request request = new Request(rpcMethod); request.parameters().add(new Int8Value(compression.getCode())); request.parameters().add(new Int32Value(uncompressedLength)); @@ -78,17 +80,16 @@ class RpcClient implements Client { private void invokeAsync(Request req, double timeout, RequestWaiter waiter) { // TODO: Consider replacing this by a watcher on the target synchronized(this) { // ensure we have exactly 1 valid connection across threads - if (target == null || ! target.isValid()) + if (! target.isValid()) { target = supervisor.connect(new Spec(hostname, port)); + } } target.invokeAsync(req, timeout, waiter); } @Override public void close() { - if (target != null) { - target.close(); - } + target.close(); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java index 9b661368972..0e8759f740e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcFillInvoker.java @@ -137,7 +137,7 @@ public class RpcFillInvoker extends FillInvoker { root.setString("ranking", rankProfile); } if (location != null) { - root.setString("location", location.toString()); + root.setString("location", location.backendString()); } Cursor gids = root.setArray("gids"); for (FastHit hit : hits) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 870f7aef9c5..74bc9e8bfbb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,28 +1,24 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; -import java.util.concurrent.Callable; /** * @author ollivir */ -public class RpcInvokerFactory extends InvokerFactory implements PingFactory { +public class RpcInvokerFactory extends InvokerFactory { /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */ private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries"); @@ -35,8 +31,11 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory { } @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node) { - return Optional.of(new RpcSearchInvoker(searcher, node, rpcResourcePool)); + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + Query query, + int maxHits, + Node node) { + return Optional.of(new RpcSearchInvoker(searcher, node, rpcResourcePool, maxHits)); } @Override @@ -57,12 +56,4 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory { return new RpcFillInvoker(rpcResourcePool, documentDb); } - public void release() { - rpcResourcePool.release(); - } - - @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { - return new RpcPing(node, monitor, rpcResourcePool); - } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index f2e22ba86dc..5e04f1d7a3e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -10,55 +10,63 @@ import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; import com.yahoo.search.dispatch.rpc.Client.ResponseOrError; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.result.ErrorMessage; import com.yahoo.yolean.Exceptions; -import java.util.concurrent.Callable; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; -public class RpcPing implements Callable<Pong> { +public class RpcPing implements Pinger, Client.ResponseReceiver { + + private static final Logger log = Logger.getLogger(RpcPing.class.getName()); private static final String RPC_METHOD = "vespa.searchprotocol.ping"; private static final CompressionType PING_COMPRESSION = CompressionType.NONE; private final Node node; private final RpcResourcePool resourcePool; private final ClusterMonitor<Node> clusterMonitor; + private final long pingSequenceId; + private final PongHandler pongHandler; - public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool) { + public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool, PongHandler pongHandler) { this.node = node; this.resourcePool = rpcResourcePool; this.clusterMonitor = clusterMonitor; + pingSequenceId = node.createPingSequenceId(); + this.pongHandler = pongHandler; } @Override - public Pong call() throws Exception { + public void ping() { try { - var queue = new LinkedBlockingQueue<ResponseOrError<ProtobufResponse>>(1); - - sendPing(queue); + sendPing(); + } catch (RuntimeException e) { + pongHandler.handle(new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + + ": " + Exceptions.toMessageString(e)))); + } + } - var responseOrError = queue.poll(clusterMonitor.getConfiguration().getRequestTimeout(), TimeUnit.MILLISECONDS); - if (responseOrError == null) { - return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node)); - } else if (responseOrError.error().isPresent()) { - return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get())); - } + private Pong toPong(ResponseOrError<ProtobufResponse> responseOrError) { + if (responseOrError == null) { + return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node)); + } else if (responseOrError.error().isPresent()) { + return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get())); + } + try { return decodeReply(responseOrError.response().get()); - } catch (RuntimeException e) { - return new Pong( - ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " + Exceptions.toMessageString(e))); + } catch (InvalidProtocolBufferException e) { + return new Pong(ErrorMessage.createBackendCommunicationError(e.getMessage())); } } - private void sendPing(LinkedBlockingQueue<ResponseOrError<ProtobufResponse>> queue) { + private void sendPing() { var connection = resourcePool.getConnection(node.key()); var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray(); double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping); - connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), - timeoutSeconds); + connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(),this, timeoutSeconds); } private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException { @@ -67,12 +75,23 @@ public class RpcPing implements Callable<Pong> { var reply = SearchProtocol.MonitorReply.parseFrom(responseBytes); if (reply.getDistributionKey() != node.key()) { - return new Pong(ErrorMessage.createBackendCommunicationError( - "Expected pong from node id " + node.key() + ", response is from id " + reply.getDistributionKey())); + return new Pong(ErrorMessage.createBackendCommunicationError("Expected pong from node id " + node.key() + + ", response is from id " + reply.getDistributionKey())); } else if (!reply.getOnline()) { return new Pong(ErrorMessage.createBackendCommunicationError("Node id " + node.key() + " reports being offline")); } else { - return new Pong(reply.getActiveDocs()); + return new Pong(reply.getActiveDocs(), reply.getIsBlockingWrites()); + } + } + + @Override + public void receive(ResponseOrError<ProtobufResponse> response) { + if (node.isLastReceivedPong(pingSequenceId)) { + pongHandler.handle(toPong(response)); + } else { + //TODO Reduce to debug or remove once we have enumerated what happens here. + log.info("Pong " + pingSequenceId + " from node " + node.key() + " in group " + node.group() + + " with hostname " + node.hostname() + " received too late, latest is " + node.getLastReceivedPongId()); } } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java new file mode 100644 index 00000000000..ac8f0a59c20 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java @@ -0,0 +1,18 @@ +package com.yahoo.search.dispatch.rpc; + +import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; + +public class RpcPingFactory implements PingFactory { + private final RpcResourcePool rpcResourcePool; + public RpcPingFactory(RpcResourcePool rpcResourcePool) { + this.rpcResourcePool = rpcResourcePool; + } + @Override + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { + return new RpcPing(node, monitor, rpcResourcePool, pongHandler); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index ca2a0c9bfb0..065489ef9a0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -2,6 +2,9 @@ package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentId; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.compress.Compressor.Compression; @@ -23,7 +26,7 @@ import java.util.Random; * * @author ollivir */ -public class RpcResourcePool { +public class RpcResourcePool extends AbstractComponent { /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -33,13 +36,15 @@ public class RpcResourcePool { /** Connections to the search nodes this talks to, indexed by node id ("partid") */ private final ImmutableMap<Integer, NodeConnectionPool> nodeConnectionPools; - public RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { + RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>(); nodeConnections.forEach((key, connection) -> builder.put(key, new NodeConnectionPool(Collections.singletonList(connection)))); this.nodeConnectionPools = builder.build(); } + @Inject public RpcResourcePool(DispatchConfig dispatchConfig) { + super(); var client = new RpcClient(dispatchConfig.numJrtTransportThreads()); // Create rpc node connection pools indexed by the node distribution key @@ -73,7 +78,9 @@ public class RpcResourcePool { } } - public void release() { + @Override + public void deconstruct() { + super.deconstruct(); nodeConnectionPools.values().forEach(NodeConnectionPool::release); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 59f17501c32..4c0b77207d5 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -5,7 +5,6 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; -import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; @@ -25,39 +24,57 @@ import java.util.concurrent.TimeUnit; * @author ollivir */ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseReceiver { + private static final String RPC_METHOD = "vespa.searchprotocol.search"; private final VespaBackEndSearcher searcher; private final Node node; private final RpcResourcePool resourcePool; private final BlockingQueue<Client.ResponseOrError<ProtobufResponse>> responses; + private final int maxHits; private Query query; - RpcSearchInvoker(VespaBackEndSearcher searcher, Node node, RpcResourcePool resourcePool) { + RpcSearchInvoker(VespaBackEndSearcher searcher, Node node, RpcResourcePool resourcePool, int maxHits) { super(Optional.of(node)); this.searcher = searcher; this.node = node; this.resourcePool = resourcePool; this.responses = new LinkedBlockingQueue<>(1); + this.maxHits = maxHits; } @Override - protected void sendSearchRequest(Query query) throws IOException { + protected Object sendSearchRequest(Query query, Object incomingContext) { this.query = query; Client.NodeConnection nodeConnection = resourcePool.getConnection(node.key()); if (nodeConnection == null) { responses.add(Client.ResponseOrError.fromError("Could not send search to unknown node " + node.key())); responseAvailable(); - return; + return incomingContext; } query.trace(false, 5, "Sending search request with jrt/protobuf to node with dist key ", node.key()); - var payload = ProtobufSerialization.serializeSearchRequest(query, searcher.getServerId()); + RpcContext context = getContext(incomingContext); double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; - Compressor.Compression compressionResult = resourcePool.compress(query, payload); - nodeConnection.request(RPC_METHOD, compressionResult.type(), payload.length, compressionResult.data(), this, timeoutSeconds); + nodeConnection.request(RPC_METHOD, + context.compressedPayload.type(), + context.compressedPayload.uncompressedSize(), + context.compressedPayload.data(), + this, + timeoutSeconds); + return context; + } + + private RpcContext getContext(Object incomingContext) { + if (incomingContext instanceof RpcContext) + return (RpcContext)incomingContext; + + return new RpcContext(resourcePool, query, + ProtobufSerialization.serializeSearchRequest(query, + Math.min(query.getHits(), maxHits), + searcher.getServerId())); } @Override @@ -104,4 +121,14 @@ public class RpcSearchInvoker extends SearchInvoker implements Client.ResponseRe return searcher.getName(); } + static class RpcContext { + + final Compressor.Compression compressedPayload; + + RpcContext(RpcResourcePool resourcePool, Query query, byte[] payload) { + compressedPayload = resourcePool.compress(query, payload); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 0e4e87b9a6a..ec616a18e09 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -21,6 +21,7 @@ public class Group { private final AtomicBoolean hasSufficientCoverage = new AtomicBoolean(true); private final AtomicBoolean hasFullCoverage = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); + private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false); public Group(int id, List<Node> nodes) { this.id = id; @@ -61,21 +62,16 @@ public class Group { return nodesUp; } - void aggregateActiveDocuments() { - long activeDocumentsInGroup = 0; - for (Node node : nodes) { - if (node.isWorking() == Boolean.TRUE) { - activeDocumentsInGroup += node.getActiveDocuments(); - } - } - activeDocuments.set(activeDocumentsInGroup); - + void aggregateNodeValues() { + activeDocuments.set(nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(Node::getActiveDocuments).sum()); + isBlockingWrites.set(nodes.stream().anyMatch(node -> node.isBlockingWrites())); } /** Returns the active documents on this node. If unknown, 0 is returned. */ - long getActiveDocuments() { - return this.activeDocuments.get(); - } + long getActiveDocuments() { return activeDocuments.get(); } + + /** Returns whether any node in this group is currently blocking write operations */ + public boolean isBlockingWrites() { return isBlockingWrites.get(); } public boolean isFullCoverageStatusChanged(boolean hasFullCoverageNow) { boolean previousState = hasFullCoverage.getAndSet(hasFullCoverageNow); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index 09ad715b471..8f465070de4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -21,6 +21,9 @@ public class Node { private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); + private final AtomicLong pingSequence = new AtomicLong(0); + private final AtomicLong lastPong = new AtomicLong(0); + private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false); public Node(int key, String hostname, int group) { this.key = key; @@ -28,6 +31,18 @@ public class Node { this.group = group; } + /** Give a monotonically increasing sequence number.*/ + public long createPingSequenceId() { return pingSequence.incrementAndGet(); } + /** Checks if this pong is received in line and accepted, or out of band and should be ignored..*/ + public boolean isLastReceivedPong(long pingId ) { + long last = lastPong.get(); + while ((pingId > last) && ! lastPong.compareAndSet(last, pingId)) { + last = lastPong.get(); + } + return last < pingId; + } + public long getLastReceivedPongId() { return lastPong.get(); } + /** Returns the unique and stable distribution key of this node */ public int key() { return key; } @@ -56,14 +71,14 @@ public class Node { } /** Updates the active documents on this node */ - void setActiveDocuments(long activeDocuments) { - this.activeDocuments.set(activeDocuments); - } + void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ - long getActiveDocuments() { - return activeDocuments.get(); - } + long getActiveDocuments() { return activeDocuments.get(); } + + public void setBlockingWrites(boolean isBlockingWrites) { this.isBlockingWrites.set(isBlockingWrites); } + + boolean isBlockingWrites() { return isBlockingWrites.get(); } @Override public int hashCode() { return Objects.hash(hostname, key, pathIndex, group); } @@ -82,7 +97,10 @@ public class Node { } @Override - public String toString() { return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + - " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + " activeDocs = " + activeDocuments.get(); } + public String toString() { + return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + + " statusIsKnown = " + statusIsKnown.get() + " working = " + working.get() + + " activeDocs = " + activeDocuments.get(); + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index b16fa941f68..2e07d8d61e6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -1,13 +1,11 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.searchcluster; -import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; -import java.util.concurrent.Callable; public interface PingFactory { - Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor); + Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java new file mode 100644 index 00000000000..b4a7ccbf98c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java @@ -0,0 +1,12 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch.searchcluster; + +/** + * Send a ping and ensure that the pong is propagated to the ponghandler. + * Should not wait as this should be done in parallel on all nodes. + * + * @author baldersheim + */ +public interface Pinger { + void ping(); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java new file mode 100644 index 00000000000..c0579b5d36e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java @@ -0,0 +1,12 @@ +package com.yahoo.search.dispatch.searchcluster; + +import com.yahoo.prelude.Pong; + +/** + * Handle the Pong result of a Ping. + * + * @author baldersheim + */ +public interface PongHandler { + void handle(Pong pong); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index fb55e330ebe..2f62b07ac04 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -10,7 +10,7 @@ import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; -import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.dispatch.TopKEstimator; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.LinkedHashMap; @@ -18,13 +18,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -43,10 +37,11 @@ public class SearchCluster implements NodeManager<Node> { private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; private final ImmutableList<Group> orderedGroups; - private final ClusterMonitor<Node> clusterMonitor; private final VipStatus vipStatus; - private PingFactory pingFactory; + private final PingFactory pingFactory; + private final TopKEstimator hitEstimator; private long nextLogTime = 0; + private static final double SKEW_FACTOR = 0.05; /** * A search node on this local machine having the entire corpus, which we therefore @@ -58,10 +53,12 @@ public class SearchCluster implements NodeManager<Node> { */ private final Optional<Node> localCorpusDispatchTarget; - public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, VipStatus vipStatus) { + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, + VipStatus vipStatus, PingFactory pingFactory) { this.clusterId = clusterId; this.dispatchConfig = dispatchConfig; this.vipStatus = vipStatus; + this.pingFactory = pingFactory; List<Node> nodes = toNodes(dispatchConfig); this.size = nodes.size(); @@ -82,31 +79,28 @@ public class SearchCluster implements NodeManager<Node> { for (Node node : nodes) nodesByHostBuilder.put(node.hostname(), node); this.nodesByHost = nodesByHostBuilder.build(); + hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability(), SKEW_FACTOR); this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), size, containerClusterSize, nodesByHost, groups); - - this.clusterMonitor = new ClusterMonitor<>(this); } - public void shutDown() { - clusterMonitor.shutdown(); + /* Testing only */ + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, + VipStatus vipStatus, PingFactory pingFactory) { + this(clusterId, dispatchConfig, 1, vipStatus, pingFactory); } - public void startClusterMonitoring(PingFactory pingFactory) { - this.pingFactory = pingFactory; - - for (var group : orderedGroups) { + public void addMonitoring(ClusterMonitor clusterMonitor) { + for (var group : orderedGroups()) { for (var node : group.nodes()) clusterMonitor.add(node, true); } } - ClusterMonitor<Node> clusterMonitor() { return clusterMonitor; } - private static Optional<Node> findLocalCorpusDispatchTarget(String selfHostname, int searchClusterSize, int containerClusterSize, @@ -137,18 +131,8 @@ public class SearchCluster implements NodeManager<Node> { private static ImmutableList<Node> toNodes(DispatchConfig dispatchConfig) { ImmutableList.Builder<Node> nodesBuilder = new ImmutableList.Builder<>(); - Predicate<DispatchConfig.Node> filter; - if (dispatchConfig.useLocalNode()) { - final String hostName = HostName.getLocalhost(); - filter = node -> node.host().equals(hostName); - } else { - filter = node -> true; - } - for (DispatchConfig.Node node : dispatchConfig.node()) { - if (filter.test(node)) { - nodesBuilder.add(new Node(node.key(), node.host(), node.group())); - } - } + for (DispatchConfig.Node node : dispatchConfig.node()) + nodesBuilder.add(new Node(node.key(), node.host(), node.group())); return nodesBuilder.build(); } @@ -167,8 +151,8 @@ public class SearchCluster implements NodeManager<Node> { /** Returns the n'th (zero-indexed) group in the cluster if possible */ public Optional<Group> group(int n) { - if (orderedGroups.size() > n) { - return Optional.of(orderedGroups.get(n)); + if (orderedGroups().size() > n) { + return Optional.of(orderedGroups().get(n)); } else { return Optional.empty(); } @@ -176,13 +160,13 @@ public class SearchCluster implements NodeManager<Node> { /** Returns the number of nodes per group - size()/groups.size() */ public int groupSize() { - if (groups.size() == 0) return size(); - return size() / groups.size(); + if (groups().size() == 0) return size(); + return size() / groups().size(); } public int groupsWithSufficientCoverage() { int covered = 0; - for (Group g : orderedGroups) { + for (Group g : orderedGroups()) { if (g.hasSufficientCoverage()) { covered++; } @@ -198,7 +182,7 @@ public class SearchCluster implements NodeManager<Node> { if ( localCorpusDispatchTarget.isEmpty()) return Optional.empty(); // Only use direct dispatch if the local group has sufficient coverage - Group localSearchGroup = groups.get(localCorpusDispatchTarget.get().group()); + Group localSearchGroup = groups().get(localCorpusDispatchTarget.get().group()); if ( ! localSearchGroup.hasSufficientCoverage()) return Optional.empty(); // Only use direct dispatch if the local search node is not down @@ -237,7 +221,10 @@ public class SearchCluster implements NodeManager<Node> { setInRotationOnlyIf(hasWorkingNodes()); } else if (usesLocalCorpusIn(node)) { // follow the status of this node - setInRotationOnlyIf(nodeIsWorking); + // Do not take this out of rotation if we're a combined cluster of size 1, + // as that can't be helpful, and leads to a deadlock where this node is never taken back in servic e + if (nodeIsWorking || size() > 1) + setInRotationOnlyIf(nodeIsWorking); } } @@ -257,7 +244,14 @@ public class SearchCluster implements NodeManager<Node> { vipStatus.removeFromRotation(clusterId); } - private boolean hasInformationAboutAllNodes() { + public int estimateHitsToFetch(int wantedHits, int numPartitions) { + return hitEstimator.estimateK(wantedHits, numPartitions); + } + public int estimateHitsToFetch(int wantedHits, int numPartitions, double topKProbability) { + return hitEstimator.estimateK(wantedHits, numPartitions, topKProbability); + } + + public boolean hasInformationAboutAllNodes() { return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null); } @@ -273,29 +267,41 @@ public class SearchCluster implements NodeManager<Node> { return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().group() == group.id(); } - /** Used by the cluster monitor to manage node status */ - @Override - public void ping(Node node, Executor executor) { - if (pingFactory == null) return; // not initialized yet + private static class PongCallback implements PongHandler { - FutureTask<Pong> futurePong = new FutureTask<>(pingFactory.createPinger(node, clusterMonitor)); - executor.execute(futurePong); - Pong pong = getPong(futurePong, node); - futurePong.cancel(true); + private final ClusterMonitor<Node> clusterMonitor; + private final Node node; - if (pong.badResponse()) { - clusterMonitor.failed(node, pong.getError(0)); - } else { - if (pong.activeDocuments().isPresent()) { - node.setActiveDocuments(pong.activeDocuments().get()); + PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) { + this.node = node; + this.clusterMonitor = clusterMonitor; + } + + @Override + public void handle(Pong pong) { + if (pong.badResponse()) { + clusterMonitor.failed(node, pong.error().get()); + } else { + if (pong.activeDocuments().isPresent()) { + node.setActiveDocuments(pong.activeDocuments().get()); + node.setBlockingWrites(pong.isBlockingWrites()); + } + clusterMonitor.responded(node); } - clusterMonitor.responded(node); } + + } + + /** Used by the cluster monitor to manage node status */ + @Override + public void ping(ClusterMonitor clusterMonitor, Node node, Executor executor) { + Pinger pinger = pingFactory.createPinger(node, clusterMonitor, new PongCallback(node, clusterMonitor)); + pinger.ping(); } private void pingIterationCompletedSingleGroup() { - Group group = groups.values().iterator().next(); - group.aggregateActiveDocuments(); + Group group = groups().values().iterator().next(); + group.aggregateNodeValues(); // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); @@ -305,21 +311,20 @@ public class SearchCluster implements NodeManager<Node> { } private void pingIterationCompletedMultipleGroups() { - int numGroups = orderedGroups.size(); + int numGroups = orderedGroups().size(); // Update active documents per group and use it to decide if the group should be active - long[] activeDocumentsInGroup = new long[numGroups]; long sumOfActiveDocuments = 0; for(int i = 0; i < numGroups; i++) { - Group group = orderedGroups.get(i); - group.aggregateActiveDocuments(); + Group group = orderedGroups().get(i); + group.aggregateNodeValues(); activeDocumentsInGroup[i] = group.getActiveDocuments(); sumOfActiveDocuments += activeDocumentsInGroup[i]; } boolean anyGroupsSufficientCoverage = false; for (int i = 0; i < numGroups; i++) { - Group group = orderedGroups.get(i); + Group group = orderedGroups().get(i); long activeDocuments = activeDocumentsInGroup[i]; long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), activeDocuments, averageDocumentsInOtherGroups); @@ -336,7 +341,7 @@ public class SearchCluster implements NodeManager<Node> { */ @Override public void pingIterationCompleted() { - int numGroups = orderedGroups.size(); + int numGroups = orderedGroups().size(); if (numGroups == 1) { pingIterationCompletedSingleGroup(); } else { @@ -363,25 +368,11 @@ public class SearchCluster implements NodeManager<Node> { return workingNodes + nodesAllowedDown >= nodesInGroup; } - private Pong getPong(FutureTask<Pong> futurePong, Node node) { - try { - return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - log.log(Level.WARNING, "Exception pinging " + node, e); - return new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + node)); - } catch (ExecutionException e) { - log.log(Level.WARNING, "Exception pinging " + node, e); - return new Pong(ErrorMessage.createUnspecifiedError("Execution was interrupted: " + node)); - } catch (TimeoutException e) { - return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Ping thread timed out")); - } - } - /** * Calculate whether a subset of nodes in a group has enough coverage */ public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) { - if (orderedGroups.size() == 1) { + if (orderedGroups().size() == 1) { boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); return sufficient; } @@ -390,14 +381,14 @@ public class SearchCluster implements NodeManager<Node> { return false; } int groupId = knownGroupId.getAsInt(); - Group group = groups.get(groupId); + Group group = groups().get(groupId); if (group == null) { return false; } int nodesInGroup = group.nodes().size(); long sumOfActiveDocuments = 0; int otherGroups = 0; - for (Group g : orderedGroups) { + for (Group g : orderedGroups()) { if (g.id() != groupId) { sumOfActiveDocuments += g.getActiveDocuments(); otherGroups++; @@ -412,6 +403,7 @@ public class SearchCluster implements NodeManager<Node> { } private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { + if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. boolean changed = group.isFullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { nextLogTime = System.currentTimeMillis() + 30 * 1000; diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java index 5f1cfccf549..6243dc694c2 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java @@ -39,8 +39,8 @@ class FederationResult { } /** - * Wait on each target for that targets timeout - * On the worst case this is the same as waiting for the max target timeout, + * Wait on each target for that targets timeout. + * In the worst case this is the same as waiting for the max target timeout, * in the average case it may be much better because lower timeout sources do not get to * drive the timeout above their own timeout value. * When this completes, results can be accessed from the TargetResults with no blocking diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index c5573dc8fee..60c5d42c531 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -78,7 +78,6 @@ public class FederationSearcher extends ForkingSearcher { /** The name of the query property containing the source name added to the query to each source by this */ public final static CompoundName SOURCENAME = new CompoundName("sourceName"); public final static CompoundName PROVIDERNAME = new CompoundName("providerName"); - /** Logging field name constants */ public static final String LOG_COUNT_PREFIX = "count_"; @@ -110,7 +109,7 @@ public class FederationSearcher extends ForkingSearcher { // for testing public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) { - this(searchChainResolver, false, PropagateSourceProperties.ALL, null); + this(searchChainResolver, false, PropagateSourceProperties.EVERY, null); } private FederationSearcher(SearchChainResolver searchChainResolver, @@ -271,13 +270,14 @@ public class FederationSearcher extends ForkingSearcher { outgoing.setTimeout(timeout); switch (propagateSourceProperties) { - case ALL: - propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, - Query.nativeProperties); + case EVERY: + propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, null); + break; + case NATIVE: case ALL: + propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, Query.nativeProperties); break; case OFFSET_HITS: - propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, - queryAndHits); + propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, queryAndHits); break; } @@ -290,10 +290,21 @@ public class FederationSearcher extends ForkingSearcher { private void propagatePerSourceQueryProperties(Query original, Query outgoing, Window window, String sourceName, String providerName, List<CompoundName> queryProperties) { - for (CompoundName key : queryProperties) { - Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key)); - if (value != null) - outgoing.properties().set(key, value); + if (queryProperties == null) { + outgoing.setHits(window.hits); + outgoing.setOffset(window.offset); + original.properties().listProperties(CompoundName.fromComponents("provider", providerName)).forEach((k, v) -> + outgoing.properties().set(k, v)); + original.properties().listProperties(CompoundName.fromComponents("source", sourceName)).forEach((k, v) -> + outgoing.properties().set(k, v)); + } + else { + for (CompoundName key : queryProperties) { + Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key)); + if (value != null) + outgoing.properties().set(key, value); + if (value != null) System.out.println("Setting " + key + " = " + value); + } } } @@ -321,7 +332,7 @@ public class FederationSearcher extends ForkingSearcher { private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { String message = String.join(" ", getMessagesSet(unresolvedSearchChainExceptions)) + - " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.'; + " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.'; return ErrorMessage.createInvalidQueryParameter(message); } @@ -343,7 +354,7 @@ public class FederationSearcher extends ForkingSearcher { } private void warnIfUnresolvedSearchChains(List<UnresolvedSearchChainException> missingTargets, - HitGroup errorHitGroup) { + HitGroup errorHitGroup) { if (!missingTargets.isEmpty()) { errorHitGroup.addError(missingSearchChainsErrorMessage(missingTargets)); } @@ -481,9 +492,9 @@ public class FederationSearcher extends ForkingSearcher { * TODO This is probably a dirty hack for bug 4711376. There are probably better ways. * But I will leave that to trd-processing@ * - * @param group The merging hitgroup to be updated if necessary - * @param orderer The per provider hit orderer. - * @return The hitorderer chosen + * @param group the merging hitgroup to be updated if necessary + * @param orderer the per provider hit orderer + * @return he hitorderer chosen */ private HitOrderer dirtyCopyIfModifiedOrderer(HitGroup group, HitOrderer orderer) { if (orderer != null) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index 8280917ca2f..3abfa3a6531 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.grouping; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -110,7 +110,7 @@ public class UniqueGroupingSearcher extends Searcher { if (null == root) { String msg = "Result group not found for deduping grouping request, returning empty result."; query.trace(msg, 3); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); throw new IllegalStateException("Failed to produce deduped result set."); } result.hits().remove(root.getId().toString()); // hide our tracks diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 7c2e774f68b..d2d89e41879 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.QueryCanonicalizer; @@ -225,9 +225,9 @@ public class GroupingExecutor extends Searcher { if (passList.isEmpty()) { throw new RuntimeException("No grouping request for pass " + pass + ", bug!"); } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : passList) { - log.log(LogLevel.DEBUG, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); } } Item passRoot; @@ -263,9 +263,9 @@ public class GroupingExecutor extends Searcher { ret = passResult; } } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : groupingMap.values()) { - log.log(LogLevel.DEBUG, "Result Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Result Grouping(" + grouping.getId() + "): " + grouping); } } return ret; diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java index 3602d21f7d8..d636d3bc925 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java @@ -23,6 +23,7 @@ import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.query.context.QueryContext; +import com.yahoo.yolean.trace.TraceNode; /** * Wrap the result of a query as an HTTP response. @@ -36,8 +37,13 @@ public class HttpSearchResponse extends ExtendedResponse { private final Renderer<Result> rendererCopy; private final Timing timing; private final HitCounts hitCounts; + private final TraceNode trace; public HttpSearchResponse(int status, Result result, Query query, Renderer renderer) { + this(status, result, query, renderer, null); + } + + HttpSearchResponse(int status, Result result, Query query, Renderer renderer, TraceNode trace) { super(status); this.query = query; this.result = result; @@ -45,6 +51,7 @@ public class HttpSearchResponse extends ExtendedResponse { this.timing = SearchResponse.createTiming(query, result); this.hitCounts = SearchResponse.createHitCounts(query, result); + this.trace = trace; populateHeaders(headers(), result.getHeaders(false)); } @@ -107,6 +114,9 @@ public class HttpSearchResponse extends ExtendedResponse { @Override public void populateAccessLogEntry(final AccessLogEntry accessLogEntry) { super.populateAccessLogEntry(accessLogEntry); + if (trace != null) { + accessLogEntry.setTrace(trace); + } populateAccessLogEntry(accessLogEntry, getHitCounts()); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 0981c6e8dad..c658d404adb 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -18,18 +18,19 @@ import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; import com.yahoo.prelude.query.QueryException; import com.yahoo.prelude.query.parser.ParseException; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.context.QueryContext; import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.search.searchchain.ExecutionFactory; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -49,6 +50,7 @@ import com.yahoo.statistics.Handle; import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Value; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.yolean.trace.TraceNode; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -58,6 +60,7 @@ import java.util.Optional; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -98,6 +101,8 @@ public class SearchHandler extends LoggingRequestHandler { private final ExecutionFactory executionFactory; + private final AtomicLong numRequestsLeftToTrace; + private final class MeanConnections implements Callback { @Override @@ -116,6 +121,28 @@ public class SearchHandler extends LoggingRequestHandler { Metric metric, Executor executor, AccessLog accessLog, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig containerHttpConfig, + ExecutionFactory executionFactory) { + this(statistics, + metric, + executor, + accessLog, + queryProfileRegistry, + executionFactory, + containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), + containerHttpConfig.hostResponseHeaderKey().equals("") ? + Optional.empty() : Optional.of(containerHttpConfig.hostResponseHeaderKey())); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, + Executor executor, + AccessLog accessLog, QueryProfilesConfig queryProfileConfig, ContainerHttpConfig containerHttpConfig, ExecutionFactory executionFactory) { @@ -125,8 +152,9 @@ public class SearchHandler extends LoggingRequestHandler { accessLog, QueryProfileConfigurer.createFromConfig(queryProfileConfig).compile(), executionFactory, + containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? - Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); + Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); } public SearchHandler(Statistics statistics, @@ -136,8 +164,19 @@ public class SearchHandler extends LoggingRequestHandler { CompiledQueryProfileRegistry queryProfileRegistry, ExecutionFactory executionFactory, Optional<String> hostResponseHeaderKey) { + this(statistics, metric, executor, accessLog, queryProfileRegistry, executionFactory, 0, hostResponseHeaderKey); + } + + private SearchHandler(Statistics statistics, + Metric metric, + Executor executor, + AccessLog accessLog, + CompiledQueryProfileRegistry queryProfileRegistry, + ExecutionFactory executionFactory, + long numQueriesToTraceOnDebugAfterStartup, + Optional<String> hostResponseHeaderKey) { super(executor, accessLog, metric, true); - log.log(LogLevel.DEBUG, "SearchHandler.init " + System.identityHashCode(this)); + log.log(Level.FINE, "SearchHandler.init " + System.identityHashCode(this)); this.queryProfileRegistry = queryProfileRegistry; this.executionFactory = executionFactory; @@ -150,6 +189,7 @@ public class SearchHandler extends LoggingRequestHandler { .setCallback(new MeanConnections())); this.hostResponseHeaderKey = hostResponseHeaderKey; + this.numRequestsLeftToTrace = new AtomicLong(numQueriesToTraceOnDebugAfterStartup); } /** @deprecated use the other constructor */ @@ -215,7 +255,6 @@ public class SearchHandler extends LoggingRequestHandler { } - @SuppressWarnings("unchecked") private HttpResponse errorResponse(HttpRequest request, ErrorMessage errorMessage) { Query query = new Query(); Result result = new Result(query, errorMessage); @@ -281,7 +320,8 @@ public class SearchHandler extends LoggingRequestHandler { // Transform result to response Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), - result, query, renderer); + result, query, renderer, + extractTraceNode(query)); if (hostResponseHeaderKey.isPresent()) response.headers().add(hostResponseHeaderKey.get(), selfHostname); @@ -292,6 +332,19 @@ public class SearchHandler extends LoggingRequestHandler { return response; } + private static TraceNode extractTraceNode(Query query) { + if (log.isLoggable(Level.FINE)) { + QueryContext queryContext = query.getContext(false); + if (queryContext != null) { + Execution.Trace trace = queryContext.getTrace(); + if (trace != null) { + return trace.traceNode(); + } + } + } + return null; + } + private static int getErrors(Result result) { return result.hits().getErrorHit() == null ? 0 : 1; } @@ -330,7 +383,13 @@ public class SearchHandler extends LoggingRequestHandler { Execution execution = executionFactory.newExecution(searchChain); query.getModel().setExecution(execution); - execution.trace().setForceTimestamps(query.properties().getBoolean(FORCE_TIMESTAMPS, false)); + if (log.isLoggable(Level.FINE) && (numRequestsLeftToTrace.getAndDecrement() > 0)) { + query.setTraceLevel(Math.max(1, query.getTraceLevel())); + execution.trace().setForceTimestamps(true); + + } else { + execution.trace().setForceTimestamps(query.properties().getBoolean(FORCE_TIMESTAMPS, false)); + } if (query.properties().getBoolean(DETAILED_TIMING_LOGGING, false)) { // check and set (instead of set directly) to avoid overwriting stuff from prepareForBreakdownAnalysis() execution.context().setDetailedDiagnostics(true); @@ -380,7 +439,7 @@ public class SearchHandler extends LoggingRequestHandler { if (searchConnections != null) { connectionStatistics(); } else { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "searchConnections is a null reference, probably a known race condition during startup.", new IllegalStateException("searchConnections reference is null.")); } @@ -389,7 +448,7 @@ public class SearchHandler extends LoggingRequestHandler { } catch (ParseException e) { ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); + log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); } catch (IllegalArgumentException e) { if ("Comparison method violates its general contract!".equals(e.getMessage())) { @@ -401,7 +460,7 @@ public class SearchHandler extends LoggingRequestHandler { else { ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, () -> error.getDetailedMessage()); + log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); } } catch (LinkageError | StackOverflowError e) { @@ -448,10 +507,10 @@ public class SearchHandler extends LoggingRequestHandler { private void log(String request, Query query, Throwable e) { // Attempted workaround for missing stack traces if (e.getStackTrace().length == 0) { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "], received exception with no context", e); } else { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + " [" + request + "]", e); + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "]", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index 13196488e98..e147b14071a 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -134,9 +134,9 @@ public class Presentation implements Cloneable { @Override public boolean equals(Object o) { - if (o == null || !(o instanceof Presentation)) return false; + if ( ! (o instanceof Presentation)) return false; Presentation p = (Presentation) o; - return QueryHelper.equals(bolding,p.bolding) && QueryHelper.equals(summary,p.summary); + return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/Properties.java b/container-search/src/main/java/com/yahoo/search/query/Properties.java index a1a70b4c3ba..a0cd4137e9f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Properties.java +++ b/container-search/src/main/java/com/yahoo/search/query/Properties.java @@ -30,8 +30,9 @@ public abstract class Properties extends com.yahoo.processing.request.Properties return (Properties)super.clone(); } - /** The query owning this property object. - * Only guaranteed to work if this instance is accessible as query.properties() + /** + * Returns the query owning this property object. + * Only guaranteed to work if this instance is accessible as query.properties() */ public Query getParentQuery() { if (chained() == null) { @@ -48,4 +49,5 @@ public abstract class Properties extends com.yahoo.processing.request.Properties if (chained() != null) chained().setParentQuery(query); } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 7444c94f491..830a3f4ef81 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -47,7 +47,7 @@ public class Ranking implements Cloneable { public static final String PROPERTIES = "properties"; static { - argumentType =new QueryProfileType(RANKING); + argumentType = new QueryProfileType(RANKING); argumentType.setStrict(true); argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(LOCATION, "string", "location")); @@ -63,7 +63,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature")); argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty")); argumentType.freeze(); - argumentTypeName=new CompoundName(argumentType.getId().getName()); + argumentTypeName = new CompoundName(argumentType.getId().getName()); } public static QueryProfileType getArgumentType() { return argumentType; } diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index cb662dcd671..65ffd29efe0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -57,12 +57,13 @@ public class Select implements Cloneable { } public Select(String where, String grouping, Query query) { - this(where, grouping, query, Collections.emptyList()); + this(where, grouping, null, query, Collections.emptyList()); } - private Select(String where, String grouping, Query query, List<GroupingRequest> groupingRequests) { + private Select(String where, String grouping, String groupingExpressionString, Query query, List<GroupingRequest> groupingRequests) { this.where = Objects.requireNonNull(where, "A Select must have a where string (possibly the empty string)"); this.grouping = Objects.requireNonNull(grouping, "A Select must have a select string (possibly the empty string)"); + this.groupingExpressionString = groupingExpressionString; this.parent = Objects.requireNonNull(query, "A Select must have a parent query"); this.groupingRequests = deepCopy(groupingRequests, this); } @@ -136,11 +137,11 @@ public class Select implements Cloneable { @Override public Object clone() { - return new Select(where, grouping, parent, groupingRequests); + return new Select(where, grouping, groupingExpressionString, parent, groupingRequests); } public Select cloneFor(Query parent) { - return new Select(where, grouping, parent, groupingRequests); + return new Select(where, grouping, groupingExpressionString, parent, groupingRequests); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index c654edda6f5..0d9acea7643 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -3,9 +3,12 @@ package com.yahoo.search.query; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.process.Normalizer; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.CompositeItem; @@ -15,7 +18,9 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.OrItem; @@ -45,7 +50,8 @@ import com.yahoo.search.yql.VespaGroupingStep; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.slime.Type; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -60,6 +66,65 @@ import static com.yahoo.slime.Type.LONG; import static com.yahoo.slime.Type.OBJECT; import static com.yahoo.slime.Type.STRING; +import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; +import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; +import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; +import static com.yahoo.search.yql.YqlParser.ANNOTATIONS; +import static com.yahoo.search.yql.YqlParser.APPROXIMATE; +import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.BOUNDS; +import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN; +import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; +import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; +import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; +import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.DISTANCE; +import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; +import static com.yahoo.search.yql.YqlParser.END_ANCHOR; +import static com.yahoo.search.yql.YqlParser.EQUIV; +import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; +import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; +import static com.yahoo.search.yql.YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; +import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; +import static com.yahoo.search.yql.YqlParser.LABEL; +import static com.yahoo.search.yql.YqlParser.NEAR; +import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; +import static com.yahoo.search.yql.YqlParser.NFKC; +import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; +import static com.yahoo.search.yql.YqlParser.ONEAR; +import static com.yahoo.search.yql.YqlParser.ORIGIN; +import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; +import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; +import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; +import static com.yahoo.search.yql.YqlParser.PHRASE; +import static com.yahoo.search.yql.YqlParser.PREDICATE; +import static com.yahoo.search.yql.YqlParser.PREFIX; +import static com.yahoo.search.yql.YqlParser.RANGE; +import static com.yahoo.search.yql.YqlParser.RANK; +import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; +import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; +import static com.yahoo.search.yql.YqlParser.START_ANCHOR; +import static com.yahoo.search.yql.YqlParser.STEM; +import static com.yahoo.search.yql.YqlParser.SUBSTRING; +import static com.yahoo.search.yql.YqlParser.SUFFIX; +import static com.yahoo.search.yql.YqlParser.TARGET_HITS; +import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; +import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; +import static com.yahoo.search.yql.YqlParser.URI; +import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; +import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE; +import static com.yahoo.search.yql.YqlParser.WAND; +import static com.yahoo.search.yql.YqlParser.WEAK_AND; +import static com.yahoo.search.yql.YqlParser.WEIGHT; +import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; + /** * The Select query language. * @@ -69,6 +134,14 @@ import static com.yahoo.slime.Type.STRING; */ public class SelectParser implements Parser { + private static final String AND = "and"; + private static final String AND_NOT = "and_not"; + private static final String CALL = "call"; + private static final String CONTAINS = "contains"; + private static final String EQ = "equals"; + private static final String MATCHES = "matches"; + private static final String OR = "or"; + Parsable query; private final IndexFacts indexFacts; private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap(); @@ -76,61 +149,7 @@ public class SelectParser implements Parser { private final Normalizer normalizer; private IndexFacts.Session indexFactsSession; - // YQL parameters and functions - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; - private static final String ORIGIN_LENGTH = "length"; - private static final String ORIGIN_OFFSET = "offset"; - private static final String ORIGIN = "origin"; - private static final String ORIGIN_ORIGINAL = "original"; - private static final String CONNECTION_ID = "id"; - private static final String CONNECTION_WEIGHT = "weight"; - private static final String CONNECTIVITY = "connectivity"; - private static final String ANNOTATIONS = "annotations"; - private static final String NFKC = "nfkc"; - private static final String USER_INPUT_LANGUAGE = "language"; - private static final String ACCENT_DROP = "accentDrop"; - private static final String ALTERNATIVES = "alternatives"; - private static final String AND_SEGMENTING = "andSegmenting"; - private static final String DISTANCE = "distance"; - private static final String DOT_PRODUCT = "dotProduct"; - private static final String EQUIV = "equiv"; - private static final String FILTER = "filter"; - private static final String HIT_LIMIT = "hitLimit"; - private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - private static final String LABEL = "label"; - private static final String NEAR = "near"; - private static final String NORMALIZE_CASE = "normalizeCase"; - private static final String ONEAR = "onear"; - private static final String PHRASE = "phrase"; - private static final String PREDICATE = "predicate"; - private static final String PREFIX = "prefix"; - private static final String RANKED = "ranked"; - private static final String RANK = "rank"; - private static final String SAME_ELEMENT = "sameElement"; - private static final String SCORE_THRESHOLD = "scoreThreshold"; - private static final String SIGNIFICANCE = "significance"; - private static final String STEM = "stem"; - private static final String SUBSTRING = "substring"; - private static final String SUFFIX = "suffix"; - private static final String TARGET_NUM_HITS = "targetNumHits"; - private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - private static final String UNIQUE_ID = "id"; - private static final String USE_POSITION_DATA = "usePositionData"; - private static final String WAND = "wand"; - private static final String WEAK_AND = "weakAnd"; - private static final String WEIGHTED_SET = "weightedSet"; - private static final String WEIGHT = "weight"; - private static final String AND = "and"; - private static final String AND_NOT = "and_not"; - private static final String OR = "or"; - private static final String EQ = "equals"; - private static final String RANGE = "range"; - private static final String CONTAINS = "contains"; - private static final String MATCHES = "matches"; - private static final String CALL = "call"; - private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); + private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, PREDICATE, RANK, WEAK_AND); public SelectParser(ParserEnvironment environment) { indexFacts = environment.getIndexFacts(); @@ -148,7 +167,7 @@ public class SelectParser implements Parser { } private QueryTree buildTree() { - Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get(); + Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString()).get(); if (inspector.field("error_message").valid()) { throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); @@ -159,7 +178,7 @@ public class SelectParser implements Parser { return new QueryTree(root); } - private Item walkJson(Inspector inspector){ + private Item walkJson(Inspector inspector) { Item[] item = {null}; inspector.traverse((ObjectTraverser) (key, value) -> { String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; @@ -197,8 +216,8 @@ public class SelectParser implements Parser { public List<VespaGroupingStep> getGroupingSteps(String grouping){ List<VespaGroupingStep> groupingSteps = new ArrayList<>(); - List<String> groupingOperations = getOperations(grouping); - for (String groupingString : groupingOperations){ + List<String> groupingOperations = toGroupingRequests(grouping); + for (String groupingString : groupingOperations) { GroupingOperation groupingOperation = GroupingOperation.fromString(groupingString); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); groupingSteps.add(groupingStep); @@ -206,24 +225,51 @@ public class SelectParser implements Parser { return groupingSteps; } - private List<String> getOperations(String grouping) { - List<String> operations = new ArrayList<>(); - Inspector inspector = SlimeUtils.jsonToSlime(grouping.getBytes()).get(); - if (inspector.field("error_message").valid()){ + /** Translates a list of grouping requests on JSON form to a list in the grouping language form */ + private List<String> toGroupingRequests(String groupingJson) { + Inspector inspector = SlimeUtils.jsonToSlime(groupingJson).get(); + if (inspector.field("error_message").valid()) { throw new QueryException("Illegal query: " + inspector.field("error_message").asString() + " at: '" + new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8) + "'"); } - inspector.traverse( (ArrayTraverser) (key, value) -> { - String groupingString = value.toString(); - groupingString = groupingString.replace(" ", "").replace("\"", "").replace("\'", "").replace(":{", "(").replace(":", "(").replace("}", ")").replace(",", ")"); - groupingString = groupingString.substring(1, groupingString.length()); - operations.add(groupingString); - }); - + List<String> operations = new ArrayList<>(); + inspector.traverse((ArrayTraverser) (__, item) -> operations.add(toGroupingRequest(item))); return operations; } + private String toGroupingRequest(Inspector groupingJson) { + StringBuilder b = new StringBuilder(); + toGroupingRequest(groupingJson, b); + return b.toString(); + } + + private void toGroupingRequest(Inspector groupingJson, StringBuilder b) { + switch (groupingJson.type()) { + case ARRAY: + groupingJson.traverse((ArrayTraverser) (index, item) -> { + toGroupingRequest(item, b); + if (index + 1 < groupingJson.entries()) + b.append(","); + }); + break; + case OBJECT: + groupingJson.traverse((ObjectTraverser) (name, object) -> { + b.append(name); + b.append("("); + toGroupingRequest(object, b); + b.append(") "); + }); + break; + case STRING: + b.append(groupingJson.asString()); + break; + default: + b.append(groupingJson.toString()); + break; + } + } + private Item buildFunctionCall(String key, Inspector value) { switch (key) { case WAND: @@ -232,6 +278,10 @@ public class SelectParser implements Parser { return buildWeightedSet(key, value); case DOT_PRODUCT: return buildDotProduct(key, value); + case GEO_LOCATION: + return buildGeoLocation(key, value); + case NEAREST_NEIGHBOR: + return buildNearestNeighbor(key, value); case PREDICATE: return buildPredicate(key, value); case RANK: @@ -239,7 +289,7 @@ public class SelectParser implements Parser { case WEAK_AND: return buildWeakAnd(key, value); default: - throw newUnexpectedArgumentException(key, DOT_PRODUCT, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); } } @@ -250,7 +300,7 @@ public class SelectParser implements Parser { }); } else if (inspector.type() == OBJECT){ - if (inspector.field("children").valid()){ + if (inspector.field("children").valid()) { inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { item.addItem(walkJson(new_value)); }); @@ -259,22 +309,22 @@ public class SelectParser implements Parser { } } - private Inspector getChildren(Inspector inspector){ + private Inspector getChildren(Inspector inspector) { if (inspector.type() == ARRAY){ return inspector; } else if (inspector.type() == OBJECT){ - if (inspector.field("children").valid()){ + if (inspector.field("children").valid()) { return inspector.field("children"); } - if (inspector.field(1).valid()){ + if (inspector.field(1).valid()) { return inspector.field(1); } } return null; } - private HashMap<Integer, Inspector> childMap(Inspector inspector){ + private HashMap<Integer, Inspector> childMap(Inspector inspector) { HashMap<Integer, Inspector> children = new HashMap<>(); if (inspector.type() == ARRAY){ inspector.traverse((ArrayTraverser) (index, new_value) -> { @@ -291,14 +341,14 @@ public class SelectParser implements Parser { return children; } - private Inspector getAnnotations(Inspector inspector){ + private Inspector getAnnotations(Inspector inspector) { if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ return inspector.field("attributes"); } return null; } - private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){ + private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation) { HashMap<String, Inspector> attributes = new HashMap<>(); if (annotation.type() == OBJECT){ annotation.traverse((ObjectTraverser) (index, new_value) -> { @@ -308,7 +358,7 @@ public class SelectParser implements Parser { return attributes; } - private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){ + private HashMap<String, Inspector> getAnnotationMap(Inspector inspector) { HashMap<String, Inspector> attributes = new HashMap<>(); if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> { @@ -376,6 +426,79 @@ public class SelectParser implements Parser { return orItem; } + private Item buildGeoLocation(String key, Inspector value) { + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 4, "Expected 4 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + var arg1 = children.get(1); + var arg2 = children.get(2); + var arg3 = children.get(3); + var loc = new Location(); + if (arg3.type() != Type.STRING) { + throw new IllegalArgumentException("Invalid geoLocation radius type "+arg3.type()+" for "+arg3); + } + double radius = DistanceParser.parse(arg3.asString()); + if (arg1.type() == Type.STRING && arg2.type() == Type.STRING) { + var c1input = children.get(1).asString(); + var c2input = children.get(2).asString(); + var coord_1 = ParsedDegree.fromString(c1input, true, false); + var coord_2 = ParsedDegree.fromString(c2input, false, true); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+c1input+"' and '"+c2input+"'"); + } + } else if (arg1.type() == Type.DOUBLE && arg2.type() == Type.DOUBLE) { + loc.setGeoCircle(arg1.asDouble(), arg2.asDouble(), radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinate types "+arg1.type()+" and "+arg2.type()); + } + var item = new GeoLocationItem(loc, field); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + + private Item buildNearestNeighbor(String key, Inspector value) { + + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + String field = children.get(0).asString(); + String property = children.get(1).asString(); + NearestNeighborItem item = new NearestNeighborItem(field, property); + Inspector annotations = getAnnotations(value); + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (TARGET_HITS.equals(annotation_name)){ + item.setTargetNumHits((int)(annotation_value.asDouble())); + } + if (TARGET_NUM_HITS.equals(annotation_name)){ + item.setTargetNumHits((int)(annotation_value.asDouble())); + } + if (HNSW_EXPLORE_ADDITIONAL_HITS.equals(annotation_name)) { + int hnswExploreAdditionalHits = (int)(annotation_value.asDouble()); + item.setHnswExploreAdditionalHits(hnswExploreAdditionalHits); + } + if (APPROXIMATE.equals(annotation_name)) { + boolean allowApproximate = annotation_value.asBool(); + item.setAllowApproximate(allowApproximate); + } + if (LABEL.equals(annotation_name)) { + item.setLabel(annotation_value.asString()); + } + }); + } + return item; + } + private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); addItemsFromInspector(weakAnd, value); @@ -383,6 +506,9 @@ public class SelectParser implements Parser { if (annotations != null){ annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (TARGET_HITS.equals(annotation_name)){ + weakAnd.setN((int)(annotation_value.asDouble())); + } if (TARGET_NUM_HITS.equals(annotation_name)){ weakAnd.setN((int)(annotation_value.asDouble())); } @@ -635,7 +761,10 @@ public class SelectParser implements Parser { HashMap<Integer, Inspector> children = childMap(value); Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); - Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + Integer target_num_hits= getIntegerAnnotation(TARGET_HITS, annotations, null); + if (target_num_hits == null) { + target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + } WandItem out = new WandItem(children.get(0).asString(), target_num_hits); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java deleted file mode 100644 index eda8bf78b68..00000000000 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllReferencesQueryProfileVisitor.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.query.profile; - -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileFieldType; -import com.yahoo.search.query.profile.types.QueryProfileType; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * @author bratseth - */ -final class AllReferencesQueryProfileVisitor extends PrefixQueryProfileVisitor { - - /** A map of query profile types */ - private Set<CompoundName> references = new HashSet<>(); - - public AllReferencesQueryProfileVisitor(CompoundName prefix) { - super(prefix); - } - - @Override - public void onValue(String name, Object value, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) {} - - @Override - public void onQueryProfileInsidePrefix(QueryProfile profile, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - references.add(currentPrefix); - } - - /** Returns the values resulting from this visiting */ - public Set<CompoundName> getResult() { return references; } - - /** Returns false - we are not done until we have seen all */ - public boolean isDone() { return false; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java deleted file mode 100644 index 6bf17d70c70..00000000000 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllTypesQueryProfileVisitor.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.query.profile; - -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileFieldType; -import com.yahoo.search.query.profile.types.QueryProfileType; - -import java.util.HashMap; -import java.util.Map; - -/** - * @author bratseth - */ -final class AllTypesQueryProfileVisitor extends PrefixQueryProfileVisitor { - - /** A map of query profile types */ - private Map<CompoundName, QueryProfileType> types = new HashMap<>(); - - public AllTypesQueryProfileVisitor(CompoundName prefix) { - super(prefix); - } - - @Override - public void onValue(String name, Object value, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) {} - - - @Override - public void onQueryProfileInsidePrefix(QueryProfile profile, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - if (profile.getType() != null) - addReachableTypes(currentPrefix, profile.getType()); - } - - private void addReachableTypes(CompoundName name, QueryProfileType type) { - types.putIfAbsent(name, type); // Types visited earlier has precedence: profile.type overrides profile.inherited.type - for (FieldDescription fieldDescription : type.fields().values()) { - if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) continue; - QueryProfileFieldType fieldType = (QueryProfileFieldType)fieldDescription.getType(); - if (fieldType.getQueryProfileType() !=null) { - addReachableTypes(name.append(fieldDescription.getName()), fieldType.getQueryProfileType()); - } - } - } - - /** Returns the values resulting from this visiting */ - public Map<CompoundName, QueryProfileType> getResult() { return types; } - - /** Returns false - we are not done until we have seen all */ - public boolean isDone() { return false; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java deleted file mode 100644 index 4bae6823500..00000000000 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllUnoverridableQueryProfileVisitor.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.query.profile; - -import com.yahoo.processing.request.CompoundName; - -import java.util.HashSet; -import java.util.Set; - -/** - * @author bratseth - */ -final class AllUnoverridableQueryProfileVisitor extends PrefixQueryProfileVisitor { - - /** A map of query profile types */ - private Set<CompoundName> unoverridables = new HashSet<>(); - - public AllUnoverridableQueryProfileVisitor(CompoundName prefix) { - super(prefix); - } - - @Override - public void onValue(String name, Object value, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - addUnoverridable(name, currentPrefix.append(name), binding, owner); - } - - @Override - public void onQueryProfileInsidePrefix(QueryProfile profile, - DimensionBinding binding, - QueryProfile owner, - DimensionValues variant) { - addUnoverridable(currentPrefix.last(), currentPrefix, binding, owner); - } - - private void addUnoverridable(String localName, - CompoundName fullName, - DimensionBinding binding, - QueryProfile owner) { - if (owner == null) return; - - Boolean isOverridable = owner.isLocalOverridable(localName, binding); - if (isOverridable != null && ! isOverridable) - unoverridables.add(fullName); - } - - /** Returns the values resulting from this visiting */ - public Set<CompoundName> getResult() { return unoverridables; } - - /** Returns false - we are not done until we have seen all */ - public boolean isDone() { return false; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java index f27500085e1..3c336c80d37 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/AllValuesQueryProfileVisitor.java @@ -3,6 +3,9 @@ package com.yahoo.search.query.profile; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.compiled.ValueWithSource; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; import java.util.Collections; import java.util.HashMap; @@ -26,7 +29,7 @@ final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor { DimensionBinding binding, QueryProfile owner, DimensionValues variant) { - putValue(localName, value, owner, variant); + putValue(localName, value, null, owner, variant, binding); } @Override @@ -34,17 +37,29 @@ final class AllValuesQueryProfileVisitor extends PrefixQueryProfileVisitor { DimensionBinding binding, QueryProfile owner, DimensionValues variant) { - putValue("", profile.getValue(), owner, variant); + putValue("", profile.getValue(), profile, owner, variant, binding); } - private void putValue(String key, Object value, QueryProfile owner, DimensionValues variant) { - if (value == null) return; + private void putValue(String key, + Object value, + QueryProfile profile, + QueryProfile owner, + DimensionValues variant, + DimensionBinding binding) { CompoundName fullName = currentPrefix.append(key); - if (fullName.isEmpty()) return; // Avoid putting a non-leaf (subtree) root in the list - if (values.containsKey(fullName.toString())) return; // The first value encountered has priority + + ValueWithSource existing = values.get(fullName.toString()); + + // The first value encountered has priority and values have priority over profiles + if (existing != null && (existing.value() != null || value == null)) return; + + Boolean isOverridable = owner != null ? owner.isLocalOverridable(key, binding) : null; values.put(fullName.toString(), new ValueWithSource(value, owner == null ? "anonymous" : owner.getSource(), + isOverridable != null && ! isOverridable, + profile != null, + profile == null ? null : profile.getType(), variant)); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java index 99f1e26b221..11864e60cec 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java @@ -135,8 +135,8 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple @Override public List<String> getDimensions() { - List<String> dimensions=super.getDimensions(); - if (dimensions!=null) return dimensions; + List<String> dimensions = super.getDimensions(); + if (dimensions != null) return dimensions; return backingProfile.getDimensions(); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java index 50bd2c58da8..e0edf9f9894 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java @@ -3,7 +3,6 @@ package com.yahoo.search.query.profile; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -22,7 +21,7 @@ public class DimensionBinding { private DimensionValues values; /** The binding from those dimensions to values, and possibly other values */ - private Map<String, String> context; // TODO: This is not needed any more + private Map<String, String> context; public static final DimensionBinding nullBinding = new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null); @@ -34,13 +33,13 @@ public class DimensionBinding { private boolean containsAllNulls; // NOTE: Map must be ordered - public static DimensionBinding createFrom(Map<String,String> values) { + public static DimensionBinding createFrom(Map<String, String> values) { return createFrom(new ArrayList<>(values.keySet()), values); } /** Creates a binding from a variant and a context. Any of the arguments may be null. */ // NOTE: Map must be ordered - public static DimensionBinding createFrom(List<String> dimensions, Map<String,String> context) { + public static DimensionBinding createFrom(List<String> dimensions, Map<String, String> context) { if (dimensions == null || dimensions.size() == 0) { if (context == null) return nullBinding; if (dimensions == null) return new DimensionBinding(null, DimensionValues.empty, context); // Null, but must preserve context @@ -51,7 +50,7 @@ public class DimensionBinding { /** Creates a binding from a variant and a context. Any of the arguments may be null. */ public static DimensionBinding createFrom(List<String> dimensions, DimensionValues dimensionValues) { - if (dimensionValues==null || dimensionValues == DimensionValues.empty) return nullBinding; + if (dimensionValues == null || dimensionValues == DimensionValues.empty) return nullBinding; // If null, preserve raw material for creating a context later (in createFor) if (dimensions == null) return new DimensionBinding(null, dimensionValues, null); @@ -61,14 +60,13 @@ public class DimensionBinding { /** Returns a binding for a (possibly) new set of variants. Variants may be null, but not bindings */ public DimensionBinding createFor(List<String> newDimensions) { - if (newDimensions==null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding - // if (this.context==null && values.length==0) return nullBinding; // No data from which to create a non-null binding - if (this.dimensions==newDimensions) return this; // Avoid creating a new object if the dimensions are the same - - Map<String,String> context=this.context; - if (context==null) - context=this.values.asContext(this.dimensions !=null ? this.dimensions : newDimensions); - return new DimensionBinding(newDimensions,extractDimensionValues(newDimensions,context),context); + if (newDimensions == null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding + if (this.dimensions == newDimensions) return this; // Avoid creating a new object if the dimensions are the same + + Map<String,String> context = this.context; + if (context == null) + context = this.values.asContext(this.dimensions != null ? this.dimensions : newDimensions); + return new DimensionBinding(newDimensions, extractDimensionValues(newDimensions, context), context); } /** @@ -76,20 +74,20 @@ public class DimensionBinding { * The array will not be modified. The context is needed in order to convert this binding to another * given another set of variant dimensions. */ - private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String,String> context) { - this.dimensions=dimensions; - this.values=values; + private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String, String> context) { + this.dimensions = dimensions; + this.values = values; this.context = context; - containsAllNulls=values.isEmpty(); + containsAllNulls = values.isEmpty(); } /** Returns a read-only list of the dimensions of this. This value is undefined if this isNull() */ public List<String> getDimensions() { return dimensions; } /** Returns a context created from the dimensions and values of this */ - public Map<String,String> getContext() { - if (context !=null) return context; - context =values.asContext(dimensions); + public Map<String, String> getContext() { + if (context != null) return context; + context = values.asContext(dimensions); return context; } @@ -102,7 +100,7 @@ public class DimensionBinding { public DimensionValues getValues() { return values; } /** Returns true only if this binding is null (contains no values for its dimensions (if any) */ - public boolean isNull() { return dimensions==null || containsAllNulls; } + public boolean isNull() { return dimensions == null || containsAllNulls; } /** * Returns an array of the dimension values corresponding to the dimensions of this from the given context, @@ -110,10 +108,10 @@ public class DimensionBinding { * Dimensions which are not set in this context get a null value. */ private static DimensionValues extractDimensionValues(List<String> dimensions, Map<String,String> context) { - String[] dimensionValues=new String[dimensions.size()]; - if (context==null || context.size()==0) return DimensionValues.createFrom(dimensionValues); - for (int i=0; i<dimensions.size(); i++) - dimensionValues[i]=context.get(dimensions.get(i)); + String[] dimensionValues = new String[dimensions.size()]; + if (context == null || context.size() == 0) return DimensionValues.createFrom(dimensionValues); + for (int i = 0; i < dimensions.size(); i++) + dimensionValues[i] = context.get(dimensions.get(i)); return DimensionValues.createFrom(dimensionValues); } @@ -138,16 +136,6 @@ public class DimensionBinding { return DimensionBinding.createFrom(combinedDimensions, combinedValues); } - /** Returns the binding of this (dimension->value) as a map */ - private Map<String, String> asMap() { - Map<String, String> map = new LinkedHashMap<>(); - for (int i = 0; i < Math.min(dimensions.size(), values.size()); i++) { - if (values.getValues()[i] != null) - map.put(dimensions.get(i), values.getValues()[i]); - } - return map; - } - /** * Returns a combined list of dimensions from two separate lists, * or null if they are incompatible. @@ -155,8 +143,12 @@ public class DimensionBinding { * (or return null if impossible). */ private List<String> combineDimensions(List<String> d1, List<String> d2) { + if (d1.equals(d2)) return d1; + if (d1.isEmpty()) return d2; + if (d2.isEmpty()) return d1; + List<String> combined = new ArrayList<>(); - int d1Index = 0, d2Index=0; + int d1Index = 0, d2Index = 0; while (d1Index < d1.size() && d2Index < d2.size()) { if (d1.get(d1Index).equals(d2.get(d2Index))) { // agreement on next element combined.add(d1.get(d1Index)); @@ -170,7 +162,7 @@ public class DimensionBinding { combined.add(d2.get(d2Index++)); } else { - return null; // no independent and no agreement + return null; // not independent and no agreement } } if (d1Index < d1.size()) @@ -186,27 +178,22 @@ public class DimensionBinding { * or null if they are incompatible. */ private Map<String, String> combineValues(Map<String, String> m1, Map<String, String> m2) { - Map<String, String> combinedValues = new LinkedHashMap<>(m1); + if (m1.isEmpty()) return m2; + if (m2.isEmpty()) return m1; + Map<String, String> combinedValues = null; for (Map.Entry<String, String> m2Entry : m2.entrySet()) { if (m2Entry.getValue() == null) continue; String m1Value = m1.get(m2Entry.getKey()); if (m1Value != null && ! m1Value.equals(m2Entry.getValue())) return null; // conflicting values of a key + if (combinedValues == null) + combinedValues = new LinkedHashMap<>(m1); combinedValues.put(m2Entry.getKey(), m2Entry.getValue()); } - return combinedValues; + return combinedValues == null ? m1 : combinedValues; } - private boolean intersects(List<String> l1, List<String> l2) { - for (String l1Item : l1) - if (l2.contains(l1Item)) - return true; - return false; - } - - /** - * Returns true if <code>this == invalidBinding</code> - */ + /** Returns true if this == invalidBinding */ public boolean isInvalid() { return this == invalidBinding; } @Override @@ -226,7 +213,7 @@ public class DimensionBinding { /** Two bindings are equal if they contain the same dimensions and the same non-null values */ @Override public boolean equals(Object o) { - if (o==this) return true; + if (o == this) return true; if (! (o instanceof DimensionBinding)) return false; DimensionBinding other = (DimensionBinding)o; if ( ! this.dimensions.equals(other.dimensions)) return false; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java index 4dc9ade62e5..e32d2dc226d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java @@ -22,8 +22,7 @@ public class ModelObjectMap extends PropertyMap { */ @Override protected boolean shouldSet(CompoundName name, Object value) { - if (value == null) return true; - return ! FieldType.isLegalFieldValue(value); + return value != null && ! FieldType.isLegalFieldValue(value); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java index e9ccdd22f98..b6b03d37da8 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java @@ -100,7 +100,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable public QueryProfileType getType() { return type; } /** Sets the type of this, or set to null to not use any type checking in this profile */ - public void setType(QueryProfileType type) { this.type=type; } + public void setType(QueryProfileType type) { this.type = type; } /** Returns the virtual variants of this, or null if none */ public QueryProfileVariants getVariants() { return variants; } @@ -265,37 +265,6 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } /** - * Lists types reachable from this, indexed by the prefix having that type. - * If this is itself typed, this' type will be included with an empty prefix - */ - public Map<CompoundName, QueryProfileType> listTypes(CompoundName prefix, Map<String, String> context) { - DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(), context); - AllTypesQueryProfileVisitor visitor = new AllTypesQueryProfileVisitor(prefix); - accept(visitor, dimensionBinding, null); - return visitor.getResult(); - } - - /** - * Lists references reachable from this. - */ - Set<CompoundName> listReferences(CompoundName prefix, Map<String, String> context) { - DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(),context); - AllReferencesQueryProfileVisitor visitor = new AllReferencesQueryProfileVisitor(prefix); - accept(visitor,dimensionBinding,null); - return visitor.getResult(); - } - - /** - * Lists every entry (value or reference) reachable from this which is not overridable - */ - Set<CompoundName> listUnoverridable(CompoundName prefix, Map<String, String> context) { - DimensionBinding dimensionBinding = DimensionBinding.createFrom(getDimensions(),context); - AllUnoverridableQueryProfileVisitor visitor = new AllUnoverridableQueryProfileVisitor(prefix); - accept(visitor, dimensionBinding, null); - return visitor.getResult(); - } - - /** * Returns a value from this query profile by resolving the given name: * <ul> * <li>The name up to the first dot is the value looked up in the value of this profile @@ -557,6 +526,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable QueryProfileVisitor visitor, DimensionBinding dimensionBinding, QueryProfile owner) { + //System.out.println(" visiting " + this); visitor.onQueryProfile(this, dimensionBinding, owner, null); if (visitor.isDone()) return; @@ -564,12 +534,13 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable if (visitor.isDone()) return; if (allowContent) { - visitContent(visitor,dimensionBinding); + visitContent(visitor, dimensionBinding); if (visitor.isDone()) return; } if (visitor.visitInherited()) visitInherited(allowContent, visitor, dimensionBinding, owner); + //System.out.println(" done visiting " + this); } protected void visitVariants(boolean allowContent, QueryProfileVisitor visitor, DimensionBinding dimensionBinding) { @@ -601,7 +572,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable visitor.acceptValue(contentKey, getContent(contentKey), dimensionBinding, this, null); } else { // get all content in this - for (Map.Entry<String,Object> entry : getContent().entrySet()) { + for (Map.Entry<String, Object> entry : getContent().entrySet()) { visitor.acceptValue(entry.getKey(), entry.getValue(), dimensionBinding, this, null); if (visitor.isDone()) return; } @@ -614,7 +585,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable } /** Returns all the content from this as an unmodifiable map */ - protected Map<String,Object> getContent() { + protected Map<String, Object> getContent() { return content.unmodifiableMap(); } @@ -759,7 +730,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable * Sets the overridability of a field in this profile, * this overrides the corresponding setting in the type (if any) */ - private void setOverridable(CompoundName fieldName,boolean overridable,DimensionBinding dimensionBinding) { + private void setOverridable(CompoundName fieldName, boolean overridable, DimensionBinding dimensionBinding) { QueryProfile parent = lookupParentExact(fieldName, true, dimensionBinding); if (parent.overridable == null) parent.overridable = new HashMap<>(); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java index cffe941b912..5dacd347c2c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java @@ -12,6 +12,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Compile a set of query profiles into compiled profiles. @@ -24,9 +25,8 @@ public class QueryProfileCompiler { public static CompiledQueryProfileRegistry compile(QueryProfileRegistry input) { CompiledQueryProfileRegistry output = new CompiledQueryProfileRegistry(input.getTypeRegistry()); - for (QueryProfile inputProfile : input.allComponents()) { + for (QueryProfile inputProfile : input.allComponents()) output.register(compile(inputProfile, output)); - } return output; } @@ -40,18 +40,20 @@ public class QueryProfileCompiler { // Resolve values for each existing variant and combine into a single data structure Set<DimensionBindingForPath> variants = collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding); variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants - log.fine(() -> "Compiling " + in.toString() + " having " + variants.size() + " variants"); + log.fine(() -> "Compiling " + in + " having " + variants.size() + " variants"); + for (DimensionBindingForPath variant : variants) { log.finer(() -> " Compiling variant " + variant); for (Map.Entry<String, ValueWithSource> entry : in.visitValues(variant.path(), variant.binding().getContext()).valuesWithSource().entrySet()) { - values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + CompoundName fullName = variant.path().append(entry.getKey()); + values.put(fullName, variant.binding(), entry.getValue()); + if (entry.getValue().isUnoverridable()) + unoverridables.put(fullName, variant.binding(), Boolean.TRUE); + if (entry.getValue().isQueryProfile()) + references.put(fullName, variant.binding(), Boolean.TRUE); + if (entry.getValue().queryProfileType() != null) + types.put(fullName, variant.binding(), entry.getValue().queryProfileType()); } - for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) - types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); - for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) - references.put(variant.path().append(reference), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored - for (CompoundName name : in.listUnoverridable(variant.path(), variant.binding().getContext())) - unoverridables.put(variant.path().append(name), variant.binding(), Boolean.TRUE); // Used as a set; value is ignored } return new CompiledQueryProfile(in.getId(), in.getType(), @@ -71,6 +73,7 @@ public class QueryProfileCompiler { */ private static Set<DimensionBindingForPath> collectVariants(CompoundName path, QueryProfile profile, DimensionBinding currentVariant) { Set<DimensionBindingForPath> variants = new HashSet<>(); + variants.addAll(collectVariantsFromValues(path, profile.getContent(), currentVariant)); variants.addAll(collectVariantsInThis(path, profile, currentVariant)); if (profile instanceof BackedOverridableQueryProfile) @@ -99,6 +102,7 @@ public class QueryProfileCompiler { */ private static Set<DimensionBindingForPath> wildcardExpanded(Set<DimensionBindingForPath> variants) { Set<DimensionBindingForPath> expanded = new HashSet<>(); + for (var variant : variants) { if (hasWildcardBeforeEnd(variant.binding())) expanded.addAll(wildcardExpanded(variant, variants)); @@ -119,10 +123,10 @@ public class QueryProfileCompiler { Set<DimensionBindingForPath> expanded = new HashSet<>(); for (var variant : variants) { if (variant.binding().isNull()) continue; + if ( ! variant.path().hasPrefix(variantToExpand.path())) continue; DimensionBinding combined = variantToExpand.binding().combineWith(variant.binding()); - if ( ! combined.isInvalid() ) { + if ( ! combined.isInvalid() ) expanded.add(new DimensionBindingForPath(combined, variantToExpand.path())); - } } return expanded; } @@ -135,7 +139,7 @@ public class QueryProfileCompiler { for (DimensionBindingForPath v1 : v1s) { if (v1.binding().isNull()) continue; for (DimensionBindingForPath v2 : v2s) { - if (v1.binding().isNull()) continue; + if (v2.binding().isNull()) continue; DimensionBinding combined = v1.binding().combineWith(v2.binding()); if ( combined.isInvalid() ) continue; @@ -157,6 +161,7 @@ public class QueryProfileCompiler { if (combinedVariant.isInvalid()) continue; // values at this point in the graph are unreachable + variants.add(new DimensionBindingForPath(combinedVariant, path)); variants.addAll(collectVariantsFromValues(path, variant.values(), combinedVariant)); for (QueryProfile variantInheritedProfile : variant.inherited()) variants.addAll(collectVariants(path, variantInheritedProfile, combinedVariant)); @@ -169,9 +174,6 @@ public class QueryProfileCompiler { Map<String, Object> values, DimensionBinding currentVariant) { Set<DimensionBindingForPath> variants = new HashSet<>(); - if ( ! values.isEmpty()) - variants.add(new DimensionBindingForPath(currentVariant, path)); // there are actual values for this variant - for (Map.Entry<String, Object> entry : values.entrySet()) { if (entry.getValue() instanceof QueryProfile) variants.addAll(collectVariants(path.append(entry.getKey()), (QueryProfile)entry.getValue(), currentVariant)); @@ -202,7 +204,7 @@ public class QueryProfileCompiler { @Override public int hashCode() { - return binding.hashCode() + 17*path.hashCode(); + return binding.hashCode() + 17 * path.hashCode(); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index ea79b10d779..d199ee44c9c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -5,10 +5,12 @@ import com.yahoo.collections.Pair; import com.yahoo.processing.request.CompoundName; import com.yahoo.processing.request.properties.PropertyMap; import com.yahoo.protect.Validator; +import com.yahoo.search.Query; import com.yahoo.search.query.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.DimensionalValue; import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import java.util.ArrayList; @@ -31,7 +33,12 @@ public class QueryProfileProperties extends Properties { /** Values which has been overridden at runtime, or null if none */ private Map<CompoundName, Object> values = null; - /** Query profile references which has been overridden at runtime, or null if none. Earlier values has precedence */ + + /** + * Query profile references which has been overridden at runtime, possibly to the null value to clear values, + * or null if none (i.e this is lazy). + * Earlier values has precedence + */ private List<Pair<CompoundName, CompiledQueryProfile>> references = null; /** Creates an instance from a profile, throws an exception if the given profile is null */ @@ -48,20 +55,21 @@ public class QueryProfileProperties extends Properties { public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { name = unalias(name, context); - Object value = null; - if (values != null) - value = values.get(name); - if (value == null) { - Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); - if (reference != null) - return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // yes; even if null + if (values != null && values.containsKey(name)) + return values.get(name); // Returns this value, even if null + + Pair<CompoundName, CompiledQueryProfile> reference = findReference(name); + if (reference != null) { + if (reference.getSecond() == null) + return null; // cleared + else + return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // even if null } - if (value == null) - value = profile.get(name, context, substitution); - if (value == null) - value = super.get(name, context, substitution); - return value; + Object value = profile.get(name, context, substitution); + if (value != null) + return value; + return super.get(name, context, substitution); } /** @@ -70,7 +78,7 @@ public class QueryProfileProperties extends Properties { * @throws IllegalArgumentException if this property cannot be set in the wrapped query profile */ @Override - public void set(CompoundName name, Object value, Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String, String> context) { // TODO: Refactor try { name = unalias(name, context); @@ -87,9 +95,15 @@ public class QueryProfileProperties extends Properties { // Check types if ( ! profile.getTypes().isEmpty()) { - for (int i = 0; i<name.size(); i++) { - QueryProfileType type = profile.getType(name.first(i), context); + QueryProfileType type; + QueryProfileType explicitTypeFromField = null; + for (int i = 0; i < name.size(); i++) { + if (explicitTypeFromField != null) + type = explicitTypeFromField; + else + type = profile.getType(name.first(i), context); if (type == null) continue; + String localName = name.get(i); FieldDescription fieldDescription = type.getField(localName); if (fieldDescription == null && type.isStrict()) @@ -97,12 +111,19 @@ public class QueryProfileProperties extends Properties { // TODO: In addition to strictness, check legality along the way - if (i == name.size()-1 && fieldDescription != null) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); - if (value == null) - throw new IllegalArgumentException("'" + value + "' is not a " + - fieldDescription.getType().toInstanceDescription()); + if (fieldDescription != null) { + if (i == name.size() - 1) { // at the end of the path, check the assignment type + value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); + if (value == null) + throw new IllegalArgumentException("'" + value + "' is not a " + + fieldDescription.getType().toInstanceDescription()); + } + else if (fieldDescription.getType() instanceof QueryProfileFieldType) { + // If a type is specified, use that instead of the type implied by the name + explicitTypeFromField = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); + } } + } } @@ -128,17 +149,31 @@ public class QueryProfileProperties extends Properties { } } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "': " + e.getMessage()); // TODO: Nest instead + throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); } } @Override - public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, + public void clearAll(CompoundName name, Map<String, String> context) { + if (references == null) + references = new ArrayList<>(); + references.add(new Pair<>(name, null)); + + if (values != null) + values.keySet().removeIf(key -> key.hasPrefix(name)); + } + + @Override + public Map<String, Object> listProperties(CompoundName path, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { path = unalias(path, context); if (context == null) context = Collections.emptyMap(); - Map<String, Object> properties = profile.listValues(path, context, substitution); + Map<String, Object> properties = new HashMap<>(); + for (var entry : profile.listValues(path, context, substitution).entrySet()) { + if (references != null && containsNullParentOf(path, references)) continue; + properties.put(entry.getKey(), entry.getValue()); + } properties.putAll(super.listProperties(path, context, substitution)); if (references != null) { @@ -155,8 +190,14 @@ public class QueryProfileProperties extends Properties { pathInReference = path.rest(refEntry.getFirst().size()); prefixToReferenceKeys = CompoundName.empty; } - for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { - properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + if (refEntry.getSecond() == null) { + if (refEntry.getFirst().hasPrefix(path)) + properties.put(prefixToReferenceKeys.toString(), null); + } + else { + for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) { + properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue()); + } } } @@ -231,6 +272,12 @@ public class QueryProfileProperties extends Properties { return null; } + private boolean containsNullParentOf(CompoundName path, List<Pair<CompoundName, CompiledQueryProfile>> properties) { + if (properties.contains(new Pair<>(path, (CompiledQueryProfile)null))) return true; + if (path.size() > 0 && containsNullParentOf(path.first(path.size() - 1), properties)) return true; + return false; + } + CompoundName unalias(CompoundName name, Map<String,String> context) { if (profile.getTypes().isEmpty()) return name; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java index 0363b50815b..eb7a6d19d91 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java @@ -62,7 +62,6 @@ public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> { int slashIndex=id.getName().lastIndexOf("/"); if (slashIndex<1) return null; String parentName=id.getName().substring(0,slashIndex); - if (parentName.equals("")) return null; ComponentSpecification parentId=new ComponentSpecification(parentName,id.getVersionSpecification()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java index 8dedda800ea..4c4d6778d86 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileVariants.java @@ -30,7 +30,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { private Map<String,FieldValues> fieldValuesByName = new HashMap<>(); /** The inherited profiles for various dimensions settings - a set of fieldvalues of List<QueryProfile> */ - private FieldValues inheritedProfiles =new FieldValues(); + private FieldValues inheritedProfiles = new FieldValues(); /** * Field and inherited profiles sorted by specificity used for all-value visiting. @@ -105,10 +105,14 @@ public class QueryProfileVariants implements Freezable, Cloneable { if (contentName != null) { if (type != null) contentName = type.unalias(contentName); + //System.out.println(" accepting single value in " + this + " for local key " + contentName); acceptSingleValue(contentName, allowContent, visitor, dimensionBinding); // Special cased for performance + //System.out.println(" done accepting single value in " + this + " for local key " + contentName); } else { + //System.out.println(" accepting all values in " + this); acceptAllValues(allowContent, visitor, type, dimensionBinding); + //System.out.println(" done accepting all values in " + this); } } @@ -144,7 +148,7 @@ public class QueryProfileVariants implements Freezable, Cloneable { if (visitor.isDone()) return; fieldIndex++; } - else if (inheritedProfileValue != null) { // Inherited is most specific at this point + else { // Inherited is most specific at this point if (inheritedProfileValue.matches(dimensionBinding.getValues())) { @SuppressWarnings("unchecked") List<QueryProfile> inheritedProfileList = (List<QueryProfile>)inheritedProfileValue.getValue(); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index 2774bd4ebf2..27f600f9ad6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -57,27 +57,44 @@ public class Binding implements Comparable<Binding> { return new Binding(generality, context); } - private Binding(int generality, Map<String, String> binding) { + /** Creates a binding from a map containing the exact bindings this will have */ + private Binding(int generality, Map<String, String> bindings) { this.generality = generality; // Map -> arrays to limit memory consumption and speed up evaluation - dimensions = new String[binding.size()]; - dimensionValues = new String[binding.size()]; + dimensions = new String[bindings.size()]; + dimensionValues = new String[bindings.size()]; int i = 0; - int bindingHash = 0; - for (Map.Entry<String,String> entry : binding.entrySet()) { + for (Map.Entry<String,String> entry : bindings.entrySet()) { dimensions[i] = entry.getKey(); dimensionValues[i] = entry.getValue(); - bindingHash += i * entry.getKey().hashCode() + 11 * i * entry.getValue().hashCode(); i++; } - this.hashCode = bindingHash; + this.hashCode = Arrays.hashCode(dimensions) + 11 * Arrays.hashCode(dimensionValues); } + Binding(DimensionalValue.BindingSpec spec, Map<String, String> bindings) { + this.generality = 0; // Not used here + + // Map -> arrays to limit memory consumption and speed up evaluation + dimensions = spec.dimensions(); + dimensionValues = new String[spec.dimensions().length]; + for (int i = 0; i < dimensions.length; i++) { + dimensionValues[i] = bindings.get(dimensions[i]); + } + this.hashCode = Arrays.hashCode(dimensions) + 11 * Arrays.hashCode(dimensionValues); + } + + /** Returns true only if this binding is null (contains no values for its dimensions (if any) */ public boolean isNull() { return dimensions.length == 0; } + /** Do not change the returtned array */ + String[] dimensions() { return dimensions; } + + String[] dimensionValues() { return dimensionValues; } + @Override public String toString() { StringBuilder b = new StringBuilder("Binding["); @@ -106,7 +123,7 @@ public class Binding implements Comparable<Binding> { * Returns true if all the dimension values in this have the same values * in the given context. */ - public boolean matches(Map<String,String> context) { + public boolean matches(Map<String, String> context) { for (int i = 0; i < dimensions.length; i++) { if ( ! dimensionValues[i].equals(context.get(dimensions[i]))) return false; } @@ -116,7 +133,7 @@ public class Binding implements Comparable<Binding> { /** * Implements a partial ordering where more specific bindings come before less specific ones, * taking both the number of bindings and their positions into account (earlier dimensions - * take precedence over later ones. + * take precedence over later ones). * <p> * The order is not well defined for bindings in different dimensional spaces. */ diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index 644d366e7d0..9be459ceeab 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -102,7 +102,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") * will return {"d" => "a.d-value","e" => "a.e-value"} */ - public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, Collections.<String,String>emptyMap()); } + public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, Collections.emptyMap()); } public final Map<String, Object> listValues(String prefix) { return listValues(new CompoundName(prefix)); } /** @@ -134,7 +134,6 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable public Map<String, Object> listValues(CompoundName prefix, Map<String, String> context, Properties substitution) { Map<String, Object> values = new HashMap<>(); for (Map.Entry<CompoundName, DimensionalValue<ValueWithSource>> entry : entries.entrySet()) { - if ( entry.getKey().size() <= prefix.size()) continue; if ( ! entry.getKey().hasPrefix(prefix)) continue; ValueWithSource valueWithSource = entry.getValue().get(context); @@ -160,6 +159,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable ValueWithSource valueWithSource = entry.getValue().get(context); if (valueWithSource == null) continue; + if (valueWithSource.value() == null) continue; valueWithSource = valueWithSource.withValue(substitute(valueWithSource.value(), context, substitution)); CompoundName suffixName = entry.getKey().rest(prefix.size()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index 7ab05d0cd1e..744c6eb6933 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -1,9 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.compiled; +import com.google.inject.Inject; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileCompiler; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; /** @@ -18,6 +23,15 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer private final QueryProfileTypeRegistry typeRegistry; + @Inject + public CompiledQueryProfileRegistry(QueryProfilesConfig config) { + QueryProfileRegistry registry = QueryProfileConfigurer.createFromConfig(config); + typeRegistry = registry.getTypeRegistry(); + for (QueryProfile inputProfile : registry.allComponents()) { + register(QueryProfileCompiler.compile(inputProfile, this)); + } + } + /** Creates a compiled query profile registry with no types */ public CompiledQueryProfileRegistry() { this(QueryProfileTypeRegistry.emptyFrozen()); @@ -28,7 +42,7 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer } /** Registers a type by its id */ - public void register(CompiledQueryProfile profile) { + public final void register(CompiledQueryProfile profile) { super.register(profile.getId(), profile); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java index b5481059ac0..c1ee49913fe 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java @@ -6,6 +6,7 @@ import com.yahoo.search.query.profile.DimensionBinding; import com.yahoo.search.query.profile.SubstituteString; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -20,20 +21,22 @@ import java.util.Set; */ public class DimensionalValue<VALUE> { - private final List<Value<VALUE>> values; + private final Map<Binding, VALUE> indexedVariants; + private final List<BindingSpec> bindingSpecs; - /** Create a set of variants which is a single value regardless of dimensions */ - public DimensionalValue(Value<VALUE> value) { - this.values = Collections.singletonList(value); - } + private DimensionalValue(List<Value<VALUE>> variants) { + Collections.sort(variants); - public DimensionalValue(List<Value<VALUE>> valueVariants) { - if (valueVariants.size() == 1) { // special cased for efficiency - this.values = Collections.singletonList(valueVariants.get(0)); - } - else { - this.values = new ArrayList<>(valueVariants); - Collections.sort(this.values); + // If there are inconsistent definitions of the same property, we should pick the first in the sort order + this.indexedVariants = new HashMap<>(); + for (Value<VALUE> variant : variants) + indexedVariants.putIfAbsent(variant.binding(), variant.value()); + + this.bindingSpecs = new ArrayList<>(); + for (Value<VALUE> variant : variants) { + BindingSpec spec = new BindingSpec(variant.binding()); + if ( ! bindingSpecs.contains(spec)) + bindingSpecs.add(spec); } } @@ -41,18 +44,21 @@ public class DimensionalValue<VALUE> { public VALUE get(Map<String, String> context) { if (context == null) context = Collections.emptyMap(); - for (Value<VALUE> value : values) { - if (value.matches(context)) - return value.value(); + + for (BindingSpec spec : bindingSpecs) { + if ( ! spec.matches(context)) continue; + VALUE value = indexedVariants.get(new Binding(spec, context)); + if (value != null) + return value; } return null; } - public boolean isEmpty() { return values.isEmpty(); } + public boolean isEmpty() { return indexedVariants.isEmpty(); } @Override public String toString() { - return values.toString(); + return indexedVariants.toString(); } public static class Builder<VALUE> { @@ -93,10 +99,10 @@ public class DimensionalValue<VALUE> { /** A value for a particular binding */ private static class Value<VALUE> implements Comparable<Value> { - private VALUE value = null; + private VALUE value; /** The minimal binding this holds for */ - private Binding binding = null; + private Binding binding; public Value(VALUE value, Binding binding) { this.value = value; @@ -126,9 +132,6 @@ public class DimensionalValue<VALUE> { return " value '" + value + "' for " + binding; } - /** - * A single value with the minimal set of dimension combinations it holds for. - */ private static class Builder<VALUE> { private final VALUE value; @@ -212,4 +215,38 @@ public class DimensionalValue<VALUE> { } + /** A list of dimensions for which there exist one or more bindings in this */ + static class BindingSpec { + + /** The dimensions of this. Unenforced invariant: Content never changes. */ + private final String[] dimensions; + + public BindingSpec(Binding binding) { + this.dimensions = binding.dimensions(); + } + + /** Do not change the returned array */ + String[] dimensions() { return dimensions; } + + /** Returns whether this context contains all the keys of this */ + public boolean matches(Map<String, String> context) { + for (int i = 0; i < dimensions.length; i++) + if ( ! context.containsKey(dimensions[i])) return false; + return true; + } + + @Override + public int hashCode() { + return Arrays.hashCode(dimensions); + } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof BindingSpec)) return false; + return Arrays.equals(((BindingSpec)other).dimensions, this.dimensions); + } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java index 925d20903c6..d2c4eaaec9b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/ValueWithSource.java @@ -2,7 +2,9 @@ package com.yahoo.search.query.profile.compiled; import com.yahoo.search.query.profile.DimensionValues; +import com.yahoo.search.query.profile.types.QueryProfileType; +import java.util.Objects; import java.util.Optional; /** @@ -14,30 +16,76 @@ public class ValueWithSource { private final Object value; - /** The source of the query profile having a value */ private final String source; + private final boolean isUnoverridable; + + private final boolean isQueryProfile; + + private final QueryProfileType type; + /** The dimension values specifying a variant in that profile, or null if it is not in a variant */ private final DimensionValues variant; - public ValueWithSource(Object value, String source, DimensionValues variant) { + public ValueWithSource(Object value, + String source, + boolean isUnoverridable, boolean isQueryProfile, QueryProfileType type, + DimensionValues variant) { this.value = value; this.source = source; + this.isUnoverridable = isUnoverridable; + this.isQueryProfile = isQueryProfile; + this.type = type; this.variant = variant; } + /** + * Returns the value at this key, or null if none + * (in which case this is references a query profile which has no value set). + */ public Object value() { return value; } + /** Returns the source of the query profile having a value */ public String source() { return source; } + /** Returns true if this value cannot be overridden in queries */ + public boolean isUnoverridable() { return isUnoverridable; } + + /** + * Returns true if this key references a query profile (i.e a non-leaf). + * In this case the value may or may not be null, as non-leafs may have values. + */ + public boolean isQueryProfile() { return isQueryProfile; } + + /** Returns tye type of this if it refers to a query profile (not a leaf value), and it has a type */ + public QueryProfileType queryProfileType() { return type; } + public ValueWithSource withValue(Object value) { - return new ValueWithSource(value, source, variant); + return new ValueWithSource(value, source, isUnoverridable, isQueryProfile, type, variant); } /** Returns the variant having this value, or empty if it's not in a variant */ public Optional<DimensionValues> variant() { return Optional.ofNullable(variant); } @Override + public int hashCode() { + // Value is always a value object. Don't include source in identity. + return Objects.hash(value, isUnoverridable, type); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ValueWithSource)) return false; + + ValueWithSource other = (ValueWithSource)o; + if ( ! Objects.equals(this.value, other.value)) return false; + if ( ! Objects.equals(this.isUnoverridable, other.isUnoverridable)) return false; + if ( ! Objects.equals(this.type, other.type)) return false; + return true; + } + + @Override public String toString() { return value + " (from query profile '" + source + "'" + diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java index 33f07a58195..1b1cdce5890 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java @@ -33,7 +33,7 @@ public class QueryProfileXMLReader { * Reads all query profile xml files in a given directory, * and all type xml files from the immediate subdirectory "types/" (if any) * - * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML + * @throws IllegalArgumentException if the directory is not readable, or if there is some error in the XML */ public QueryProfileRegistry read(String directory) { List<NamedReader> queryProfileReaders = new ArrayList<>(); @@ -58,7 +58,7 @@ public class QueryProfileXMLReader { return read(queryProfileTypeReaders,queryProfileReaders); } catch (IOException e) { - throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'",e); + throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'", e); } finally { closeAll(queryProfileReaders); @@ -105,14 +105,14 @@ public class QueryProfileXMLReader { "' must be 'query-profile-type', not '" + root.getNodeName() + "'"); } - String idString=root.getAttribute("id"); + String idString = root.getAttribute("id"); if (idString == null || idString.equals("")) throw new IllegalArgumentException("'" + reader.getName() + "' has no 'id' attribute in the root element"); ComponentId id = new ComponentId(idString); - validateFileNameToId(reader.getName(),id,"query profile type"); + validateFileNameToId(reader.getName(), id,"query profile type"); QueryProfileType type = new QueryProfileType(id); - type.setMatchAsPath(XML.getChild(root,"match") != null); - type.setStrict(XML.getChild(root,"strict") != null); + type.setMatchAsPath(XML.getChild(root, "match") != null); + type.setStrict(XML.getChild(root, "strict") != null); registry.register(type); queryProfileTypeElements.add(root); } @@ -145,7 +145,7 @@ public class QueryProfileXMLReader { queryProfile.setType(type); } - Element dimensions = XML.getChild(root,"dimensions"); + Element dimensions = XML.getChild(root, "dimensions"); if (dimensions != null) queryProfile.setDimensions(toArray(XML.getValue(dimensions))); @@ -215,7 +215,7 @@ public class QueryProfileXMLReader { try { String fieldTypeName = field.getAttribute("type"); if (fieldTypeName == null) throw new IllegalArgumentException("Field '" + field + "' has no 'type' attribute"); - FieldType fieldType=FieldType.fromString(fieldTypeName,registry); + FieldType fieldType = FieldType.fromString(fieldTypeName, registry); type.addField(new FieldDescription(name, fieldType, field.getAttribute("alias"), diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index b8290fa092b..6c30f1a8b05 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -97,7 +97,7 @@ public class FieldDescription implements Comparable<FieldDescription> { this.type = type; // Forbidden until we can figure out the right semantics - if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases is not allowed with compound names"); + if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases are not allowed with compound names"); this.aliases = ImmutableList.copyOf(aliases); this.mandatory = mandatory; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index 07c9e4475ec..c02aada2062 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.FreezableSimpleComponent; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.OverridableQueryProfile; import com.yahoo.search.query.profile.QueryProfile; import java.util.ArrayList; @@ -24,6 +25,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; public class QueryProfileType extends FreezableSimpleComponent { private final CompoundName componentIdAsCompoundName; + /** The fields of this query profile type */ private Map<String, FieldDescription> fields; @@ -217,25 +219,38 @@ public class QueryProfileType extends FreezableSimpleComponent { /** Returns the type of the given query profile type declared as a field in this */ public QueryProfileType getType(String localName) { - FieldDescription fieldDescription=getField(localName); - if (fieldDescription ==null) return null; + FieldDescription fieldDescription = getField(localName); + if (fieldDescription == null) return null; if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) return null; return ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); } + /** Returns the field type of the given name under this, of null if none */ + public FieldType getFieldType(CompoundName name) { + FieldDescription field = getField(name.first()); + if (field == null) return null; + + FieldType fieldType = field.getType(); + if (name.size() == 1) return fieldType; + + if ( ! (fieldType instanceof QueryProfileFieldType)) return null; + + return ((QueryProfileFieldType)fieldType).getQueryProfileType().getFieldType(name.rest()); + } + /** * Returns the description of the field with the given name in this type or an inherited type * (depth first left to right search). Returns null if the field is not defined in this or an inherited profile. */ public FieldDescription getField(String name) { - FieldDescription field=fields.get(name); - if ( field!=null ) return field; + FieldDescription field = fields.get(name); + if ( field != null ) return field; if ( isFrozen() ) return null; // Inherited are collapsed into this for (QueryProfileType inheritedType : this.inherited() ) { - field=inheritedType.getField(name); - if (field!=null) return field; + field = inheritedType.getField(name); + if (field != null) return field; } return null; @@ -276,7 +291,7 @@ public class QueryProfileType extends FreezableSimpleComponent { // Add (/to) a query profile type containing the rest of the name. // (we do not need the field description settings for intermediate query profile types // as the leaf entry will enforce them) - QueryProfileType type = getOrCreateQueryProfileType(name.first(), registry); + QueryProfileType type = extendOrCreateQueryProfileType(name.first(), registry); type.addField(fieldDescription.withName(name.rest()), registry); } else { @@ -288,27 +303,42 @@ public class QueryProfileType extends FreezableSimpleComponent { addAlias(alias, fieldDescription.getName()); } - private QueryProfileType getOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) { + private QueryProfileType extendOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) { + QueryProfileType type = null; FieldDescription fieldDescription = getField(name); if (fieldDescription != null) { - if ( ! ( fieldDescription.getType() instanceof QueryProfileFieldType)) + if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " + "already a " + fieldDescription.getType()); QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType(); - QueryProfileType type = fieldType.getQueryProfileType(); - if (type == null) { // an as-yet untyped reference; add type - type = new QueryProfileType(name); - registry.register(type.getId(), type); - fields.put(name, fieldDescription.withType(new QueryProfileFieldType(type))); - } - return type; + type = fieldType.getQueryProfileType(); + } + + if (type == null) { + type = registry.getComponent(name); + } + + // found in registry but not already added in *this* type (getField also checks parents): extend it + if (type != null && ! fields.containsKey(name)) { + type = new QueryProfileType(ComponentId.createAnonymousComponentId(type.getIdString()), + new HashMap<>(), + List.of(type)); + } + + if (type == null) { // create it + type = new QueryProfileType(ComponentId.createAnonymousComponentId(name)); + } + + if (fieldDescription == null) { + fieldDescription = new FieldDescription(name, new QueryProfileFieldType(type)); } else { - QueryProfileType type = new QueryProfileType(name); - registry.register(type.getId(), type); - fields.put(name, new FieldDescription(name, new QueryProfileFieldType(type))); - return type; + fieldDescription = fieldDescription.withType(new QueryProfileFieldType(type)); } + + registry.register(type); + fields.put(name, fieldDescription); + return type; } private void addAlias(String alias, String field) { @@ -362,6 +392,7 @@ public class QueryProfileType extends FreezableSimpleComponent { return other.getId().equals(this.getId()); } + @Override public String toString() { return "query profile type '" + getId() + "'"; } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java index a4a82d27f8e..83e8dd530ad 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java @@ -37,6 +37,8 @@ public class PropertyAliases extends Properties { * @return the real name if an alias or the input name itself */ protected CompoundName unalias(CompoundName nameOrAlias) { + if (aliases.isEmpty()) return nameOrAlias; + if (nameOrAlias.size() > 1) return nameOrAlias; // aliases are simple names CompoundName properName = aliases.get(nameOrAlias.getLowerCasedName()); return (properName != null) ? properName : nameOrAlias; } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java index 30fc98ac6b1..643e215daef 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java @@ -26,7 +26,10 @@ public class PropertyMap extends Properties { /** The properties of this */ private Map<CompoundName, Object> properties = new LinkedHashMap<>(); - public void set(CompoundName name, Object value, Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String, String> context) { + if (value == null) // Both clear and forward + properties.remove(name); + if (shouldSet(name, value)) properties.put(name, value); else @@ -37,7 +40,7 @@ public class PropertyMap extends Properties { * Return true if this value should be set in this map, false if the set should be propagated instead * This default implementation always returns true. */ - protected boolean shouldSet(CompoundName name,Object value) { return true; } + protected boolean shouldSet(CompoundName name, Object value) { return true; } @Override public Object get(CompoundName name, Map<String,String> context, diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index c06c84fcc36..d5f88ba99ec 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -44,7 +44,7 @@ public class QueryProperties extends Properties { @Override public Object get(CompoundName key, - Map<String,String> context, + Map<String, String> context, com.yahoo.processing.request.Properties substitution) { if (key.size() == 2 && key.first().equals(Model.MODEL)) { Model model = query.getModel(); @@ -69,7 +69,7 @@ public class QueryProperties extends Properties { if (key.last().equals(Ranking.QUERYCACHE)) return ranking.getQueryCache(); if (key.last().equals(Ranking.LIST_FEATURES)) return ranking.getListFeatures(); } - else if (key.size()>=3 && key.get(1).equals(Ranking.MATCH_PHASE)) { + else if (key.size() >= 3 && key.get(1).equals(Ranking.MATCH_PHASE)) { if (key.size() == 3) { MatchPhase matchPhase = ranking.getMatchPhase(); if (key.last().equals(MatchPhase.ATTRIBUTE)) return matchPhase.getAttribute(); @@ -144,7 +144,6 @@ public class QueryProperties extends Properties { return super.get(key, context, substitution); } - @SuppressWarnings("deprecation") @Override public void set(CompoundName key, Object value, Map<String,String> context) { // Note: The defaults here are never used @@ -172,7 +171,7 @@ public class QueryProperties extends Properties { else if (key.last().equals(Model.RESTRICT)) model.setRestrict(asString(value,"")); else - throwIllegalParameter(key.last(),Model.MODEL); + throwIllegalParameter(key.last(), Model.MODEL); } else if (key.first().equals(Ranking.RANKING)) { Ranking ranking = query.getRanking(); @@ -189,46 +188,66 @@ public class QueryProperties extends Properties { ranking.setQueryCache(asBoolean(value, false)); else if (key.last().equals(Ranking.LIST_FEATURES)) ranking.setListFeatures(asBoolean(value,false)); + else + throwIllegalParameter(key.last(), Ranking.RANKING); } - else if (key.size()>=3 && key.get(1).equals(Ranking.MATCH_PHASE)) { + else if (key.size() >= 3 && key.get(1).equals(Ranking.MATCH_PHASE)) { if (key.size() == 3) { MatchPhase matchPhase = ranking.getMatchPhase(); - if (key.last().equals(MatchPhase.ATTRIBUTE)) { + if (key.last().equals(MatchPhase.ATTRIBUTE)) matchPhase.setAttribute(asString(value, null)); - } else if (key.last().equals(MatchPhase.ASCENDING)) { + else if (key.last().equals(MatchPhase.ASCENDING)) matchPhase.setAscending(asBoolean(value, false)); - } else if (key.last().equals(MatchPhase.MAX_HITS)) { + else if (key.last().equals(MatchPhase.MAX_HITS)) matchPhase.setMaxHits(asLong(value, null)); - } else if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) { + else if (key.last().equals(MatchPhase.MAX_FILTER_COVERAGE)) matchPhase.setMaxFilterCoverage(asDouble(value, 0.2)); - } + else + throwIllegalParameter(key.rest().toString(), Ranking.MATCH_PHASE); } else if (key.size() > 3 && key.get(2).equals(Ranking.DIVERSITY)) { Diversity diversity = ranking.getMatchPhase().getDiversity(); if (key.last().equals(Diversity.ATTRIBUTE)) { diversity.setAttribute(asString(value, null)); - } else if (key.last().equals(Diversity.MINGROUPS)) { + } + else if (key.last().equals(Diversity.MINGROUPS)) { diversity.setMinGroups(asLong(value, null)); - } else if ((key.size() > 4) && key.get(3).equals(Diversity.CUTOFF)) { - if (key.last().equals(Diversity.FACTOR)) { + } + else if ((key.size() > 4) && key.get(3).equals(Diversity.CUTOFF)) { + if (key.last().equals(Diversity.FACTOR)) diversity.setCutoffFactor(asDouble(value, 10.0)); - } else if (key.last().equals(Diversity.STRATEGY)) { + else if (key.last().equals(Diversity.STRATEGY)) diversity.setCutoffStrategy(asString(value, "loose")); - } + else + throwIllegalParameter(key.rest().toString(), Diversity.CUTOFF); + } + else { + throwIllegalParameter(key.rest().toString(), Ranking.DIVERSITY); } } } else if (key.size() == 3 && key.get(1).equals(Ranking.SOFTTIMEOUT)) { SoftTimeout soft = ranking.getSoftTimeout(); - if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, true)); - if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, null)); - if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, null)); + if (key.last().equals(SoftTimeout.ENABLE)) + soft.setEnable(asBoolean(value, true)); + else if (key.last().equals(SoftTimeout.FACTOR)) + soft.setFactor(asDouble(value, null)); + else if (key.last().equals(SoftTimeout.TAILCOST)) + soft.setTailcost(asDouble(value, null)); + else + throwIllegalParameter(key.rest().toString(), Ranking.SOFTTIMEOUT); } else if (key.size() == 3 && key.get(1).equals(Ranking.MATCHING)) { Matching matching = ranking.getMatching(); - if (key.last().equals(Matching.TERMWISELIMIT)) matching.setTermwiselimit(asDouble(value, 1.0)); - if (key.last().equals(Matching.NUMTHREADSPERSEARCH)) matching.setNumThreadsPerSearch(asInteger(value, 1)); - if (key.last().equals(Matching.NUMSEARCHPARTITIIONS)) matching.setNumSearchPartitions(asInteger(value, 1)); - if (key.last().equals(Matching.MINHITSPERTHREAD)) matching.setMinHitsPerThread(asInteger(value, 0)); + if (key.last().equals(Matching.TERMWISELIMIT)) + matching.setTermwiselimit(asDouble(value, 1.0)); + else if (key.last().equals(Matching.NUMTHREADSPERSEARCH)) + matching.setNumThreadsPerSearch(asInteger(value, 1)); + else if (key.last().equals(Matching.NUMSEARCHPARTITIIONS)) + matching.setNumSearchPartitions(asInteger(value, 1)); + else if (key.last().equals(Matching.MINHITSPERTHREAD)) + matching.setMinHitsPerThread(asInteger(value, 0)); + else + throwIllegalParameter(key.rest().toString(), Ranking.MATCHING); } else if (key.size() > 2) { String restKey = key.rest().rest().toString(); @@ -237,7 +256,7 @@ public class QueryProperties extends Properties { else if (key.get(1).equals(Ranking.PROPERTIES)) ranking.getProperties().put(restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("properties"))); else - throwIllegalParameter(key.rest().toString(),Ranking.RANKING); + throwIllegalParameter(key.rest().toString(), Ranking.RANKING); } } else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) { @@ -259,11 +278,12 @@ public class QueryProperties extends Properties { query.getSelect().setGroupingExpressionString(asString(value, "")); } else if (key.size() == 2) { - if (key.last().equals(Select.WHERE)) { + if (key.last().equals(Select.WHERE)) query.getSelect().setWhereString(asString(value, "")); - } else if (key.last().equals(Select.GROUPING)) { + else if (key.last().equals(Select.GROUPING)) query.getSelect().setGroupingString(asString(value, "")); - } + else + throwIllegalParameter(key.rest().toString(), Select.SELECT); } else { throwIllegalParameter(key.last(), Select.SELECT); @@ -294,7 +314,7 @@ public class QueryProperties extends Properties { super.set(key,value,context); } catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation - if (e.getMessage().startsWith("Could not set")) + if (e.getMessage() != null && e.getMessage().startsWith("Could not set")) throw e; else throw new IllegalArgumentException("Could not set '" + key + "' to '" + value + "'", e); @@ -316,11 +336,23 @@ public class QueryProperties extends Properties { return properties; } + @SuppressWarnings("deprecation") private void setRankingFeature(Query query, String key, Object value) { - if (value instanceof Tensor) - query.getRanking().getFeatures().put(key, (Tensor)value); - else - query.getRanking().getFeatures().put(key, asString(value, "")); + if (value instanceof Tensor) { + query.getRanking().getFeatures().put(key, (Tensor) value); + } + else if (value instanceof Double) { + query.getRanking().getFeatures().put(key, (Double) value); + } + else { + String valueString = asString(value, ""); + try { + query.getRanking().getFeatures().put(key, Double.parseDouble(valueString)); + } + catch (IllegalArgumentException e) { // TODO: Throw instead on Vespa 8 + query.getRanking().getFeatures().put(key, valueString); + } + } } private Object toSpecifiedType(String key, Object value, QueryProfileType type) { @@ -333,7 +365,7 @@ public class QueryProperties extends Properties { private void throwIllegalParameter(String key,String namespace) { throw new IllegalArgumentException("'" + key + "' is not a valid property in '" + namespace + - "'. See the search api for valid keys starting by '" + namespace + "'."); + "'. See the query api for valid keys starting by '" + namespace + "'."); } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java index ee09521fa74..6cf27fc9a3e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java @@ -7,7 +7,7 @@ import com.yahoo.search.query.Properties; import java.util.Map; /** - * Turns get(name) into get(name,request) using the request given at construction time. + * Turns get(name) into get(name, request) using the request given at construction time. * This is used to allow the query's request to be supplied to all property requests * without forcing users of the query.properties() to supply this explicitly. * @@ -22,18 +22,18 @@ public class RequestContextProperties extends Properties { } @Override - public Object get(CompoundName name,Map<String,String> context, + public Object get(CompoundName name, Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.get(name, context == null ? requestMap : context, substitution); } @Override - public void set(CompoundName name,Object value,Map<String,String> context) { + public void set(CompoundName name, Object value, Map<String,String> context) { super.set(name, value, context == null ? requestMap : context); } @Override - public Map<String, Object> listProperties(CompoundName path,Map<String,String> context, + public Map<String, Object> listProperties(CompoundName path, Map<String,String> context, com.yahoo.processing.request.Properties substitution) { return super.listProperties(path, context == null ? requestMap : context, substitution); } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java index 3bc3a629c5d..88ff7188cb4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java @@ -40,19 +40,29 @@ public class RankFeatures implements Cloneable { features.put(name, value); } - /** Sets a rank feature to a value represented as a string */ + /** + * Sets a rank feature to a value represented as a string. + * + * @deprecated set either a double or a tensor + */ + @Deprecated // TODO: Remove on Vespa 8 public void put(String name, String value) { features.put(name, value); } - /** Returns a rank feature as a string by full name or null if not set */ + /** + * Returns a rank feature as a string by full name or null if not set + * + * @deprecated use getTensor (or getDouble) instead + */ + @Deprecated // TODO: Remove on Vespa 8 public String get(String name) { Object value = features.get(name); if (value == null) return null; return value.toString(); } - /** Returns this value as whatever type it was stored as. Returns null if the value is not set. */ + /** Returns this value as either a Double, Tensor or String. Returns null if the value is not set. */ public Object getObject(String name) { return features.get(name); } @@ -70,14 +80,15 @@ public class RankFeatures implements Cloneable { } /** - * Returns a tensor rank feature, or empty if there is no value with this name. + * Returns a rank feature as a tensor, or empty if there is no value with this name. * - * @throws IllegalArgumentException if the value is set but is not a tensor + * @throws IllegalArgumentException if the value is a string, not a tensor or double */ public Optional<Tensor> getTensor(String name) { Object feature = features.get(name); if (feature == null) return Optional.empty(); if (feature instanceof Tensor) return Optional.of((Tensor)feature); + if (feature instanceof Double) return Optional.of(Tensor.from((Double)feature)); throw new IllegalArgumentException("Expected a tensor value of '" + name + "' but has " + feature); } diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java index b73ef1298ee..d66aff7bcd7 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java @@ -2,7 +2,7 @@ package com.yahoo.search.query.rewrite; import com.yahoo.fsa.FSA; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.search.Query; import com.yahoo.search.intent.model.IntentModel; import com.yahoo.search.intent.model.InterpretationNode; @@ -289,7 +289,7 @@ public class RewriterUtils { * @param msg Log message */ public static void log(Logger logger, String msg) { - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, logger.getName() + ": " + msg); } /** @@ -303,7 +303,7 @@ public class RewriterUtils { if(query!=null) { query.trace(logger.getName() + ": " + msg, true, TRACELEVEL); } - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, logger.getName() + ": " + msg); } /** diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java index 1e8f436a05a..25488aa7bbc 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java @@ -44,4 +44,5 @@ public class VespaLowercasingSearcher extends LowercasingSearcher { Index index = indexFacts.getIndex(sb.toString()); return index.isLowercase() || index.isAttribute(); } + } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index af453983f89..31f8194b3b7 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.TreeNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; +import com.yahoo.container.logging.TraceRenderer; import com.yahoo.data.JsonProducer; import com.yahoo.data.access.Inspectable; import com.yahoo.data.access.Inspector; @@ -44,8 +45,6 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; import com.yahoo.tensor.Tensor; -import com.yahoo.yolean.trace.TraceNode; -import com.yahoo.yolean.trace.TraceVisitor; import org.json.JSONArray; import org.json.JSONObject; @@ -111,10 +110,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final String ROOT = "root"; private static final String SOURCE = "source"; private static final String TOTAL_COUNT = "totalCount"; - private static final String TRACE = "trace"; - private static final String TRACE_CHILDREN = "children"; - private static final String TRACE_MESSAGE = "message"; - private static final String TRACE_TIMESTAMP = "timestamp"; private static final String TIMING = "timing"; private static final String QUERY_TIME = "querytime"; private static final String SUMMARY_FETCH_TIME = "summaryfetchtime"; @@ -132,145 +127,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private LongSupplier timeSource; private OutputStream stream; - private class TraceRenderer extends TraceVisitor { - private final long basetime; - private boolean hasFieldName = false; - int emittedChildNesting = 0; - int currentChildNesting = 0; - private boolean insideOpenObject = false; - - TraceRenderer(long basetime) { - this.basetime = basetime; - } - - @Override - public void entering(TraceNode node) { - ++currentChildNesting; - } - - @Override - public void leaving(TraceNode node) { - conditionalEndObject(); - if (currentChildNesting == emittedChildNesting) { - try { - generator.writeEndArray(); - generator.writeEndObject(); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - --emittedChildNesting; - } - --currentChildNesting; - } - - @Override - public void visit(TraceNode node) { - try { - doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext()); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - - private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException { - boolean dirty = false; - if (timestamp != 0L) { - header(); - generator.writeStartObject(); - generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime); - dirty = true; - } - if (payload != null) { - if (!dirty) { - header(); - generator.writeStartObject(); - } - generator.writeFieldName(TRACE_MESSAGE); - fieldConsumer.renderFieldContentsDirect(payload); - dirty = true; - } - if (dirty) { - if (!hasChildren) { - generator.writeEndObject(); - } else { - setInsideOpenObject(true); - } - } - } - - private void header() { - fieldName(); - for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) { - startChildArray(); - } - emittedChildNesting = currentChildNesting; - } - - private void startChildArray() { - try { - conditionalStartObject(); - generator.writeArrayFieldStart(TRACE_CHILDREN); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - - private void conditionalStartObject() throws IOException { - if (!isInsideOpenObject()) { - generator.writeStartObject(); - } else { - setInsideOpenObject(false); - } - } - - private void conditionalEndObject() { - if (isInsideOpenObject()) { - // This triggers if we were inside a data node with payload and - // subnodes, but none of the subnodes contained data - try { - generator.writeEndObject(); - setInsideOpenObject(false); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - } - } - - private void fieldName() { - if (hasFieldName) { - return; - } - - try { - generator.writeFieldName(TRACE); - } catch (IOException e) { - throw new TraceRenderWrapper(e); - } - hasFieldName = true; - } - - boolean isInsideOpenObject() { - return insideOpenObject; - } - - void setInsideOpenObject(boolean insideOpenObject) { - this.insideOpenObject = insideOpenObject; - } - } - - private static final class TraceRenderWrapper extends RuntimeException { - - /** - * Should never be serialized, but this is still needed. - */ - private static final long serialVersionUID = 2L; - - TraceRenderWrapper(IOException wrapped) { - super(wrapped); - } - - } - public JsonRenderer() { this(null); } @@ -352,8 +208,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { long basetime = trace.traceNode().timestamp(); if (basetime == 0L) basetime = getResult().getElapsedTime().first(); - trace.accept(new TraceRenderer(basetime)); - } catch (TraceRenderWrapper e) { + trace.accept(new TraceRenderer(generator, fieldConsumer, basetime)); + } catch (TraceRenderer.TraceRenderWrapper e) { throw new IOException(e); } } @@ -641,11 +497,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private String getJsonCallback() { Result result = getResult(); - if (result != null) { - Query query = result.getQuery(); - if (query != null) { - return query.properties().getString(JSON_CALLBACK, null); - } + Query query = result.getQuery(); + if (query != null) { + return query.properties().getString(JSON_CALLBACK, null); } return null; } @@ -671,7 +525,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { * This instance is reused for all hits of a Result since we are in a single-threaded context * and want to limit object creation. */ - public static class FieldConsumer implements Hit.RawUtf8Consumer { + public static class FieldConsumer implements Hit.RawUtf8Consumer, TraceRenderer.FieldConsumer { private final JsonGenerator generator; private final boolean debugRendering; @@ -788,11 +642,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { if (field instanceof Inspectable && ! (field instanceof FeatureData)) { renderInspector(((Inspectable)field).inspect()); } else { - renderFieldContentsDirect(field); + accept(field); } } - private void renderFieldContentsDirect(Object field) throws IOException { + @Override + public void accept(Object field) throws IOException { if (field == null) { generator.writeNull(); } else if (field instanceof Boolean) { diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java index 9d0e110a6dd..d79122ac9c4 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java @@ -2,7 +2,7 @@ package com.yahoo.search.rendering; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.URI; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.hitfield.HitField; @@ -395,7 +395,7 @@ public final class SyncDefaultRenderer extends Renderer { if (e instanceof IOException) { throw (IOException) e; } else { - log.log(LogLevel.WARNING, "Exception thrown when rendering the result:", e); + log.log(Level.WARNING, "Exception thrown when rendering the result:", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index fc416c0d930..c14b3f39bc1 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -473,7 +473,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi private Map<String, Object> getFieldMap(int minSize) { if (fields == null) { // Compensate for loadfactor and then some, rounded up.... - fields = new LinkedHashMap<>(2*minSize); + fields = new LinkedHashMap<>(2 * minSize); } return fields; } @@ -505,7 +505,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } /** Returns the types of this as a modifiable set. Modifications to this set are directly reflected in this hit */ - //TODO This shoudld not be exposed as a modifiable set + // TODO: This should not be exposed as a modifiable set public Set<String> types() { if (types == null) types = new ArraySet<>(1); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 5cc34ff5b28..84fe88d0292 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -3,7 +3,6 @@ package com.yahoo.search.searchchain; import com.yahoo.component.chain.Chain; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; @@ -11,7 +10,6 @@ import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; -import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java index d39a488626b..e346a766738 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java @@ -97,11 +97,10 @@ public class DocumentSourceSearcher extends Searcher { public Result search(Query query, Execution execution) { queryCount++; Result r = unFilledResults.get(getQueryKeyClone(query)); - if (r == null) { + if (r == null) r = defaultFilledResult.clone(); - } else { + else r = r.clone(); - } r.setQuery(query); r.hits().trim(query.getOffset(), query.getHits()); @@ -182,11 +181,8 @@ public class DocumentSourceSearcher extends Searcher { * reset. For testing - not reliable if multiple threads makes * queries simultaneously */ - public int getQueryCount() { - return queryCount; - } + public int getQueryCount() { return queryCount; } + + public void resetQueryCount() { queryCount = 0; } - public void resetQueryCount() { - queryCount=0; - } } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index 018f4fc7aa9..3c0453f8900 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -11,7 +11,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.Counter; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.prelude.query.CompositeItem; @@ -51,8 +51,8 @@ public class InputCheckingSearcher extends Searcher { try { checkQuery(query); } catch (IllegalArgumentException e) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); } return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage())); } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java index 8cae081cada..aca2998cba3 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java @@ -4,37 +4,33 @@ package com.yahoo.search.searchers; import com.google.common.annotations.Beta; -import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NearestNeighborItem; -import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.ToolBox; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.query.ranking.RankProperties; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.yolean.chain.After; +import com.yahoo.yolean.chain.Before; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -// This depends on tensors in query.getRanking which are moved to rank.properties during query.prepare() -// Query.prepare is done at the same time as canonicalization (by GroupingExecutor), so use that constraint. -@After(QueryCanonicalizer.queryCanonicalization) - /** * Validates any NearestNeighborItem query items. * * @author arnej */ @Beta +@Before(GroupingExecutor.COMPONENT_NAME) // Must happen before query.prepare() public class ValidateNearestNeighborSearcher extends Searcher { private Map<String, TensorType> validAttributes = new HashMap<>(); @@ -56,7 +52,7 @@ public class ValidateNearestNeighborSearcher extends Searcher { } private Optional<ErrorMessage> validate(Query query) { - NNVisitor visitor = new NNVisitor(query.getRanking().getProperties(), validAttributes); + NNVisitor visitor = new NNVisitor(query.getRanking().getProperties(), validAttributes, query); ToolBox.visit(visitor, query.getModel().getQueryTree().getRoot()); return visitor.errorMessage; } @@ -65,26 +61,26 @@ public class ValidateNearestNeighborSearcher extends Searcher { public Optional<ErrorMessage> errorMessage = Optional.empty(); - private RankProperties rankProperties; - private Map<String, TensorType> validAttributes; + private final RankProperties rankProperties; + private final Map<String, TensorType> validAttributes; + private final Query query; - public NNVisitor(RankProperties rankProperties, Map<String, TensorType> validAttributes) { + public NNVisitor(RankProperties rankProperties, Map<String, TensorType> validAttributes, Query query) { this.rankProperties = rankProperties; this.validAttributes = validAttributes; + this.query = query; } @Override public boolean visit(Item item) { if (item instanceof NearestNeighborItem) { - validate((NearestNeighborItem) item); + String error = validate((NearestNeighborItem)item); + if (error != null) + errorMessage = Optional.of(ErrorMessage.createIllegalQuery(error)); } return true; } - private void setError(String description) { - errorMessage = Optional.of(ErrorMessage.createIllegalQuery(description)); - } - private static boolean isCompatible(TensorType lhs, TensorType rhs) { return lhs.dimensions().equals(rhs.dimensions()); } @@ -98,50 +94,27 @@ public class ValidateNearestNeighborSearcher extends Searcher { return true; } - private void validate(NearestNeighborItem item) { - int target = item.getTargetNumHits(); - if (target < 1) { - setError(item.toString() + " has invalid targetNumHits"); - return; - } - String qprop = item.getQueryTensorName(); - List<Object> rankPropValList = rankProperties.asMap().get(qprop); - if (rankPropValList == null) { - setError(item.toString() + " query tensor not found"); - return; - } - if (rankPropValList.size() != 1) { - setError(item.toString() + " query tensor does not have a single value"); - return; - } - Object rankPropValue = rankPropValList.get(0); - if (! (rankPropValue instanceof Tensor)) { - setError(item.toString() + " query tensor should be a tensor, was: "+ - (rankPropValue == null ? "null" : rankPropValue.getClass().toString())); - return; - } - Tensor qTensor = (Tensor)rankPropValue; - TensorType qTensorType = qTensor.type(); - - String field = item.getIndexName(); - if (validAttributes.containsKey(field)) { - TensorType fTensorType = validAttributes.get(field); - if (fTensorType == null) { - setError(item.toString() + " field is not a tensor"); - return; - } - if (! isCompatible(fTensorType, qTensorType)) { - setError(item.toString() + " field type "+fTensorType+" does not match query tensor type "+qTensorType); - return; - } - if (! isDenseVector(fTensorType)) { - setError(item.toString() + " tensor type "+fTensorType+" is not a dense vector"); - return; - } - } else { - setError(item.toString() + " field is not an attribute"); - return; - } + /** Returns an error message if this is invalid, or null if it is valid */ + private String validate(NearestNeighborItem item) { + if (item.getTargetNumHits() < 1) + return item + " has invalid targetHits " + item.getTargetNumHits() + ": Must be >= 1"; + + String queryFeatureName = "query(" + item.getQueryTensorName() + ")"; + Optional<Tensor> queryTensor = query.getRanking().getFeatures().getTensor(queryFeatureName); + if (queryTensor.isEmpty()) + return item + " requires a tensor rank feature " + queryFeatureName + " but this is not present"; + + if ( ! validAttributes.containsKey(item.getIndexName())) + return item + " field is not an attribute"; + TensorType fieldType = validAttributes.get(item.getIndexName()); + if (fieldType == null) return item + " field is not a tensor"; + if ( ! isDenseVector(fieldType)) + return item + " tensor type " + fieldType + " is not a dense vector"; + + if ( ! isCompatible(fieldType, queryTensor.get().type())) + return item + " field type " + fieldType + " does not match query type " + queryTensor.get().type(); + + return null; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index df9f2af0cce..b2791875d27 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -128,8 +128,7 @@ public class FieldFiller extends Searcher { Set<String> summaryFields = result.getQuery().getPresentation().getSummaryFields(); - if (summaryFields.isEmpty() || - summaryClass == null || + if (summaryFields.isEmpty() || summaryClass == null || result.getQuery().properties().getBoolean(FIELD_FILLER_DISABLE)) { return; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 6eef1252998..22328fb026e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -16,6 +16,7 @@ import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; import static com.yahoo.search.yql.YqlParser.END_ANCHOR; import static com.yahoo.search.yql.YqlParser.EQUIV; import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; import static com.yahoo.search.yql.YqlParser.LABEL; @@ -72,6 +73,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; @@ -689,6 +691,26 @@ public class VespaSerializer { } + private static class GeoLocationSerializer extends Serializer<GeoLocationItem> { + @Override + void onExit(StringBuilder destination, GeoLocationItem item) { } + @Override + boolean serialize(StringBuilder destination, GeoLocationItem item) { + String annotations = leafAnnotations(item); + if (annotations.length() > 0) { + destination.append("([{").append(annotations).append("}]"); + } + destination.append(GEO_LOCATION).append('('); + destination.append(item.getIndexName()).append(", "); + var loc = item.getLocation(); + destination.append(loc.degNS()).append(", "); + destination.append(loc.degEW()).append(", "); + destination.append('"').append(loc.degRadius()).append(" deg").append('"'); + destination.append(')'); + return false; + } + } + private static class NearestNeighborSerializer extends Serializer<NearestNeighborItem> { @Override @@ -701,7 +723,18 @@ public class VespaSerializer { destination.append(leafAnnotations(item)); comma(destination, initLen); int targetNumHits = item.getTargetNumHits(); - destination.append("\"targetNumHits\": ").append(targetNumHits); + annotationKey(destination, "targetNumHits").append(targetNumHits); + int explore = item.getHnswExploreAdditionalHits(); + if (explore != 0) { + comma(destination, initLen); + String key = YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; + annotationKey(destination, key).append(explore); + } + boolean allow_approx = item.getAllowApproximate(); + if (! allow_approx) { + comma(destination, initLen); + annotationKey(destination, "approximate").append(allow_approx); + } destination.append("}]"); destination.append(NEAREST_NEIGHBOR).append('('); destination.append(item.getIndexName()).append(", "); @@ -1152,6 +1185,7 @@ public class VespaSerializer { dispatchBuilder.put(EquivItem.class, new EquivSerializer()); dispatchBuilder.put(ExactStringItem.class, new WordSerializer()); dispatchBuilder.put(IntItem.class, new NumberSerializer()); + dispatchBuilder.put(GeoLocationItem.class, new GeoLocationSerializer()); dispatchBuilder.put(BoolItem.class, new BoolSerializer()); dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(NearItem.class, new NearSerializer()); @@ -1347,6 +1381,11 @@ public class VespaSerializer { } } + private static StringBuilder annotationKey(StringBuilder annotation, String val) { + annotation.append("\"").append(val).append("\": "); + return annotation; + } + private static void comma(StringBuilder annotation, int initLen) { if (annotation.length() > initLen) { annotation.append(", "); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 8d013e501e8..6a464a1503b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -19,11 +19,14 @@ import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; import com.yahoo.collections.LazyMap; import com.yahoo.collections.LazySet; +import com.yahoo.geo.DistanceParser; +import com.yahoo.geo.ParsedDegree; import com.yahoo.language.Language; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.BoolItem; @@ -34,6 +37,7 @@ import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; @@ -94,8 +98,8 @@ import com.yahoo.search.query.parser.ParserFactory; */ public class YqlParser implements Parser { - private static final String DESCENDING_HITS_ORDER = "descending"; - private static final String ASCENDING_HITS_ORDER = "ascending"; + public static final String DESCENDING_HITS_ORDER = "descending"; + public static final String ASCENDING_HITS_ORDER = "ascending"; private enum SegmentWhen { NEVER, POSSIBLY, ALWAYS; @@ -107,12 +111,12 @@ public class YqlParser implements Parser { private static final Integer DEFAULT_HITS = 10; private static final Integer DEFAULT_OFFSET = 0; - private static final Integer DEFAULT_TARGET_NUM_HITS = 10; + public static final Integer DEFAULT_TARGET_NUM_HITS = 10; private static final String ACCENT_DROP_DESCRIPTION = "setting for whether to remove accents if field implies it"; - private static final String ANNOTATIONS = "annotations"; + public static final String ANNOTATIONS = "annotations"; private static final String FILTER_DESCRIPTION = "term filter setting"; private static final String IMPLICIT_TRANSFORMS_DESCRIPTION = "setting for whether built-in query transformers should touch the term"; - private static final String NFKC = "nfkc"; + public static final String NFKC = "nfkc"; private static final String NORMALIZE_CASE_DESCRIPTION = "setting for whether to do case normalization if field implies it"; private static final String ORIGIN_DESCRIPTION = "string origin for a term"; private static final String RANKED_DESCRIPTION = "setting for whether to use term for ranking"; @@ -121,7 +125,7 @@ public class YqlParser implements Parser { private static final String USER_INPUT_ALLOW_EMPTY = "allowEmpty"; private static final String USER_INPUT_DEFAULT_INDEX = "defaultIndex"; private static final String USER_INPUT_GRAMMAR = "grammar"; - private static final String USER_INPUT_LANGUAGE = "language"; + public static final String USER_INPUT_LANGUAGE = "language"; private static final String USER_INPUT_RAW = "raw"; private static final String USER_INPUT_SEGMENT = "segment"; private static final String USER_INPUT = "userInput"; @@ -134,52 +138,56 @@ public class YqlParser implements Parser { public static final String SORTING_LOCALE = "locale"; public static final String SORTING_STRENGTH = "strength"; - static final String ACCENT_DROP = "accentDrop"; - static final String ALTERNATIVES = "alternatives"; - static final String AND_SEGMENTING = "andSegmenting"; - static final String BOUNDS = "bounds"; - static final String BOUNDS_LEFT_OPEN = "leftOpen"; - static final String BOUNDS_OPEN = "open"; - static final String BOUNDS_RIGHT_OPEN = "rightOpen"; - static final String CONNECTION_ID = "id"; - static final String CONNECTION_WEIGHT = "weight"; - static final String CONNECTIVITY = "connectivity"; - static final String DISTANCE = "distance"; - static final String DOT_PRODUCT = "dotProduct"; - static final String EQUIV = "equiv"; - static final String FILTER = "filter"; - static final String HIT_LIMIT = "hitLimit"; - static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; - static final String LABEL = "label"; - static final String NEAR = "near"; - static final String NEAREST_NEIGHBOR = "nearestNeighbor"; - static final String NORMALIZE_CASE = "normalizeCase"; - static final String ONEAR = "onear"; - static final String ORIGIN_LENGTH = "length"; - static final String ORIGIN_OFFSET = "offset"; - static final String ORIGIN = "origin"; - static final String ORIGIN_ORIGINAL = "original"; - static final String PHRASE = "phrase"; - static final String PREDICATE = "predicate"; - static final String PREFIX = "prefix"; - static final String RANGE = "range"; - static final String RANKED = "ranked"; - static final String RANK = "rank"; - static final String SAME_ELEMENT = "sameElement"; - static final String SCORE_THRESHOLD = "scoreThreshold"; - static final String SIGNIFICANCE = "significance"; - static final String STEM = "stem"; - static final String SUBSTRING = "substring"; - static final String SUFFIX = "suffix"; - static final String TARGET_NUM_HITS = "targetNumHits"; - static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; - static final String UNIQUE_ID = "id"; - static final String USE_POSITION_DATA = "usePositionData"; - static final String WAND = "wand"; - static final String WEAK_AND = "weakAnd"; - static final String WEIGHTED_SET = "weightedSet"; - static final String WEIGHT = "weight"; - static final String URI = "uri"; + public static final String ACCENT_DROP = "accentDrop"; + public static final String ALTERNATIVES = "alternatives"; + public static final String AND_SEGMENTING = "andSegmenting"; + public static final String APPROXIMATE = "approximate"; + public static final String BOUNDS = "bounds"; + public static final String BOUNDS_LEFT_OPEN = "leftOpen"; + public static final String BOUNDS_OPEN = "open"; + public static final String BOUNDS_RIGHT_OPEN = "rightOpen"; + public static final String CONNECTION_ID = "id"; + public static final String CONNECTION_WEIGHT = "weight"; + public static final String CONNECTIVITY = "connectivity"; + public static final String DISTANCE = "distance"; + public static final String DOT_PRODUCT = "dotProduct"; + public static final String EQUIV = "equiv"; + public static final String FILTER = "filter"; + public static final String GEO_LOCATION = "geoLocation"; + public static final String HIT_LIMIT = "hitLimit"; + public static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits"; + public static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; + public static final String LABEL = "label"; + public static final String NEAR = "near"; + public static final String NEAREST_NEIGHBOR = "nearestNeighbor"; + public static final String NORMALIZE_CASE = "normalizeCase"; + public static final String ONEAR = "onear"; + public static final String ORIGIN_LENGTH = "length"; + public static final String ORIGIN_OFFSET = "offset"; + public static final String ORIGIN = "origin"; + public static final String ORIGIN_ORIGINAL = "original"; + public static final String PHRASE = "phrase"; + public static final String PREDICATE = "predicate"; + public static final String PREFIX = "prefix"; + public static final String RANGE = "range"; + public static final String RANKED = "ranked"; + public static final String RANK = "rank"; + public static final String SAME_ELEMENT = "sameElement"; + public static final String SCORE_THRESHOLD = "scoreThreshold"; + public static final String SIGNIFICANCE = "significance"; + public static final String STEM = "stem"; + public static final String SUBSTRING = "substring"; + public static final String SUFFIX = "suffix"; + public static final String TARGET_HITS = "targetHits"; + public static final String TARGET_NUM_HITS = "targetNumHits"; + public static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; + public static final String UNIQUE_ID = "id"; + public static final String USE_POSITION_DATA = "usePositionData"; + public static final String WAND = "wand"; + public static final String WEAK_AND = "weakAnd"; + public static final String WEIGHTED_SET = "weightedSet"; + public static final String WEIGHT = "weight"; + public static final String URI = "uri"; private final IndexFacts indexFacts; private final List<ConnectedItem> connectedItems = new ArrayList<>(); @@ -369,6 +377,8 @@ public class YqlParser implements Parser { return buildWeightedSet(ast); case DOT_PRODUCT: return buildDotProduct(ast); + case GEO_LOCATION: + return buildGeoLocation(ast); case NEAREST_NEIGHBOR: return buildNearestNeighbor(ast); case PREDICATE: @@ -410,20 +420,55 @@ public class YqlParser implements Parser { return fillWeightedSet(ast, args.get(1), new DotProductItem(getIndex(args.get(0)))); } + private Item buildGeoLocation(OperatorNode<ExpressionOperator> ast) { + List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); + Preconditions.checkArgument(args.size() == 4, "Expected 4 arguments, got %s.", args.size()); + String field = fetchFieldRead(args.get(0)); + var coord_1 = ParsedDegree.fromString(fetchFieldRead(args.get(1)), true, false); + var coord_2 = ParsedDegree.fromString(fetchFieldRead(args.get(2)), false, true); + double radius = DistanceParser.parse(fetchFieldRead(args.get(3))); + var loc = new Location(); + if (coord_1.isLatitude && coord_2.isLongitude) { + loc.setGeoCircle(coord_1.degrees, coord_2.degrees, radius); + } else if (coord_2.isLatitude && coord_1.isLongitude) { + loc.setGeoCircle(coord_2.degrees, coord_1.degrees, radius); + } else { + throw new IllegalArgumentException("Invalid geoLocation coordinates '"+coord_1+"' and '"+coord_2+"'"); + } + var item = new GeoLocationItem(loc, field); + String label = getAnnotation(ast, LABEL, String.class, null, "item label"); + if (label != null) { + item.setLabel(label); + } + return item; + } + private Item buildNearestNeighbor(OperatorNode<ExpressionOperator> ast) { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); String field = fetchFieldRead(args.get(0)); String property = fetchFieldRead(args.get(1)); NearestNeighborItem item = new NearestNeighborItem(field, property); - Integer targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, + Integer.class, null, "desired minimum hits to produce"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, Integer.class, null, "desired minimum hits to produce"); + } if (targetNumHits != null) { item.setTargetNumHits(targetNumHits); } + Integer hnswExploreAdditionalHits = getAnnotation(ast, HNSW_EXPLORE_ADDITIONAL_HITS, + Integer.class, null, "number of extra hits to explore for HNSW algorithm"); + if (hnswExploreAdditionalHits != null) { + item.setHnswExploreAdditionalHits(hnswExploreAdditionalHits); + } + Boolean allowApproximate = getAnnotation(ast, APPROXIMATE, + Boolean.class, Boolean.TRUE, "allow approximate nearest neighbor search"); + item.setAllowApproximate(allowApproximate); String label = getAnnotation(ast, LABEL, String.class, null, "item label"); if (label != null) { - item.setLabel(label); + item.setLabel(label); } return item; } @@ -494,9 +539,13 @@ public class YqlParser implements Parser { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() == 2, "Expected 2 arguments, got %s.", args.size()); - WandItem out = new WandItem(getIndex(args.get(0)), getAnnotation(ast, - TARGET_NUM_HITS, Integer.class, DEFAULT_TARGET_NUM_HITS, - "desired number of hits to accumulate in wand")); + Integer targetNumHits = getAnnotation(ast, TARGET_HITS, + Integer.class, null, "desired number of hits to accumulate in wand"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(ast, TARGET_NUM_HITS, + Integer.class, DEFAULT_TARGET_NUM_HITS, "desired number of hits to accumulate in wand"); + } + WandItem out = new WandItem(getIndex(args.get(0)), targetNumHits); Double scoreThreshold = getAnnotation(ast, SCORE_THRESHOLD, Double.class, null, "min score for hit inclusion"); if (scoreThreshold != null) { @@ -883,6 +932,8 @@ public class YqlParser implements Parser { private static String fetchFieldRead(OperatorNode<ExpressionOperator> ast) { switch (ast.getOperator()) { + case LITERAL: + return ast.getArgument(0).toString(); case READ_FIELD: return ast.getArgument(1); case PROPREF: @@ -1018,8 +1069,12 @@ public class YqlParser implements Parser { private CompositeItem buildWeakAnd(OperatorNode<ExpressionOperator> spec) { WeakAndItem weakAnd = new WeakAndItem(); - Integer targetNumHits = getAnnotation(spec, TARGET_NUM_HITS, + Integer targetNumHits = getAnnotation(spec, TARGET_HITS, Integer.class, null, "desired minimum hits to produce"); + if (targetNumHits == null) { + targetNumHits = getAnnotation(spec, TARGET_NUM_HITS, + Integer.class, null, "desired minimum hits to produce"); + } if (targetNumHits != null) { weakAnd.setN(targetNumHits); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java index 3c96b00d628..b9e19d6a1b6 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java @@ -15,12 +15,12 @@ import java.util.concurrent.TimeUnit; * Encapsulates all trace-related components and options used by the streaming search Searcher. * * Provides a DEFAULT static instance which has the following characteristics: - * - Approximately 1 query every 2 seconds is traced + * - Approximately 1 query every second is traced * - Trace level is set to 7 for traced queries - * - Only emits traces for queries that have timed out and where the elapsed time is at least 5x + * - Only emits traces for queries that have timed out and where the elapsed time is at least 2x * of the timeout specified in the query itself * - Emits traces to the Vespa log - * - Only 1 trace every 10 seconds may be emitted to the log + * - Only 2 traces every 10 seconds may be emitted to the log */ public class TracingOptions { @@ -50,11 +50,11 @@ public class TracingOptions { public static final TracingOptions DEFAULT; public static final int DEFAULT_TRACE_LEVEL_OVERRIDE = 7; // TODO determine appropriate trace level // Traces won't be exported unless the query has timed out with a duration that is > timeout * multiplier - public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 5.0; + public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 2.0; static { - SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(0.5); - SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 1); + SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(1); + SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 2); TraceExporter traceExporter = new SamplingTraceExporter(new LoggingTraceExporter(), logExportSampler); DEFAULT = new TracingOptions(queryTraceSampler, traceExporter, System::nanoTime, DEFAULT_TRACE_LEVEL_OVERRIDE, TRACE_TIMEOUT_MULTIPLIER_THRESHOLD); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 4750bac551c..8bd7bdac8a5 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.fs4.DocsumPacket; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.routing.Route; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; @@ -300,9 +300,9 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { String expUserId = query.properties().getString(streamingUserid); String expGroupName = query.properties().getString(streamingGroupname); - LogLevel logLevel = LogLevel.ERROR; + Level logLevel = Level.SEVERE; if (skippedEarlierResult) { - logLevel = LogLevel.DEBUG; + logLevel = Level.FINE; } DocumentId docId; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 795b62663d5..2b0d8adc8ef 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -17,7 +17,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.routing.Route; @@ -142,9 +142,9 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private int inferSessionTraceLevel(Query query) { int implicitLevel = traceLevelOverride; - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { implicitLevel = 9; - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { implicitLevel = 7; } return Math.max(query.getTraceLevel(), implicitLevel); @@ -330,20 +330,20 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { VisitorSession session = visitorSessionFactory.createVisitorSession(params); try { if ( !session.waitUntilDone(query.getTimeout())) { - log.log(LogLevel.DEBUG, "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); + log.log(Level.FINE, "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); session.abort(); throw new TimeoutException("Query timed out in " + VdsStreamingSearcher.class.getName()); } } finally { session.destroy(); sessionTrace = session.getTrace(); - log.log(LogLevel.DEBUG, () -> sessionTrace.toString()); + log.log(Level.FINE, () -> sessionTrace.toString()); query.trace(sessionTrace.toString(), false, 9); } if (params.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); } } else { throw new IllegalArgumentException("Query failed: " // TODO: Is it necessary to use a runtime exception? @@ -384,8 +384,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onSearchResult(SearchResult sr) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got SearchResult for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Got SearchResult for query with selection " + params.getDocumentSelection()); } handleSearchResult(sr); } @@ -393,8 +393,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void handleSearchResult(SearchResult sr) { final int hitCountTotal = sr.getTotalHitCount(); final int hitCount = sr.getHitCount(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); } List<SearchResult.Hit> newHits = new ArrayList<>(hitCount); @@ -412,15 +412,15 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } private void mergeGroupingMaps(Map<Integer, byte []> newGroupingMap) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); } for(Integer key : newGroupingMap.keySet()) { byte [] value = newGroupingMap.get(key); Grouping newGrouping = new Grouping(); - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Received group with key " + key + " and size " + value.length); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Received group with key " + key + " and size " + value.length); } BufferSerializer buf = new BufferSerializer( new GrowableByteBuffer(ByteBuffer.wrap(value)) ); newGrouping.deserialize(buf); @@ -440,16 +440,16 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onDocumentSummary(DocumentSummary ds) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); } handleSummary(ds); } private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); } synchronized (summaryMap) { for (int i = 0; i < summaryCount; i++) { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java index 0aaf301e071..230af8971c2 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors.tracing; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.function.Supplier; import java.util.logging.Logger; @@ -17,7 +17,7 @@ public class LoggingTraceExporter implements TraceExporter { public void maybeExport(Supplier<TraceDescription> traceDescriptionSupplier) { var traceDescription = traceDescriptionSupplier.get(); if (traceDescription.getTrace() != null) { - log.log(LogLevel.WARNING, String.format("%s: %s", traceDescription.getDescription(), + log.log(Level.WARNING, String.format("%s: %s", traceDescription.getDescription(), traceDescription.getTrace().toString())); } } diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def index 031877ada81..95e9d4575dd 100644 --- a/container-search/src/main/resources/configdefinitions/qr-start.def +++ b/container-search/src/main/resources/configdefinitions/qr-start.def @@ -20,6 +20,9 @@ jvm.minHeapsize int default=1536 restart ## Stack size (in kilobytes) jvm.stacksize int default=512 restart +## CompressedOOps size in megabytes +jvm.compressedClassSpaceSize int default=32 restart + ## Base value of maximum direct memory size (in megabytes) jvm.baseMaxDirectMemorySize int default=75 restart diff --git a/container-search/src/main/resources/configdefinitions/strict-contracts.def b/container-search/src/main/resources/configdefinitions/strict-contracts.def index f9dd788c054..5ceb37db8d1 100644 --- a/container-search/src/main/resources/configdefinitions/strict-contracts.def +++ b/container-search/src/main/resources/configdefinitions/strict-contracts.def @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=search.federation +## DEPRECATED: This config will be removed on Vespa 8 ## A config to control whether to activate strict adherence to public contracts ## in the container. Usually, the container tries to do a best effort of hiding ## some undesirable effects of the the public contracts. Modifying this config @@ -11,11 +12,9 @@ namespace=search.federation ## can be construed to be unnecessary. searchchains bool default=false -# WARNING: Beta feature, might be removed soon. -# Propagate source.(sourceName).{QueryProperties.PER_SOURCE_QUERY_PROPERTIES} and -# provider.(providerName).{QueryProperties.PER_SOURCE_QUERY_PROPERTIES} -# to the outgoing query. -# All means all in QueryProperties.PER_SOURCE_QUERY_PROPERTIES -# OFFSET_HITS means {Query.HITS, Query.OFFSET} -# NONE means {} -propagateSourceProperties enum {ALL, OFFSET_HITS, NONE} default=ALL +# EVERY, // Propagate any property starting by source.[sourceName] and provider.[providerName] +# NATIVE, // Propagate native properties only +# ALL, // Deprecated synonym of NATIVE +# OFFSET_HITS, // Propagate offset ands hits only +# NONE // propagate no properties +propagateSourceProperties enum {EVERY, NATIVE, ALL, OFFSET_HITS, NONE} default=EVERY diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5b6a4b68930..8b7a57c38e7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; @@ -514,8 +515,10 @@ public class ClusterSearcherTestCase { DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder(); documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1")); - Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"), - new DispatchConfig.Builder().build(), + DispatchConfig dispatchConfig = new DispatchConfig.Builder().build(); + Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig), + ComponentId.createAnonymousComponentId("test-id"), + dispatchConfig, createClusterInfoConfig(), vipStatus, new MockMetric()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index b5e54b46e5a..63475c9c189 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -14,7 +14,6 @@ import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.grouping.GroupingRequest; @@ -151,7 +150,8 @@ public class FastSearcherTestCase { VipStatus vipStatus = new VipStatus(b.build()); List<Node> nodes_1 = ImmutableList.of(new Node(0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1)); - Dispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus); + MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); + dispatch_1.clusterMonitor.shutdown(); vipStatus.addToRotation(clusterName); assertTrue(vipStatus.isInRotation()); dispatch_1.deconstruct(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 440e3b8d78f..2a2c8410b2c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -2,8 +2,10 @@ package com.yahoo.prelude.fastsearch.test; import com.yahoo.container.handler.VipStatus; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; +import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; @@ -13,25 +15,27 @@ import java.util.List; class MockDispatcher extends Dispatcher { + public final ClusterMonitor clusterMonitor; + public static MockDispatcher create(List<Node> nodes) { var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes)); - return create(nodes, rpcResourcePool, 1, new VipStatus()); + return create(nodes, rpcResourcePool, new VipStatus()); } - public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, - int containerClusterSize, VipStatus vipStatus) { + public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, VipStatus vipStatus) { var dispatchConfig = toDispatchConfig(nodes); - var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus); - return new MockDispatcher(searchCluster, dispatchConfig, rpcResourcePool); + var searchCluster = new SearchCluster("a", dispatchConfig, vipStatus, new RpcPingFactory(rpcResourcePool)); + return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool); } - private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) { - this(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster)); + private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) { + this(clusterMonitor, searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster)); } - private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) { - super(searchCluster, dispatchConfig, invokerFactory, new MockMetric()); + private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) { + super(clusterMonitor, searchCluster, dispatchConfig, invokerFactory, new MockMetric()); + this.clusterMonitor = clusterMonitor; } static DispatchConfig toDispatchConfig(List<Node> nodes) { diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java index 5cae40bd10d..df35d8dbdea 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java @@ -11,6 +11,7 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Collections; import static org.junit.Assert.assertEquals; @@ -34,7 +35,7 @@ public class ExactMatchAndDefaultIndexTestCase { q.getModel().setExecution(new Execution(new Execution.Context(null, facts, null, null, null))); assertEquals("AND testexact:a/b testexact:foo.com", q.getModel().getQueryTree().getRoot().toString()); q = new Query("?query=" + enc("a/b foo.com")); - assertEquals("AND \"a b\" \"foo com\"", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND a b foo com", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -44,11 +45,7 @@ public class ExactMatchAndDefaultIndexTestCase { } private String enc(String s) { - try { - return URLEncoder.encode(s, "utf-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } + return URLEncoder.encode(s, StandardCharsets.UTF_8); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index 0fdad1a1f9c..80b2f845e84 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -7,6 +7,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; @@ -48,7 +49,9 @@ public class ParseTestCase { @Test public void testTermWithIndexPrefix() { - tester.assertParsed("url:foobar", "url:foobar", Query.Type.ANY); + tester.assertParsed("url:foobar", + "url:foobar", + Query.Type.ANY); } @Test @@ -59,104 +62,98 @@ public class ParseTestCase { @Test public void testMultipleTermsWithUTF8EncodingOred() { tester.assertParsed("OR l\u00e5gen delta M\u00dcNICH M\u00fcnchen", - "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", Query.Type.ANY); + "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", + Query.Type.ANY); } @Test public void testMultipleTermsWithMultiplePrefixes() { tester.assertParsed("RANK (+bar -normal.title:foo -baz) url:foobar", - "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); + "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY); } @Test public void testSimpleQueryDefaultOr() { - tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", - Query.Type.ANY); + tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", Query.Type.ANY); } @Test public void testOrAndNot() { tester.assertParsed("RANK (+(AND baz bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); + "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY); } @Test public void testSimpleOrNestedAnd() { tester.assertParsed("RANK (OR foo bar baz) foobar xyzzy", - "foobar +(foo bar baz) xyzzy", Query.Type.ANY); + "foobar +(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testSimpleOrNestedNot() { tester.assertParsed("+(OR foobar xyzzy) -(AND foo bar baz)", - "foobar -(foo bar baz) xyzzy", Query.Type.ANY); + "foobar -(foo bar baz) xyzzy", Query.Type.ANY); } @Test public void testOrNotNestedAnd() { - tester.assertParsed( - "RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrAndNotNestedNot() { - tester.assertParsed( - "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAnd() { - tester.assertParsed( - "RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", - "foobar +(fo ba foba) foo bar +(foz baraz) baz", Query.Type.ANY); + tester.assertParsed("RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz", + "foobar +(fo ba foba) foo bar +(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrMultipleNestedNot() { - tester.assertParsed( - "+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", - "foobar -(fo ba foba) foo bar -(foz baraz) baz", Query.Type.ANY); + tester.assertParsed("+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)", + "foobar -(fo ba foba) foo bar -(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAnd() { - tester.assertParsed( - "RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", - "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo", + "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedNot() { - tester.assertParsed( - "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", - "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo", + "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar", + Query.Type.ANY); } @Test public void testOrMultipleNestedAndNot() { - tester.assertParsed( - "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", - "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz", + "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz", + Query.Type.ANY); } @Test public void testOrAndNotMultipleNestedAndNot() { - tester.assertParsed( - "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", - "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar", + "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy", + Query.Type.ANY); } @Test public void testExplicitPhrase() { - Item root=tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); + Item root = tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY); assertTrue(root instanceof PhraseItem); assertTrue(((PhraseItem)root).isExplicit()); } @@ -164,21 +161,20 @@ public class ParseTestCase { @Test public void testPhraseWithIndex() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", Query.Type.ANY); + "normal.title:\"foo bar foobar\"", Query.Type.ANY); } @Test public void testPhrasesAndTerms() { tester.assertParsed("OR \"foo bar foobar\" xyzzy \"baz gaz faz\"", - "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); + "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY); } @Test public void testPhrasesAndTermsWithOperators() { - tester.assertParsed( - "RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", - "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", - Query.Type.ANY); + tester.assertParsed("RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy", + "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar", + Query.Type.ANY); } @Test @@ -188,38 +184,40 @@ public class ParseTestCase { @Test public void testTermWithCatalogAndIndexPrefixDefaultAnd() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", - Query.Type.ALL); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ALL); } @Test public void testMultipleTermsWithMultiplePrefixesDefaultAnd() { tester.assertParsed("+(AND url:foobar bar) -normal.title:foo -baz", - "url:foobar +bar -normal.title:foo -baz", Query.Type.ALL); + "url:foobar +bar -normal.title:foo -baz", + Query.Type.ALL); } @Test public void testSimpleQueryDefaultAnd() { - tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", - Query.Type.ALL); + tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", Query.Type.ALL); } @Test public void testNotDefaultAnd() { - tester.assertParsed( - "+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", - "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", Query.Type.ALL); + tester.assertParsed("+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)", + "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", + Query.Type.ALL); } @Test public void testSimpleTermQueryDefaultPhrase() { - tester.assertParsed("foobar", "foobar", Query.Type.PHRASE); + tester.assertParsed("foobar", + "foobar", + Query.Type.PHRASE); } @Test public void testSimpleQueryDefaultPhrase() { - Item root=tester.assertParsed("\"foobar foo bar baz\"", "foobar foo bar baz", - Query.Type.PHRASE); + Item root = tester.assertParsed("\"foobar foo bar baz\"", + "foobar foo bar baz", + Query.Type.PHRASE); assertTrue(root instanceof PhraseItem); assertFalse(((PhraseItem)root).isExplicit()); } @@ -227,23 +225,25 @@ public class ParseTestCase { @Test public void testMultipleTermsWithMultiplePrefixesDefaultPhrase() { tester.assertParsed("\"url foobar bar normal title foo baz\"", - "url:foobar +bar -normal.title:foo -baz", Query.Type.PHRASE); + "url:foobar +bar -normal.title:foo -baz", + Query.Type.PHRASE); } @Test public void testOdd1() { - tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.ALL); + tester.assertParsed("AND window print error", "+window.print() +error", + Query.Type.ALL); } @Test public void testOdd2() { - tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"",Query.Type.ALL); + tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"", + Query.Type.ALL); } @Test public void testOdd2Uppercase() { - tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", - Query.Type.ALL); + tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", Query.Type.ALL); } @Test @@ -280,19 +280,19 @@ public class ParseTestCase { @Test public void testNestedCompositesDefaultOr() { tester.assertParsed("RANK (OR foobar bar baz) foo xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY); } @Test public void testNestedCompositesDefaultAnd() { tester.assertParsed("AND foo (OR foobar bar baz) xyzzy", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL); } @Test public void testNestedCompositesPhraseDefault() { tester.assertParsed("\"foo foobar bar baz xyzzy\"", - "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); + "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE); } @Test @@ -349,8 +349,7 @@ public class ParseTestCase { @Test public void testNumericWithIndex() { - tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", - Query.Type.ANY); + tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", Query.Type.ANY); } @Test @@ -361,14 +360,14 @@ public class ParseTestCase { @Test public void testMultipleIntegerWithIndex() { tester.assertParsed("OR document.size:[34;454] date:>1234567890", - "document.size:[34;454] date:>1234567890", Query.Type.ANY); + "document.size:[34;454] date:>1234567890", Query.Type.ANY); } @Test public void testMixedNumericAndOtherTerms() { tester.assertParsed("RANK (AND document.size:<1024 xyzzy) foo date:>123456890", - "foo +document.size:<1024 +xyzzy date:>123456890", - Query.Type.ANY); + "foo +document.size:<1024 +xyzzy date:>123456890", + Query.Type.ANY); } @Test @@ -378,20 +377,18 @@ public class ParseTestCase { @Test public void testItemPhraseEmptyPhrase() { - tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", - Query.Type.ANY); + tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", Query.Type.ANY); } @Test public void testSimpleQuery() { - tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", - Query.Type.ANY); + tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \" f g 4 2\"\" maybe", Query.Type.ANY); } @Test public void testExcessivePluses() { tester.assertParsed("+(AND other is nothing) -test", - "++other +++++is ++++++nothing -test", Query.Type.ANY); + "++other +++++is ++++++nothing -test", Query.Type.ANY); } @Test @@ -401,39 +398,38 @@ public class ParseTestCase { @Test public void testPlusesAndMinuses() { - Item root=tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ANY); - assertTrue(root instanceof PhraseItem); - assertFalse(((PhraseItem)root).isExplicit()); + tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ANY); } @Test public void testNumbers() { - tester.assertParsed("\"123 2132odfd 934032 32423\"", - "123+2132odfd.934032,,32423", Query.Type.ANY); + tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ANY); } @Test public void testOtherSignsInQuote() { - tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", - Query.Type.ANY); + tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ANY); } @Test public void testGribberish() { tester.assertParsed("1349832840234l3040roer\u00e6lf12", - ",1349832840234l3040roer\u00e6lf12", Query.Type.ANY); + ",1349832840234l3040roer\u00e6lf12", + Query.Type.ANY); } @Test public void testUrl() { - tester.assertParsed("www:\"www hotelaiguablava com\"", - "+www:www.hotelaiguablava:com", Query.Type.ANY); + tester.assertParsed("AND www:www www:hotelaiguablava www:com", + "+www:www.hotelaiguablava:com", + Query.Type.ANY); } @Test public void testUrlGribberish() { - tester.assertParsed("OR \"3 16\" fast.type:lycosoffensive", - "[ 3:16 fast.type:lycosoffensive", Query.Type.ANY); + tester.assertParsed("OR (AND 3 16) fast.type:lycosoffensive", + "[ 3:16 fast.type:lycosoffensive", + Query.Type.ANY); } @Test @@ -475,8 +471,7 @@ public class ParseTestCase { @Test public void testPrefixWithDotAdvanced() { - tester.assertParsed("normal.title:foobar", "normal.title:foobar", - Query.Type.ADVANCED); + tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ADVANCED); } @Test @@ -486,20 +481,21 @@ public class ParseTestCase { @Test public void testSimplePhraseAdvanced() { - tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", - Query.Type.ADVANCED); + tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ADVANCED); } @Test public void testSimplePhraseWithIndexAdvanced() { tester.assertParsed("normal.title:\"foo bar foobar\"", - "normal.title:\"foo bar foobar\"", Query.Type.ADVANCED); + "normal.title:\"foo bar foobar\"", + Query.Type.ADVANCED); } @Test public void testMultiplePhrasesAdvanced() { tester.assertParsed("AND \"foo bar foobar\" \"baz gaz faz\"", - "\"foo bar foobar\" and \"baz gaz faz\"", Query.Type.ADVANCED); + "\"foo bar foobar\" and \"baz gaz faz\"", + Query.Type.ADVANCED); } @Test @@ -661,23 +657,23 @@ public class ParseTestCase { @Test public void testImplicitPhrase1Advanced() { - tester.assertParsed("\"test if\"", "--test+-if", Query.Type.ADVANCED); + tester.assertParsed("AND test if", "--test+-if", Query.Type.ADVANCED); } @Test public void testImplicitPhrase2Advanced() { - tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ADVANCED); + tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ADVANCED); } @Test public void testImplicitPhrase3Advanced() { - tester.assertParsed("\"123 2132odfd 934032 32423\"", + tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ADVANCED); } @Test public void testImplicitPhrase4Advanced() { - tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", Query.Type.ADVANCED); + tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ADVANCED); } @Test @@ -730,7 +726,7 @@ public class ParseTestCase { @Test public void testSingleHyphen() { - tester.assertParsed("\"a b\"", "a-b", Query.Type.ALL); + tester.assertParsed("AND a b", "a-b", Query.Type.ALL); } @Test @@ -883,27 +879,27 @@ public class ParseTestCase { @Test public void testSimpleDotPhraseAny() { - tester.assertParsed("OR a \"b c\" d", "a b.c d", Query.Type.ANY); + tester.assertParsed("OR a (AND b c) d", "a b.c d", Query.Type.ANY); } @Test public void testSimpleHyphenPhraseAny() { - tester.assertParsed("OR a \"b c\" d", "a b-c d", Query.Type.ANY); + tester.assertParsed("OR a (AND b c) d", "a b-c d", Query.Type.ANY); } @Test public void testAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR \"a b\" c d", "a.b c d", Query.Type.ANY); + tester.assertParsed("OR (AND a b) c d", "a.b c d", Query.Type.ANY); } @Test public void testYetAnotherSimpleDotPhraseAny() { - tester.assertParsed("OR a b \"c d\"", "a b c.d", Query.Type.ANY); + tester.assertParsed("OR a b (AND c d)", "a b c.d", Query.Type.ANY); } @Test public void testVariousSeparatorsPhraseAny() { - tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ANY); + tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ANY); } @Test @@ -918,45 +914,44 @@ public class ParseTestCase { @Test public void testIndexedDottedPhraseAny() { - tester.assertParsed("OR a url:\"b c\" d", "a url:b.c d", Query.Type.ANY); + tester.assertParsed("OR a (AND url:b url:c) d", "a url:b.c d", Query.Type.ANY); } @Test public void testIndexedPlusedPhraseAny() { - tester.assertParsed("OR a normal.title:\"b c\" d", "a normal.title:b+c d", - Query.Type.ANY); + tester.assertParsed("OR a (AND normal.title:b normal.title:c) d", "a normal.title:b+c d", Query.Type.ANY); } @Test public void testNestedNotAny() { tester.assertParsed( - "RANK (+(OR normal.title:foobar url:\"www pvv org\") -foo) a", + "RANK (+(OR normal.title:foobar (AND url:www url:pvv url:org)) -foo) a", "a +(normal.title:foobar url:www.pvv.org) -foo", Query.Type.ANY); } @Test public void testDottedPhraseAdvanced() { - tester.assertParsed("OR a \"b c\"", "a or b.c", Query.Type.ADVANCED); + tester.assertParsed("OR a (AND b c)", "a or b.c", Query.Type.ADVANCED); } @Test public void testHyphenPhraseAdvanced() { - tester.assertParsed("OR (AND a \"b c\") d", "a and b-c or d", Query.Type.ADVANCED); + tester.assertParsed("OR (AND a (AND b c)) d", "a and b-c or d", Query.Type.ADVANCED); } @Test public void testAnotherDottedPhraseAdvanced() { - tester.assertParsed("OR \"a b\" c", "a.b or c", Query.Type.ADVANCED); + tester.assertParsed("OR (AND a b) c", "a.b or c", Query.Type.ADVANCED); } @Test public void testNottedDottedPhraseAdvanced() { - tester.assertParsed("+a -\"c d\"", "a andnot c.d", Query.Type.ADVANCED); + tester.assertParsed("+a -(AND c d)", "a andnot c.d", Query.Type.ADVANCED); } @Test public void testVariousSeparatorsPhraseAdvanced() { - tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ADVANCED); + tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ADVANCED); } @Test @@ -976,14 +971,14 @@ public class ParseTestCase { @Test public void testNestedPlussedPhraseAdvanced() { - tester.assertParsed("AND (OR a normal.title:\"b c\") d", + tester.assertParsed("AND (OR a (AND normal.title:b normal.title:c)) d", "a or normal.title:b+c and d", Query.Type.ADVANCED); } @Test public void testNottedNestedDottedPhraseAdvanced() { tester.assertParsed( - "+(AND a (OR normal.title:foobar url:\"www pvv org\")) -foo", + "+(AND a (OR normal.title:foobar (AND url:www url:pvv url:org))) -foo", "a and (normal.title:foobar or url:www.pvv.org) andnot foo", Query.Type.ADVANCED); } @@ -995,7 +990,7 @@ public class ParseTestCase { @Test public void testPlusedTwiceThenQuotedPhraseAny() { - tester.assertParsed("\"a b c d\"", "a+b+\"c d\"", Query.Type.ANY); + tester.assertParsed("AND a b c d", "a+b+\"c d\"", Query.Type.ANY); } @Test @@ -1005,7 +1000,7 @@ public class ParseTestCase { @Test public void testPhrasesInBraces() { - tester.assertParsed("url.domain:\"microsoft com\"", + tester.assertParsed("AND url.domain:microsoft url.domain:com", "+(url.domain:microsoft.com)", Query.Type.ALL); } @@ -1053,17 +1048,17 @@ public class ParseTestCase { @Test public void testPhraseNotPrefix() { - tester.assertParsed("OR foo \"prefix bar\"", "foo prefix*bar", Query.Type.ANY); + tester.assertParsed("OR foo (AND prefix bar)", "foo prefix*bar", Query.Type.ANY); } @Test public void testPhraseNotSubstring() { - tester.assertParsed("OR foo \"substring bar\"", "foo *substring*bar", Query.Type.ANY); + tester.assertParsed("OR foo (AND substring bar)", "foo *substring*bar", Query.Type.ANY); } @Test public void testPhraseNotSuffix() { - tester.assertParsed("OR \"foo suffix\" bar", "foo*suffix bar", Query.Type.ANY); + tester.assertParsed("OR (AND foo suffix) bar", "foo*suffix bar", Query.Type.ANY); } @Test @@ -1086,20 +1081,17 @@ public class ParseTestCase { @Test public void testIndexedPhraseNotPrefix() { - tester.assertParsed("foo.bar:\"prefix xyzzy\"", "foo.bar:prefix*xyzzy", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:prefix foo.bar:xyzzy", "foo.bar:prefix*xyzzy", Query.Type.ANY); } @Test public void testIndexedPhraseNotSubstring() { - tester.assertParsed("foo.bar:\"substring xyzzy\"", "foo.bar:*substring*xyzzy", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:substring foo.bar:xyzzy", "foo.bar:*substring*xyzzy", Query.Type.ANY); } @Test public void testIndexedPhraseNotSuffix() { - tester.assertParsed("foo.bar:\"xyzzy suffix\"", "foo.bar:xyzzy*suffix", - Query.Type.ANY); + tester.assertParsed("AND foo.bar:xyzzy foo.bar:suffix", "foo.bar:xyzzy*suffix", Query.Type.ANY); } @Test @@ -1120,20 +1112,20 @@ public class ParseTestCase { assertTrue(root instanceof SuffixItem); } - /** Non existing index → phrase **/ + /** Non existing index → and **/ @Test public void testNonIndexPhraseNotPrefix() { - tester.assertParsed("\"void prefix\"", "void:prefix*", Query.Type.ANY); + tester.assertParsed("AND void prefix", "void:prefix*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSubstring() { - tester.assertParsed("\"void substring\"", "void:*substring*", Query.Type.ANY); + tester.assertParsed("AND void substring", "void:*substring*", Query.Type.ANY); } @Test public void testNonIndexPhraseNotSuffix() { - tester.assertParsed("\"void suffix\"", "void:*suffix", Query.Type.ANY); + tester.assertParsed("AND void suffix", "void:*suffix", Query.Type.ANY); } /** Explicit phrase → remove '*' **/ @@ -1198,7 +1190,7 @@ public class ParseTestCase { /** Extra spaces with index **/ @Test public void testIndexPrefixExtraSpace() { - tester.assertParsed("\"foo prefix\"", "foo:prefix *", Query.Type.ANY); + tester.assertParsed("AND foo prefix", "foo:prefix *", Query.Type.ANY); } @Test @@ -1419,7 +1411,7 @@ public class ParseTestCase { @Test public void testMultipleDifferentPhraseSeparators() { - tester.assertParsed("\"foo bar\"", "foo.-.bar", Query.Type.ANY); + tester.assertParsed("AND foo bar", "foo.-.bar", Query.Type.ANY); } @Test @@ -1430,19 +1422,17 @@ public class ParseTestCase { @Test public void testReallyNoisyQuery1() { - tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", - Query.Type.ALL); + tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", Query.Type.ALL); } @Test public void testReallyNoisyQuery2() { - tester.assertParsed("\"\u03bc\u03bc hei\"", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); + tester.assertParsed("AND \u03bc\u03bc hei", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL); } @Test public void testReallyNoisyQuery3() { - tester.assertParsed("AND \"hei hallo\" du der", "hei-hallo;du;der", - Query.Type.ALL); + tester.assertParsed("AND hei hallo du der", "hei-hallo;du;der", Query.Type.ALL); } @Test @@ -1478,7 +1468,7 @@ public class ParseTestCase { @Test public void testTheStupidSymbolsWhichAreNowWordCharactersInUnicode() { - tester.assertParsed("\"yz a\"", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); + tester.assertParsed("AND yz a", "yz\u00A8\u00AA\u00AF", Query.Type.ANY); } @Test @@ -1498,7 +1488,7 @@ public class ParseTestCase { @Test public void testImplicitPhrasingWithIndex() { - tester.assertParsed("a:\"b c\"", "a:/b/c", Query.Type.ANY); + tester.assertParsed("AND a:b a:c", "a:/b/c", Query.Type.ANY); } @Test @@ -1508,7 +1498,7 @@ public class ParseTestCase { @Test public void testSingleNoisyPhraseWithIndex() { - tester.assertParsed("mail:\"yahoo com\"", "mail:@yahoo.com", Query.Type.ANY); + tester.assertParsed("AND mail:yahoo mail:com", "mail:@yahoo.com", Query.Type.ANY); } @Test @@ -1599,7 +1589,7 @@ public class ParseTestCase { "url.all:http://www.newsadvance.com/servlet/Satellite?pagename=LNA/MGArticle/IMD_BasicArticle&c=MGArticle&cid=1031782787014&path=!mgnetwork!diversions", Query.Type.ALL); tester.assertParsed( - "AND ull:\"http www neue oz de information pub Boulevard index html file a 3 s 4 file\" s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", + "AND ull:http ull:www ull:neue ull:oz ull:de ull:information ull:pub ull:Boulevard ull:index ull:html ull:file ull:a ull:3 ull:s ull:4 ull:file s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"", "ull:http://www.neue-oz.de/information/pub_Boulevard/index.html?file=a:3:{s:4:\"file\";s:37:\"iptc-bdt-20050607-294-dpa_9001170.txt\";s:3:\"dir\";s:26:\"/opt/DPA/parsed/boulevard/\";s:7:\"bereich\";s:9:\"Boulevard\";}", Query.Type.ALL); } @@ -1640,7 +1630,7 @@ public class ParseTestCase { @Test public void testTooLongQueryTerms() { - tester.assertParsed("AND \"545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"!1000 \"2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"", + tester.assertParsed("AND 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof 2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", "+/545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof&filter=ew:545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof!1000 =.2b..2f.545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof", Query.Type.ALL); } @@ -1648,22 +1638,22 @@ public class ParseTestCase { @Test public void testNonSpecialTokenParsing() { ParsingTester customTester = new ParsingTester(new SpecialTokens("default")); - customTester.assertParsed("OR c or c with \"tcp ip\"", "c# or c++ with tcp/ip", Query.Type.ANY); + customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY); } @Test public void testNonIndexWithColons1() { - tester.assertParsed("OR this is \"notan iindex\"", "this is notan:iindex", Query.Type.ANY); + tester.assertParsed("OR this is (AND notan iindex)", "this is notan:iindex", Query.Type.ANY); } @Test public void testNonIndexWithColons2() { - tester.assertParsed("OR this is \"notan iindex either\"", "this is notan:iindex:either", Query.Type.ANY); + tester.assertParsed("OR this is (AND notan iindex either)", "this is notan:iindex:either", Query.Type.ANY); } @Test public void testIndexThenUnderscoreTermBecomesIndex() { - tester.assertParsed("name:\"batch article\"", "name:batch_article", Query.Type.ANY); + tester.assertParsed("AND name:batch name:article", "name:batch_article", Query.Type.ANY); } @Test @@ -1671,17 +1661,17 @@ public class ParseTestCase { // "first" "second" and "third" are segments in the test language Item item = tester.parseQuery("name:firstsecondthird", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertTrue(item instanceof PhraseSegmentItem); - PhraseSegmentItem phrase = (PhraseSegmentItem) item; + assertTrue(item instanceof AndSegmentItem); + AndSegmentItem segment = (AndSegmentItem) item; - assertEquals(3, phrase.getItemCount()); - assertEquals("name:first", phrase.getItem(0).toString()); - assertEquals("name:second", phrase.getItem(1).toString()); - assertEquals("name:third", phrase.getItem(2).toString()); + assertEquals(3, segment.getItemCount()); + assertEquals("name:first", segment.getItem(0).toString()); + assertEquals("name:second", segment.getItem(1).toString()); + assertEquals("name:third", segment.getItem(2).toString()); - assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(2)).getIndexName()); } @Test @@ -1690,21 +1680,21 @@ public class ParseTestCase { Item item = tester.parseQuery("name:\"firstsecondthird\"", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); assertTrue(item instanceof PhraseSegmentItem); - PhraseSegmentItem phrase = (PhraseSegmentItem) item; + PhraseSegmentItem segment = (PhraseSegmentItem) item; - assertEquals(3, phrase.getItemCount()); - assertEquals("name:first", phrase.getItem(0).toString()); - assertEquals("name:second", phrase.getItem(1).toString()); - assertEquals("name:third", phrase.getItem(2).toString()); + assertEquals(3, segment.getItemCount()); + assertEquals("name:first", segment.getItem(0).toString()); + assertEquals("name:second", segment.getItem(1).toString()); + assertEquals("name:third", segment.getItem(2).toString()); - assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName()); - assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName()); - assertEquals("name", ((WordItem)phrase.getItem(2)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName()); + assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName()); + assertEquals("name", ((WordItem)segment.getItem(2)).getIndexName()); } @Test public void testAndItemAndImplicitPhrase() { - tester.assertParsed("\"\u00d8 \u00d8 \u00d8 \u00d9\"", + tester.assertParsed("AND \u00d8 \u00d8 \u00d8 \u00d9", "\u00d8\u00b9\u00d8\u00b1\u00d8\u00a8\u00d9", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1736,7 +1726,7 @@ public class ParseTestCase { @Test public void testFakeCJKSegmentingOfMultiplePhrases() { Item item = tester.parseQuery("name:firstsecond.s", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE); - assertEquals("name:\"'first second' s\"", item.toString()); + assertEquals("AND (SAND name:first name:second) name:s", item.toString()); } @Test @@ -1801,7 +1791,7 @@ public class ParseTestCase { @Test public void testCommaOnlyLeadsToImplicitPhrasing() { - tester.assertParsed("\"A B C\"", "A,B,C", Query.Type.ALL); + tester.assertParsed("AND A B C", "A,B,C", Query.Type.ALL); } @Test @@ -1873,8 +1863,8 @@ public class ParseTestCase { @Test public void testJPMobileExceptionQuery() { - tester.assertParsed("OR concat and \"make string\" 1 47 or", - "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); + tester.assertParsed("OR concat and (AND make string) 1 47 or", + "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL); } @Test @@ -1882,7 +1872,7 @@ public class ParseTestCase { tester.assertParsed("b", "a: b", Query.Type.ALL); tester.assertParsed("AND a b", "a : b", Query.Type.ALL); tester.assertParsed("AND a b", "a :b", Query.Type.ALL); - tester.assertParsed("\"a b\"", "a.:b", Query.Type.ALL); + tester.assertParsed("AND a b", "a.:b", Query.Type.ALL); tester.assertParsed("a:b", "a:b", Query.Type.ALL); } @@ -1917,8 +1907,7 @@ public class ParseTestCase { tester.assertParsed("AND ringtone AND (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ALL); - // The last one here is a little weird, but it's not a problem, - // so I let it pass for now... + // The last one here is a little weird, but it's not a problem, so let it pass for now... tester.assertParsed("OR (OR ringtone AND) (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")", "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )", Query.Type.ANY); @@ -1926,7 +1915,7 @@ public class ParseTestCase { @Test public void testMixedCaseIndexNames() { - tester.assertParsed("AND mixedCase:a mixedCase:b \"notAnIndex c\" mixedCase:d", + tester.assertParsed("AND mixedCase:a mixedCase:b notAnIndex c mixedCase:d", "mixedcase:a MIXEDCASE:b notAnIndex:c mixedCase:d", Query.Type.ALL); } @@ -1934,7 +1923,7 @@ public class ParseTestCase { /** CJK special tokens should be recognized also on non-boundaries */ @Test public void testChineseSpecialTokens() { - tester.assertParsed("AND \"cat tcp/ip zu\" \"foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz\"", + tester.assertParsed("AND cat tcp/ip zu foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz", "cattcp/ipzu foo.netbar.net.netC#c++bar.net.netwiz","", Query.Type.ALL, Language.CHINESE_SIMPLIFIED); } @@ -1945,7 +1934,7 @@ public class ParseTestCase { @Test public void testChineseSpecialTokensWithMultiSegmentReplace() { // special-token-fs is a special token, to be replaced by firstsecond, first and second are segments in test - tester.assertParsed("AND \"tcp/ip firstsecond dotnet\" firstsecond 'first second'","tcp/ipspecial-token-fs.net special-token-fs firstsecond", + tester.assertParsed("AND tcp/ip firstsecond dotnet firstsecond (SAND first second)","tcp/ipspecial-token-fs.net special-token-fs firstsecond", "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, TestLinguistics.INSTANCE); } @@ -2014,7 +2003,7 @@ public class ParseTestCase { @Test public void testVersionNumbers() { - tester.assertParsed("\"1 0 9\"", "1.0.9", Query.Type.ALL); + tester.assertParsed("AND 1 0 9", "1.0.9", Query.Type.ALL); } @Test @@ -2321,7 +2310,7 @@ public class ParseTestCase { @Test public void testOdd1Web() { - tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.WEB); + tester.assertParsed("AND window print error", "+window.print() +error",Query.Type.WEB); } @Test @@ -2351,13 +2340,13 @@ public class ParseTestCase { @Test public void testDefaultWebIndices() { - tester.assertParsed("\"notanindex b\"","notanindex:b",Query.Type.WEB); - tester.assertParsed("site:\"b $\"","site:b",Query.Type.WEB); - tester.assertParsed("hostname:b","hostname:b",Query.Type.WEB); - tester.assertParsed("link:b","link:b",Query.Type.WEB); - tester.assertParsed("url:b","url:b",Query.Type.WEB); - tester.assertParsed("inurl:b","inurl:b",Query.Type.WEB); - tester.assertParsed("intitle:b","intitle:b",Query.Type.WEB); + tester.assertParsed("AND notanindex b","notanindex:b", Query.Type.WEB); + tester.assertParsed("site:\"b $\"","site:b", Query.Type.WEB); + tester.assertParsed("hostname:b","hostname:b", Query.Type.WEB); + tester.assertParsed("link:b","link:b", Query.Type.WEB); + tester.assertParsed("url:b","url:b", Query.Type.WEB); + tester.assertParsed("inurl:b","inurl:b", Query.Type.WEB); + tester.assertParsed("intitle:b","intitle:b", Query.Type.WEB); } @Test @@ -2527,7 +2516,7 @@ public class ParseTestCase { @Test public void testSiteAndSegmentPhrasesFollowedByText() { - tester.assertParsed("AND host.all:\"www abc com x y-z $\" 'a b'", + tester.assertParsed("AND host.all:\"www abc com x y-z $\" (SAND a b)", "host.all:www.abc.com/x'y-z a'b", "", Query.Type.ALL, Language.ENGLISH); } @@ -2544,7 +2533,7 @@ public class ParseTestCase { @Test public void testNonAsciiNumber() { - tester.assertParsed("title:\"199 119 201 149\"", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); + tester.assertParsed("AND title:199 title:119 title:201 title:149", "title:１９９．１１９．２０１．１４９", Query.Type.ALL); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java index 12d993e8d41..aa2e9dbcf75 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -46,7 +46,7 @@ public class TokenizerTestCase { Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); tokenizer.setSpecialTokens(createSpecialTokens()); - List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1"); + List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1 & &a"); assertEquals(new Token(WORD, "drive"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -69,6 +69,11 @@ public class TokenizerTestCase { assertEquals(new Token(WORD, "ugcapi"), tokens.get(18)); assertEquals(new Token(UNDERSCORE, "_"), tokens.get(19)); assertEquals(new Token(NUMBER, "1"), tokens.get(20)); + assertEquals(new Token(SPACE, " "), tokens.get(21)); + assertEquals(new Token(NOISE, "<NOISE>"), tokens.get(22)); + assertEquals(new Token(SPACE, " "), tokens.get(23)); + assertEquals(new Token(NOISE, "<NOISE>"), tokens.get(24)); + assertEquals(new Token(WORD, "a"), tokens.get(25)); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java index 91cf5015cba..0ca4b8aa615 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java @@ -45,7 +45,7 @@ public class CJKSearcherTestCase { @Test public void testCjkQueryWithOverlappingTokens() { // The test language segmenter will segment "bcd" into the overlapping tokens "bc" "cd" - assertTransformed("bcd", "'bc cd'", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, + assertTransformed("bcd", "SAND bc cd", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL, TestLinguistics.INSTANCE); // While "efg" will be segmented into one of the standard options, "e" "fg" diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java index 12e756a07ee..023cd3c2849 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java @@ -71,7 +71,7 @@ public class LiteralBoostSearcherTestCase { @Test public void testQueryWithoutBoost() { - assertEquals("RANK (AND \"nonexistant a\" \"nonexistant b\") default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", + assertEquals("RANK (AND nonexistant a nonexistant b) default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b", transformQuery("?query=nonexistant:a nonexistant:b&source=cluster1&restrict=type1")); } diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java index 11922cf640a..36137abd9b8 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java @@ -1,15 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.querytransform.test; +import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.WordItem; import com.yahoo.prelude.querytransform.QueryRewrite; +import com.yahoo.prelude.querytransform.RecallSearcher; import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -38,9 +45,17 @@ public class QueryRewriteTestCase { assertRewritten(query, "OR sddocname:per foo bar"); ((OrItem)query.getModel().getQueryTree().getRoot()).getItem(2).setRanked(false); // set 'bar' unranked assertRewritten(query, "OR sddocname:per foo"); - assertRewritten("sddocname:per OR foo OR (bar AND fuz)", "per", "OR sddocname:per foo (AND bar fuz)"); + } + @Test + public void testRankContributingTermsAreNotRemovedOnFullRecall() { + Query query = new Query(QueryTestCase.httpEncode("?query=default:term1 OR default:term2 OR default:term3 OR sddocname:per&type=adv&recall=+id:1&restrict=per")); + RecallSearcher searcher = new RecallSearcher(); + Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query); + assertNull(result.hits().getError()); + assertNull(QueryCanonicalizer.canonicalize(query)); + assertRewritten(query, "AND (OR default:term1 default:term2 default:term3 sddocname:per) |id:1"); } @Test @@ -88,6 +103,7 @@ public class QueryRewriteTestCase { private static void assertRewritten(Query query, String expectedOptimizedQuery) { QueryRewrite.optimizeByRestrict(query); + QueryRewrite.optimizeAndNot(query); QueryRewrite.collapseSingleComposites(query); assertEquals(expectedOptimizedQuery, query.getModel().getQueryTree().toString()); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index 9da1c184505..0086f1b3571 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -108,7 +108,7 @@ public class BlendingSearcherTestCase { entry.getValue())); } - StrictContractsConfig contracts = new StrictContractsConfig(new StrictContractsConfig.Builder()); + StrictContractsConfig contracts = new StrictContractsConfig.Builder().build(); FederationSearcher fedSearcher = new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<>()); @@ -124,7 +124,6 @@ public class BlendingSearcherTestCase { @Test public void testitTwoPhase() { - DocumentSourceSearcher chain1 = new DocumentSourceSearcher(); DocumentSourceSearcher chain2 = new DocumentSourceSearcher(); DocumentSourceSearcher chain3 = new DocumentSourceSearcher(); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java index 4875121a501..12619bf0a5e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -40,34 +40,8 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") public class FieldCollapsingSearcherTestCase { - private FastHit createHit(String uri,int relevancy,int mid) { - FastHit hit = new FastHit(uri,relevancy); - hit.setField("amid", String.valueOf(mid)); - return hit; - } - - private void assertHit(String uri,int relevancy,int mid,Hit hit) { - assertEquals(uri,hit.getId().toString()); - assertEquals(relevancy, ((int) hit.getRelevance().getScore())); - assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); - } - - private static class ZeroHitsControl extends com.yahoo.search.Searcher { - public int queryCount = 0; - public com.yahoo.search.Result search(com.yahoo.search.Query query, - com.yahoo.search.searchchain.Execution execution) { - ++queryCount; - if (query.getHits() == 0) { - return new Result(query); - } else { - return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); - } - } - } - @Test public void testFieldCollapsingWithoutHits() { // Set up @@ -116,14 +90,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -133,10 +107,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -152,14 +126,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -169,10 +143,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10,0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9,1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8,2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7,3, r.hits().get(3)); } @Test @@ -185,14 +159,14 @@ public class FieldCollapsingSearcherTestCase { Query q = new Query("?query=test_collapse"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -220,16 +194,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,0)); - r.hits().add(createHit("http://acme.org/d.html", 8,0)); - r.hits().add(createHit("http://acme.org/e.html", 8,0)); - r.hits().add(createHit("http://acme.org/f.html", 7,0)); - r.hits().add(createHit("http://acme.org/g.html", 7,0)); - r.hits().add(createHit("http://acme.org/h.html", 6,0)); - r.hits().add(createHit("http://acme.org/i.html", 5,1)); - r.hits().add(createHit("http://acme.org/j.html", 4,2)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 0)); + r.hits().add(createHit("http://acme.org/d.html", 8, 0)); + r.hits().add(createHit("http://acme.org/e.html", 8, 0)); + r.hits().add(createHit("http://acme.org/f.html", 7, 0)); + r.hits().add(createHit("http://acme.org/g.html", 7, 0)); + r.hits().add(createHit("http://acme.org/h.html", 6, 0)); + r.hits().add(createHit("http://acme.org/i.html", 5, 1)); + r.hits().add(createHit("http://acme.org/j.html", 4, 2)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -239,15 +213,15 @@ public class FieldCollapsingSearcherTestCase { assertEquals(2, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/i.html", 5,1,r.hits().get(1)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/i.html", 5, 1, r.hits().get(1)); // Next results docsource.resetQueryCount(); r = doSearch(collapse, q, 2, 2, chained); assertEquals(1, r.getHitCount()); assertEquals(2, docsource.getQueryCount()); - assertHit("http://acme.org/j.html",4,2,r.hits().get(0)); + assertHit("http://acme.org/j.html",4, 2, r.hits().get(0)); } /** @@ -265,16 +239,16 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,1)); - r.hits().add(createHit("http://acme.org/b.html",10,1)); - r.hits().add(createHit("http://acme.org/c.html",10,0)); - r.hits().add(createHit("http://acme.org/d.html",10,0)); - r.hits().add(createHit("http://acme.org/e.html",10,0)); - r.hits().add(createHit("http://acme.org/f.html",10,0)); - r.hits().add(createHit("http://acme.org/g.html",10,0)); - r.hits().add(createHit("http://acme.org/h.html",10,0)); - r.hits().add(createHit("http://acme.org/i.html",10,0)); - r.hits().add(createHit("http://acme.org/j.html",10,1)); + r.hits().add(createHit("http://acme.org/a.html", 10, 1)); + r.hits().add(createHit("http://acme.org/b.html", 10, 1)); + r.hits().add(createHit("http://acme.org/c.html", 10, 0)); + r.hits().add(createHit("http://acme.org/d.html", 10, 0)); + r.hits().add(createHit("http://acme.org/e.html", 10, 0)); + r.hits().add(createHit("http://acme.org/f.html", 10, 0)); + r.hits().add(createHit("http://acme.org/g.html", 10, 0)); + r.hits().add(createHit("http://acme.org/h.html", 10, 0)); + r.hits().add(createHit("http://acme.org/i.html", 10, 0)); + r.hits().add(createHit("http://acme.org/j.html", 10, 1)); r.setTotalHitCount(10); docsource.addResult(q, r); @@ -287,17 +261,6 @@ public class FieldCollapsingSearcherTestCase { assertHit("http://acme.org/c.html",10,0,r.hits().get(1)); } - public static class QueryMessupSearcher extends Searcher { - public Result search(com.yahoo.search.Query query, Execution execution) { - AndItem a = new AndItem(); - a.addItem(query.getModel().getQueryTree().getRoot()); - a.addItem(new WordItem("b")); - query.getModel().getQueryTree().setRoot(a); - - return execution.search(query); - } - } - @Test public void testQueryTransformAndCollapsing() { // Set up @@ -309,9 +272,9 @@ public class FieldCollapsingSearcherTestCase { chained.put(collapse, messUp); chained.put(messUp, docsource); - // Caveat: Collapse is set to false, because that's what the - // collapser asks for - Query q = new Query("?query=test_collapse+b&collapsefield=amid"); + // Caveat: Collapse is set to false, because that's what the collapser asks for + Query q = new Query("?query=%22test%20collapse%22+b&collapsefield=amid"); + System.out.println(q); // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); @@ -327,13 +290,13 @@ public class FieldCollapsingSearcherTestCase { docsource.addResult(q, r); // Test basic collapsing on mid - q = new Query("?query=test_collapse&collapsefield=amid"); + q = new Query("?query=%22test%20collapse%22&collapsefield=amid"); r = doSearch(collapse, q, 0, 2, chained); assertEquals(2, docsource.getQueryCount()); assertEquals(2, r.getHitCount()); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/h.html", 6,1,r.hits().get(1)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/h.html", 6, 1, r.hits().get(1)); } @Test @@ -367,10 +330,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(4, r.getHitCount()); assertEquals(1, docsource.getQueryCount()); assertTrue(r.isFilled("placeholder")); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); docsource.resetQueryCount(); // Test basic collapsing on mid @@ -381,10 +344,10 @@ public class FieldCollapsingSearcherTestCase { assertEquals(1, docsource.getQueryCount()); assertFalse(r.isFilled("placeholder")); assertTrue(r.isFilled("short")); - assertHit("http://acme.org/a.html",10,0,r.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,r.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,r.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,r.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, r.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3)); } @Test @@ -400,14 +363,14 @@ public class FieldCollapsingSearcherTestCase { // The searcher turns off collapsing further on in the chain q.properties().set("collapse", "0"); Result r = new Result(q); - r.hits().add(createHit("http://acme.org/a.html",10,0)); - r.hits().add(createHit("http://acme.org/b.html", 9,0)); - r.hits().add(createHit("http://acme.org/c.html", 9,1)); - r.hits().add(createHit("http://acme.org/d.html", 8,1)); - r.hits().add(createHit("http://acme.org/e.html", 8,2)); - r.hits().add(createHit("http://acme.org/f.html", 7,2)); - r.hits().add(createHit("http://acme.org/g.html", 7,3)); - r.hits().add(createHit("http://acme.org/h.html", 6,3)); + r.hits().add(createHit("http://acme.org/a.html",10, 0)); + r.hits().add(createHit("http://acme.org/b.html", 9, 0)); + r.hits().add(createHit("http://acme.org/c.html", 9, 1)); + r.hits().add(createHit("http://acme.org/d.html", 8, 1)); + r.hits().add(createHit("http://acme.org/e.html", 8, 2)); + r.hits().add(createHit("http://acme.org/f.html", 7, 2)); + r.hits().add(createHit("http://acme.org/g.html", 7, 3)); + r.hits().add(createHit("http://acme.org/h.html", 6, 3)); r.setTotalHitCount(8); docsource.addResult(q, r); @@ -416,29 +379,28 @@ public class FieldCollapsingSearcherTestCase { Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); // Assert that the regular hits are collapsed - assertEquals(4+1, result.getHitCount()); + assertEquals(4 + 1, result.getHitCount()); assertEquals(1, docsource.getQueryCount()); - assertHit("http://acme.org/a.html",10,0,result.hits().get(0)); - assertHit("http://acme.org/c.html", 9,1,result.hits().get(1)); - assertHit("http://acme.org/e.html", 8,2,result.hits().get(2)); - assertHit("http://acme.org/g.html", 7,3,result.hits().get(3)); + assertHit("http://acme.org/a.html",10, 0, result.hits().get(0)); + assertHit("http://acme.org/c.html", 9, 1, result.hits().get(1)); + assertHit("http://acme.org/e.html", 8, 2, result.hits().get(2)); + assertHit("http://acme.org/g.html", 7, 3, result.hits().get(3)); // Assert that the aggregation group hierarchy is left intact - HitGroup root= getFirstGroupIn(result.hits()); + HitGroup root = getFirstGroupIn(result.hits()); assertNotNull(root); - assertEquals("group:root:",root.getId().stringValue().substring(0,11)); // The id ends by a global counter currently - assertEquals(1,root.size()); - HitGroup groupList= (GroupList)root.get("grouplist:g1"); + assertEquals("group:root:",root.getId().stringValue().substring(0, 11)); // The id ends by a global counter currently + assertEquals(1, root.size()); + HitGroup groupList = (GroupList)root.get("grouplist:g1"); assertNotNull(groupList); - assertEquals(1,groupList.size()); - HitGroup group= (HitGroup)groupList.get("group:long:37"); + assertEquals(1, groupList.size()); + HitGroup group = (HitGroup)groupList.get("group:long:37"); assertNotNull(group); } private Group getFirstGroupIn(HitGroup hits) { - for (Hit h : hits) { + for (Hit h : hits) if (h instanceof Group) return (Group)h; - } return null; } @@ -450,9 +412,8 @@ public class FieldCollapsingSearcherTestCase { private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain, Map<Searcher, Searcher> chained) { List<Searcher> searchers = new ArrayList<>(); - for (Searcher current = topOfChain; current != null; current = chained.get(current)) { + for (Searcher current = topOfChain; current != null; current = chained.get(current)) searchers.add(current); - } return new Chain<>(searchers); } @@ -470,7 +431,7 @@ public class FieldCollapsingSearcherTestCase { @Override public Result search(Query query, Execution execution) { - Result r=execution.search(query); + Result r = execution.search(query); r.hits().add(createAggregationGroup("g1")); return r; } @@ -479,10 +440,51 @@ public class FieldCollapsingSearcherTestCase { Group root = new Group(new RootId(0), new Relevance(1)); GroupList groupList = new GroupList(label); root.add(groupList); - Group value=new Group(new LongId(37l),new Relevance(2.11)); + Group value = new Group(new LongId(37l), new Relevance(2.11)); groupList.add(value); return root; } } + private FastHit createHit(String uri,int relevancy,int mid) { + FastHit hit = new FastHit(uri,relevancy); + hit.setField("amid", String.valueOf(mid)); + return hit; + } + + private void assertHit(String uri,int relevancy,int mid,Hit hit) { + assertEquals(uri,hit.getId().toString()); + assertEquals(relevancy, ((int) hit.getRelevance().getScore())); + assertEquals(mid,Integer.parseInt((String) hit.getField("amid"))); + } + + private static class ZeroHitsControl extends com.yahoo.search.Searcher { + + public int queryCount = 0; + + @Override + public Result search(Query query, Execution execution) { + ++queryCount; + if (query.getHits() == 0) { + return new Result(query); + } else { + return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits.")); + } + } + } + + public static class QueryMessupSearcher extends Searcher { + + @Override + public Result search(com.yahoo.search.Query query, Execution execution) { + AndItem a = new AndItem(); + a.addItem(query.getModel().getQueryTree().getRoot()); + a.addItem(new WordItem("b")); + query.getModel().getQueryTree().setRoot(a); + + return execution.search(query); + } + + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java index e2973f8ed65..aa48e8494f2 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java @@ -118,6 +118,13 @@ public class PosSearcherTestCase { q.properties().set("pos.units", "udeg"); doSearch(searcher, q, 0, 10); assertEquals("(2,0,0,18026,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + + q = new Query(); + q.properties().set("pos.ll", "N0;E0"); + q.properties().set("pos.radius", "-1"); + doSearch(searcher, q, 0, 10); + assertEquals("(2,0,0,-1,0,1,0,4294967295)", q.getRanking().getLocation().toString()); + assertEquals("(2,0,0,536870912,0,1,0,4294967295)", q.getRanking().getLocation().backendString()); } /** @@ -128,13 +135,13 @@ public class PosSearcherTestCase { Query q = new Query(); q.properties().set("pos.xy", "22500;22500"); doSearch(searcher, q, 0, 10); - assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString()); + assertEquals("(2,22500,22500,450668,0,1,0)", q.getRanking().getLocation().toString()); q = new Query(); q.properties().set("pos.xy", "22500;22500"); q.properties().set("pos.units", "unknown"); doSearch(searcher, q, 0, 10); - assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString()); + assertEquals("(2,22500,22500,450668,0,1,0)", q.getRanking().getLocation().toString()); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java index 1b9ca1cd29b..2187cb89ae2 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.prelude.searcher.test; import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; @@ -20,8 +19,6 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.YqlParser; import org.junit.Test; -import java.util.*; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -46,6 +43,14 @@ public class ValidatePredicateSearcherTestCase { assertEquals(ErrorMessage.createIllegalQuery("age=200 outside configured predicate bounds."), r.hits().getError()); } + @Test + public void queryFailsWhenPredicateFieldIsUsedInTermSearch() { + ValidatePredicateSearcher searcher = new ValidatePredicateSearcher(); + String q = "select * from sources * where predicate_field CONTAINS \"true\";"; + Result r = doSearch(searcher, q, "predicate-bounds [0..99]"); + assertEquals(ErrorMessage.createIllegalQuery("Index 'predicate_field' is predicate attribute and can only be used in conjunction with a predicate query operator."), r.hits().getError()); + } + private static Result doSearch(ValidatePredicateSearcher searcher, String yqlQuery, String command) { QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery)); Query query = new Query(); @@ -53,6 +58,7 @@ public class ValidatePredicateSearcherTestCase { SearchDefinition searchDefinition = new SearchDefinition("document"); Index index = new Index("predicate_field"); + index.setPredicate(true); index.addCommand(command); searchDefinition.addIndex(index); IndexFacts indexFacts = new IndexFacts(new IndexModel(searchDefinition)); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java index 65011ffb562..f4bf957e29a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java @@ -52,6 +52,7 @@ public class ValidateSortingSearcherTestCase { assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[rank]")); assertEquals("[ASCENDING:[docid]]", quoteAndTransform("+[docid]")); assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevancy]")); + assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevance]")); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java index b8db5e4d90f..a4cf7d8c380 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java @@ -23,7 +23,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"first third\"", q); + assertSemantics("AND first third", q); } @Test @@ -32,7 +32,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"bc first third fg\"", q); + assertSemantics("AND bc first third fg", q); } @Test @@ -41,7 +41,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("+bc -\"first third\"", q); + assertSemantics("+bc -(AND first third)", q); } @Test @@ -50,7 +50,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0"); q.getModel().getQueryTree().setRoot(a); - assertSemantics("\"9 2 7 0 bc third 2 3 8 9\"", q); + assertSemantics("AND 9 2 7 0 bc third 2 3 8 9", q); } private static Item parseQuery(String query) { diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index 82a5a0c7a24..e2ac44316e7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -73,7 +73,7 @@ public class IndexFactsTestCase { Query q = newQuery("?query=a:b", indexFacts); assertEquals("a:b", q.getModel().getQueryTree().getRoot().toString()); q = newQuery("?query=notarealindex:b", indexFacts); - assertEquals("\"notarealindex b\"", q.getModel().getQueryTree().getRoot().toString()); + assertEquals("AND notarealindex b", q.getModel().getQueryTree().getRoot().toString()); } @Test @@ -302,8 +302,8 @@ public class IndexFactsTestCase { IndexFacts.Session session2 = indexFacts.newSession(query2.getModel().getSources(), query2.getModel().getRestrict()); assertTrue(session1.getIndex("url").isUriIndex()); assertTrue(session2.getIndex("url").isUriIndex()); - assertEquals("url:\"https foo bar\"", query1.getModel().getQueryTree().toString()); - assertEquals("url:\"https foo bar\"", query2.getModel().getQueryTree().toString()); + assertEquals("AND url:https url:foo url:bar", query1.getModel().getQueryTree().toString()); + assertEquals("AND url:https url:foo url:bar", query2.getModel().getQueryTree().toString()); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index 7b3327c04af..4b2ced1b771 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -39,16 +39,16 @@ public class QueryTestCase { public void testSimpleQueryParsing () { Query q = newQuery("/search?query=foobar&offset=10&hits=20"); assertEquals("foobar",((WordItem) q.getModel().getQueryTree().getRoot()).getWord()); - assertEquals(10,q.getOffset()); - assertEquals(20,q.getHits()); + assertEquals(10, q.getOffset()); + assertEquals(20, q.getHits()); } @Test public void testAdvancedQueryParsing () { Query q = newQuery("/search?query=fOObar and kanoo&offset=10&hits=20&filter=-foo +bar&type=adv&suggestonly=true"); assertEquals("AND (+(AND fOObar kanoo) -|foo) |bar", q.getModel().getQueryTree().getRoot().toString()); - assertEquals(10,q.getOffset()); - assertEquals(20,q.getHits()); + assertEquals(10, q.getOffset()); + assertEquals(20, q.getHits()); assertEquals(true, q.properties().getBoolean("suggestonly", false)); } @@ -56,10 +56,10 @@ public class QueryTestCase { public void testAnyQueryParsing () { Query q = newQuery("/search?query=foobar and kanoo&offset=10&hits=10&type=any&suggestonly=true&filter=-fast.type:offensive&encoding=latin1"); assertEquals("+(OR foobar and kanoo) -|fast.type:offensive", q.getModel().getQueryTree().getRoot().toString()); - assertEquals(10,q.getOffset()); - assertEquals(10,q.getHits()); + assertEquals(10, q.getOffset()); + assertEquals(10, q.getHits()); assertEquals(true, q.properties().getBoolean("suggestonly", false)); - assertEquals("latin1",q.getModel().getEncoding()); + assertEquals("latin1", q.getModel().getEncoding()); } @Test @@ -71,8 +71,8 @@ public class QueryTestCase { +"interest:www+yahoo+com!136" +"&hits=20&offset=0&vectorranking=queryrank"); assertEquals("/p13n", q.getHttpRequest().getUri().getPath()); - assertEquals(0,q.getOffset()); - assertEquals(20,q.getHits()); + assertEquals(0, q.getOffset()); + assertEquals(20, q.getHits()); assertEquals("queryrank", q.properties().get("vectorranking")); } @@ -84,7 +84,7 @@ public class QueryTestCase { @Test public void testGetParamInt() { Query q = newQuery("/search?query=foo%20bar&someint=10¬int=hello"); - assertEquals(10,(int)q.properties().getInteger("someint")); + assertEquals(10, (int)q.properties().getInteger("someint")); // provoke an exception. if exception is not triggered // we fail the test. @@ -99,7 +99,7 @@ public class QueryTestCase { @Test public void testUtf8Decoding() { Query q = new Query("/?query=beyonc%C3%A9"); - assertEquals("beyonc\u00e9",((WordItem) q.getModel().getQueryTree().getRoot()).getWord()); + assertEquals("beyonc\u00e9", ((WordItem) q.getModel().getQueryTree().getRoot()).getWord()); } @Test @@ -226,14 +226,14 @@ public class QueryTestCase { public void testNaNHitValue() { assertQueryError( "?query=test&hits=NaN", - containsString("Could not set 'hits' to 'NaN': Not a valid integer")); + containsString("Could not set 'hits' to 'NaN': 'NaN' is not a valid integer")); } @Test public void testNoneHitValue() { assertQueryError( "?query=test&hits=(none)", - containsString("Could not set 'hits' to '(none)': Not a valid integer")); + containsString("Could not set 'hits' to '(none)': '(none)' is not a valid integer")); } @Test @@ -247,14 +247,14 @@ public class QueryTestCase { public void testNaNOffsetValue() { assertQueryError( "?query=test&offset=NaN", - containsString("Could not set 'offset' to 'NaN': Not a valid integer")); + containsString("Could not set 'offset' to 'NaN': 'NaN' is not a valid integer")); } @Test public void testNoneOffsetValue() { assertQueryError( "?query=test&offset=(none)", - containsString("Could not set 'offset' to '(none)': Not a valid integer")); + containsString("Could not set 'offset' to '(none)': '(none)' is not a valid integer")); } @Test @@ -263,7 +263,7 @@ public class QueryTestCase { "?query=test&hits=(none)&offset=-10", anyOf( containsString("Could not set 'offset' to '-10': Must be a positive number"), - containsString("Could not set 'hits' to '(none)': Not a valid integer"))); + containsString("Could not set 'hits' to '(none)': '(none)' is not a valid integer"))); } @Test @@ -271,7 +271,7 @@ public class QueryTestCase { assertQueryError( "?query=test&hits=(none)&offset=-10", anyOf( - containsString("Could not set 'hits' to '(none)': Not a valid integer"), + containsString("Could not set 'hits' to '(none)': '(none)' is not a valid integer"), containsString("Could not set 'offset' to '-10': Must be a positive number"))); } @@ -325,7 +325,7 @@ public class QueryTestCase { query.getRanking().setFreshness("now"); assertTrue(query.getRanking().getFreshness().getSystemTimeInSecondsSinceEpoch() >= query.getRanking().getFreshness().getRefTime()); - int presize= query.errors().size(); + int presize = query.errors().size(); query.getRanking().setFreshness("sometimeslater"); int postsize = query.errors().size(); @@ -335,12 +335,12 @@ public class QueryTestCase { @Test public void testCopy() { Query qs = newQuery("?query=test&rankfeature.something=2"); - assertEquals("test",qs.getModel().getQueryTree().toString()); + assertEquals("test", qs.getModel().getQueryTree().toString()); assertEquals((int)qs.properties().getInteger("rankfeature.something"),2); Query qp = new Query(qs); assertEquals("test", qp.getModel().getQueryTree().getRoot().toString()); assertFalse(qp.getRanking().getFeatures().isEmpty()); - assertEquals("2", qp.getRanking().getFeatures().get("something")); + assertEquals(2.0, qp.getRanking().getFeatures().getDouble("something").getAsDouble(), 0.000001); } private Query newQuery(String queryString) { diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java index 2992d8ab896..8dcc25e4b3b 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java @@ -56,11 +56,10 @@ public class ClusterSearcherTestCase { @Override public Pong ping(Ping ping, Execution execution) { - Pong pong = new Pong(); - if (isBlocking()) { - pong.addError(ErrorMessage.createTimeout("Dummy timeout")); - } - return new Pong(); + if (isBlocking()) + return new Pong(ErrorMessage.createTimeout("Dummy timeout")); + else + return new Pong(); } public boolean isBlocking() { diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index c90d2774bd1..a824edd1996 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -182,10 +182,10 @@ public class ClusteredConnectionTestCase { @Override public Pong ping(Ping ping,Connection connection) { - Pong pong = new Pong(); - if (connection.getResponse() == null) - pong.addError(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); - return pong; + if (connection.getResponse() != null) + return new Pong(); + else + return new Pong(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 291b0f4890a..eaafb2d8b8a 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -1,22 +1,21 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.prelude.fastsearch.test.MockMetric; -import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import org.junit.Test; import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.concurrent.Callable; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; import static org.junit.Assert.assertEquals; @@ -36,12 +35,13 @@ public class DispatcherTest { q.getModel().setSearchPath("1/0"); // second node in first group MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (nodes, a) -> { assertEquals(1, nodes.size()); - assertEquals(2, nodes.get(0).key()); + assertEquals(1, nodes.get(0).key()); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(q, null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -53,9 +53,10 @@ public class DispatcherTest { } }; MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -69,9 +70,10 @@ public class DispatcherTest { assertTrue(acceptIncompleteCoverage); return true; }); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); SearchInvoker invoker = disp.getSearchInvoker(new Query(), null); invokerFactory.verifyAllEventsProcessed(); + disp.deconstruct(); } @Test @@ -80,8 +82,9 @@ public class DispatcherTest { SearchCluster cl = new MockSearchCluster("1", 2, 1); MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false); - Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric()); + Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric()); disp.getSearchInvoker(new Query(), null); + disp.deconstruct(); fail("Expected exception"); } catch (IllegalStateException e) { @@ -89,6 +92,53 @@ public class DispatcherTest { } } + @Test + public void testGroup0IsSelected() { + SearchCluster cluster = new MockSearchCluster("1", 3, 1); + Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric()); + cluster.pingIterationCompleted(); + assertEquals(0, + dispatcher.getSearchInvoker(new Query(), null).distributionKey().get().longValue()); + dispatcher.deconstruct(); + } + + @Test + public void testGroup0IsSkippedWhenItIsBlockingFeed() { + SearchCluster cluster = new MockSearchCluster("1", 3, 1); + Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric()); + cluster.group(0).get().nodes().get(0).setBlockingWrites(true); + cluster.pingIterationCompleted(); + assertEquals("Blocking group is avoided", + 1, + (dispatcher.getSearchInvoker(new Query(), null).distributionKey().get()).longValue()); + dispatcher.deconstruct(); + } + + @Test + public void testGroup0IsSelectedWhenMoreAreBlockingFeed() { + SearchCluster cluster = new MockSearchCluster("1", 3, 1); + Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric()); + cluster.group(0).get().nodes().get(0).setBlockingWrites(true); + cluster.group(1).get().nodes().get(0).setBlockingWrites(true); + cluster.pingIterationCompleted(); + assertEquals("Blocking group is used when multiple groups are blocking", + 0, + dispatcher.getSearchInvoker(new Query(), null).distributionKey().get().longValue()); + dispatcher.deconstruct(); + } + + @Test + public void testGroup0IsSelectedWhenItIsBlockingFeedWhenNoOthers() { + SearchCluster cluster = new MockSearchCluster("1", 1, 1); + Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric()); + cluster.group(0).get().nodes().get(0).setBlockingWrites(true); + cluster.pingIterationCompleted(); + assertEquals("Blocking group is used when there is no alternative", + 0, + (dispatcher.getSearchInvoker(new Query(), null).distributionKey().get()).longValue()); + dispatcher.deconstruct(); + } + interface FactoryStep { boolean returnInvoker(List<Node> nodes, boolean acceptIncompleteCoverage); } @@ -108,14 +158,15 @@ public class DispatcherTest { Query query, OptionalInt groupId, List<Node> nodes, - boolean acceptIncompleteCoverage) { + boolean acceptIncompleteCoverage, + int maxHitsPerNode) { if (step >= events.length) { throw new RuntimeException("Was not expecting more calls to getSearchInvoker"); } boolean nonEmpty = events[step].returnInvoker(nodes, acceptIncompleteCoverage); step++; if (nonEmpty) { - return Optional.of(new MockInvoker(1)); + return Optional.of(new MockInvoker(nodes.get(0).key())); } else { return Optional.empty(); } @@ -126,7 +177,10 @@ public class DispatcherTest { } @Override - protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, Query query, Node node) { + protected Optional<SearchInvoker> createNodeSearchInvoker(VespaBackEndSearcher searcher, + Query query, + int maxHitsPerNode, + Node node) { fail("Unexpected call to createNodeSearchInvoker"); return null; } @@ -138,9 +192,10 @@ public class DispatcherTest { } @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { fail("Unexpected call to createPinger"); return null; } } + } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 27685426cf8..2bfa778a2ba 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -204,6 +204,33 @@ public class InterleavedSearchInvokerTest { private static final List<Double> A5Aux = Arrays.asList(-1.0,11.0,8.5,7.5,-7.0,3.0,2.0); private static final List<Double> B5Aux = Arrays.asList(9.0,8.0,-3.0,7.0,6.0,1.0, -1.0); + private void validateThatTopKProbabilityOverrideTakesEffect(Double topKProbability, int expectedK) throws IOException { + InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5); + query.setHits(8); + query.properties().set(Dispatcher.topKProbability, topKProbability); + SearchInvoker [] invokers = invoker.invokers().toArray(new SearchInvoker[0]); + Result result = invoker.search(query, null); + assertEquals(2, invokers.length); + assertEquals(expectedK, ((MockInvoker)invokers[0]).hitsRequested); + assertEquals(8, result.hits().size()); + assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA); + assertEquals(9.0, result.hits().get(1).getRelevance().getScore(), DELTA); + assertEquals(8.5, result.hits().get(2).getRelevance().getScore(), DELTA); + assertEquals(8.0, result.hits().get(3).getRelevance().getScore(), DELTA); + assertEquals(7.5, result.hits().get(4).getRelevance().getScore(), DELTA); + assertEquals(7.0, result.hits().get(5).getRelevance().getScore(), DELTA); + assertEquals(6.0, result.hits().get(6).getRelevance().getScore(), DELTA); + assertEquals(3.0, result.hits().get(7).getRelevance().getScore(), DELTA); + assertEquals(0, result.getQuery().getOffset()); + assertEquals(8, result.getQuery().getHits()); + } + + @Test + public void requireThatTopKProbabilityOverrideTakesEffect() throws IOException { + validateThatTopKProbabilityOverrideTakesEffect(null, 8); + validateThatTopKProbabilityOverrideTakesEffect(0.8, 7); + } + @Test public void requireThatMergeOfConcreteHitsObeySorting() throws IOException { InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java index 085a9b24993..8d81c5d8521 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java @@ -37,10 +37,14 @@ public class LeanHitTest { } @Test public void testOrderingBySortData() { - assertEquals(0, new LeanHit(gidA, 0, 0, gidA).compareTo(new LeanHit(gidA, 0, 0, gidA))); - verifyTransitiveOrdering(new LeanHit(gidA, 0, 0, gidA), - new LeanHit(gidA, 0, 0, gidB), - new LeanHit(gidA, 0, 0, gidC)); + assertEquals(0, new LeanHit(gidA, 0, 0, 0.0, gidA).compareTo(new LeanHit(gidA, 0, 0, 0.0, gidA))); + verifyTransitiveOrdering(new LeanHit(gidA, 0, 0, 0.0, gidA), + new LeanHit(gidA, 0, 0, 0.0, gidB), + new LeanHit(gidA, 0, 0, 0.0, gidC)); + } + @Test + public void testRelevanceIsKeptEvenWithBySortData() { + assertEquals(1.3, new LeanHit(gidA, 0, 0, 1.3, gidA).getRelevance(), 0.0); } @Test public void testNaN2negativeInfinity() { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 0496194f8ed..36b476e2936 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -29,7 +29,7 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -43,7 +43,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -59,7 +59,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 0); Node n3 = new Node(0, "test-node3", 1); Node n4 = new Node(1, "test-node4", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -70,7 +70,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null,null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java index c5fbda7c2f5..459dcc83ab0 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -17,6 +17,7 @@ class MockInvoker extends SearchInvoker { private final Coverage coverage; private Query query; private List<Hit> hits; + int hitsRequested; protected MockInvoker(int key, Coverage coverage) { super(Optional.of(new Node(key, "?", 0))); @@ -33,8 +34,10 @@ class MockInvoker extends SearchInvoker { } @Override - protected void sendSearchRequest(Query query) throws IOException { + protected Object sendSearchRequest(Query query, Object context) throws IOException { this.query = query; + hitsRequested = query.getHits(); + return context; } @Override diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 0bcc30d9b10..4afb6186c60 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -18,6 +18,7 @@ import java.util.Optional; * @author ollivir */ public class MockSearchCluster extends SearchCluster { + private final int numGroups; private final int numNodesPerGroup; private final ImmutableList<Group> orderedGroups; @@ -29,20 +30,19 @@ public class MockSearchCluster extends SearchCluster { } public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { - super(clusterId, dispatchConfig, 1, null); + super(clusterId, dispatchConfig, null, null); ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder(); - int dk = 1; + int distributionKey = 0; for (int group = 0; group < groups; group++) { List<Node> nodes = new ArrayList<>(); for (int node = 0; node < nodesPerGroup; node++) { - Node n = new Node(dk, "host" + dk, group); - n.setWorking(true); + Node n = new Node(distributionKey, "host" + distributionKey, group); nodes.add(n); hostBuilder.put(n.hostname(), n); - dk++; + distributionKey++; } Group g = new Group(group, nodes); groupBuilder.put(group, g); @@ -120,6 +120,7 @@ public class MockSearchCluster extends SearchCluster { builder.minGroupCoverage(99.0); builder.maxNodesDownPerGroup(0); builder.minSearchCoverage(minSearchCoverage); + builder.distributionPolicy(DispatchConfig.DistributionPolicy.Enum.ROUNDROBIN); if (minSearchCoverage < 100.0) { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); @@ -130,4 +131,5 @@ public class MockSearchCluster extends SearchCluster { } return new DispatchConfig(builder); } + } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java index 5a4457780e2..58042dcf228 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertThat; * @author ollivir */ public class SearchPathTest { + @Test public void requreThatSearchPathsAreParsedCorrectly() { assertThat(SearchPath.fromString("0/0").get().toString(), equalTo("0/0")); @@ -71,11 +72,11 @@ public class SearchPathTest { public void searchPathMustFilterNodesBasedOnDefinition() { MockSearchCluster cluster = new MockSearchCluster("a",3, 3); - assertThat(distKeysAsString(SearchPath.selectNodes("1/1", cluster)), equalTo("5")); - assertThat(distKeysAsString(SearchPath.selectNodes("/1", cluster)), equalTo("4,5,6")); - assertThat(distKeysAsString(SearchPath.selectNodes("0,1/2", cluster)), equalTo("7,8")); - assertThat(distKeysAsString(SearchPath.selectNodes("[1,3>/1", cluster)), equalTo("5,6")); - assertThat(distKeysAsString(SearchPath.selectNodes("[1,88>/1", cluster)), equalTo("5,6")); + assertThat(distKeysAsString(SearchPath.selectNodes("1/1", cluster)), equalTo("4")); + assertThat(distKeysAsString(SearchPath.selectNodes("/1", cluster)), equalTo("3,4,5")); + assertThat(distKeysAsString(SearchPath.selectNodes("0,1/2", cluster)), equalTo("6,7")); + assertThat(distKeysAsString(SearchPath.selectNodes("[1,3>/1", cluster)), equalTo("4,5")); + assertThat(distKeysAsString(SearchPath.selectNodes("[1,88>/1", cluster)), equalTo("4,5")); } private static String distKeysAsString(Collection<Node> nodes) { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java new file mode 100644 index 00000000000..795c7cfef20 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java @@ -0,0 +1,168 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import org.junit.Test; + +import java.util.Locale; + +import static org.junit.Assert.assertEquals; + +public class TopKEstimatorTest { + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbability() { + TopKEstimator estimator = new TopKEstimator(30, 0.999); + assertEquals(91.97368471911312, estimator.estimateExactK(200, 3), 0.0); + assertEquals(92, estimator.estimateK(200, 3)); + assertEquals(37.96328109101396, estimator.estimateExactK(200, 10), 0.0); + assertEquals(38, estimator.estimateK(200, 10)); + assertEquals(23.815737601023095, estimator.estimateExactK(200, 20), 0.0); + assertEquals(24, estimator.estimateK(200, 20)); + + assertEquals(37.96328109101396, estimator.estimateExactK(200, 10, 0.999), 0.0); + assertEquals(38, estimator.estimateK(200, 10, 0.999)); + assertEquals(34.36212304875885, estimator.estimateExactK(200, 10, 0.99), 0.0); + assertEquals(35, estimator.estimateK(200, 10, 0.99)); + assertEquals(41.44244358524574, estimator.estimateExactK(200, 10, 0.9999), 0.0); + assertEquals(42, estimator.estimateK(200, 10, 0.9999)); + assertEquals(44.909040374464155, estimator.estimateExactK(200, 10, 0.99999), 0.0); + assertEquals(45, estimator.estimateK(200, 10, 0.99999)); + } + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K200() { + TopKEstimator estimator = new TopKEstimator(30, 0.99999); + assertEquals(200, estimator.estimateExactK(200, 1), 0.0); + assertEquals(200, estimator.estimateK(200, 1)); + assertEquals(137.4727798056239, estimator.estimateExactK(200, 2), 0.0); + assertEquals(102.95409291533568, estimator.estimateExactK(200, 3), 0.0); + assertEquals(44.909040374464155, estimator.estimateExactK(200, 10), 0.0); + assertEquals(28.86025772029091, estimator.estimateExactK(200, 20), 0.0); + } + + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K20() { + TopKEstimator estimator = new TopKEstimator(30, 0.99999); + assertEquals(20, estimator.estimateExactK(20, 1), 0.0); + assertEquals(20, estimator.estimateK(20, 1)); + assertEquals(21.849933444373328, estimator.estimateExactK(20, 2), 0.0); + assertEquals(18.14175840378403, estimator.estimateExactK(20, 3), 0.0); + assertEquals(9.87693019124002, estimator.estimateExactK(20, 10), 0.0); + assertEquals(6.964137165389415, estimator.estimateExactK(20, 20), 0.0); + } + + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K10_Five9() { + TopKEstimator estimator = new TopKEstimator(30, 0.99999); + assertEquals(10, estimator.estimateExactK(10, 1), 0.0); + assertEquals(10, estimator.estimateK(10, 1)); + assertEquals(13.379168295125641, estimator.estimateExactK(10, 2), 0.0); + assertEquals(11.447448515386741, estimator.estimateExactK(10, 3), 0.0); + assertEquals(6.569830753158866, estimator.estimateExactK(10, 10), 0.0); + assertEquals(4.717281833573569, estimator.estimateExactK(10, 20), 0.0); + } + + @Test + public void requireHitsAreEstimatedAccordingToPartitionsAndProbabilityForVaryingN_K10_Four9() { + TopKEstimator estimator = new TopKEstimator(30, 0.9999); + assertEquals(10, estimator.estimateExactK(10, 1), 0.0); + assertEquals(10, estimator.estimateK(10, 1)); + assertEquals(12.087323848369289, estimator.estimateExactK(10, 2), 0.0); + assertEquals(10.230749855131009, estimator.estimateExactK(10, 3), 0.0); + assertEquals(5.794676146031378, estimator.estimateExactK(10, 10), 0.0); + assertEquals(4.152394782937266, estimator.estimateExactK(10, 20), 0.0); + } + + @Test + public void requireEstimatesAreRoundeUp() { + TopKEstimator estimator = new TopKEstimator(30, 0.9999); + assertEquals(5.794676146031378, estimator.estimateExactK(10, 10), 0.0); + assertEquals(6, estimator.estimateK(10, 10)); + } + + @Test + public void requireEstimatesAreCappedAtInputK() { + TopKEstimator estimator = new TopKEstimator(30, 0.9999); + assertEquals(12.087323848369289, estimator.estimateExactK(10, 2), 0.0); + assertEquals(10, estimator.estimateK(10, 2)); + } + + @Test + public void requireThatLargeKAreSane() { + System.out.println(dumpProbability(10, 0.05)); + TopKEstimator idealEstimator = new TopKEstimator(30, 0.9999); + TopKEstimator skewedEstimator = new TopKEstimator(30, 0.9999, 0.05); + int [] K = {10, 20, 40, 80, 100, 200, 400, 800, 1000, 2000, 4000, 8000, 10000, 20000, 40000, 80000, 100000}; + int [] expecedWithZeroSkew = {6, 9, 14, 22, 26, 42, 71, 123, 148, 268, 496, 936, 1152, 2215, 4304, 8429, 10480}; + int [] expecedWith5pSkew = {6, 10, 14, 23, 26, 43, 73, 128, 154, 280, 518, 979, 1205, 2319, 4509, 8837, 10989}; + for (int i = 0; i < K.length; i++) { + assertEquals(expecedWithZeroSkew[i], idealEstimator.estimateK(K[i], 10)); + assertEquals(expecedWith5pSkew[i], skewedEstimator.estimateK(K[i], 10)); + } + + String expected = + "Prob/Hits: 1.0000000000 0.9999000000 0.9999900000 0.9999990000 0.9999999000 0.9999999900 0.9999999990 0.9999999999\n" + + " 10: 10.000 6.000 7.000 8.000 9.000 10.000 10.000 10.000\n" + + " 20: 10.000 4.500 5.000 5.500 6.500 7.000 7.500 8.000\n" + + " 40: 10.000 3.500 4.000 4.250 4.750 5.250 5.500 6.000\n" + + " 80: 10.000 2.750 3.000 3.250 3.625 3.875 4.250 4.500\n" + + " 100: 10.000 2.600 2.800 3.100 3.300 3.600 3.900 4.200\n" + + " 200: 10.000 2.100 2.250 2.450 2.650 2.800 3.000 3.200\n" + + " 400: 10.000 1.775 1.900 2.025 2.150 2.275 2.425 2.575\n" + + " 800: 10.000 1.538 1.625 1.713 1.813 1.900 2.000 2.100\n" + + " 1000: 10.000 1.480 1.560 1.640 1.720 1.810 1.890 1.990\n" + + " 2000: 10.000 1.340 1.395 1.450 1.510 1.570 1.630 1.695\n" + + " 4000: 10.000 1.240 1.280 1.320 1.360 1.403 1.445 1.493\n" + + " 8000: 10.000 1.170 1.198 1.225 1.254 1.284 1.315 1.348\n" + + " 10000: 10.000 1.152 1.177 1.202 1.227 1.254 1.282 1.311\n" + + " 20000: 10.000 1.108 1.125 1.143 1.161 1.180 1.199 1.220\n" + + " 40000: 10.000 1.076 1.088 1.101 1.114 1.127 1.141 1.156\n" + + " 80000: 10.000 1.054 1.062 1.071 1.080 1.090 1.100 1.110\n" + + " 100000: 10.000 1.048 1.056 1.064 1.072 1.080 1.089 1.098\n"; + assertEquals(expected, dumpProbability(10, 0.0)); + String expectedSkew = + "Prob/Hits: 1.0000000000 0.9999000000 0.9999900000 0.9999990000 0.9999999000 0.9999999900 0.9999999990 0.9999999999\n" + + " 10: 10.000 6.000 7.000 8.000 9.000 10.000 10.000 10.000\n" + + " 20: 10.000 5.000 5.500 6.000 6.500 7.000 7.500 8.500\n" + + " 40: 10.000 3.500 4.000 4.500 4.750 5.250 5.750 6.250\n" + + " 80: 10.000 2.875 3.125 3.375 3.750 4.000 4.375 4.625\n" + + " 100: 10.000 2.600 2.900 3.100 3.400 3.700 4.000 4.300\n" + + " 200: 10.000 2.150 2.350 2.500 2.700 2.900 3.100 3.300\n" + + " 400: 10.000 1.825 1.950 2.075 2.225 2.350 2.500 2.650\n" + + " 800: 10.000 1.600 1.688 1.775 1.875 1.975 2.075 2.175\n" + + " 1000: 10.000 1.540 1.620 1.700 1.790 1.870 1.960 2.060\n" + + " 2000: 10.000 1.400 1.455 1.510 1.570 1.630 1.695 1.760\n" + + " 4000: 10.000 1.295 1.335 1.375 1.418 1.460 1.505 1.553\n" + + " 8000: 10.000 1.224 1.251 1.280 1.309 1.340 1.371 1.405\n" + + " 10000: 10.000 1.205 1.230 1.255 1.282 1.309 1.337 1.367\n" + + " 20000: 10.000 1.160 1.177 1.195 1.214 1.233 1.253 1.275\n" + + " 40000: 10.000 1.127 1.140 1.153 1.166 1.179 1.194 1.209\n" + + " 80000: 10.000 1.105 1.114 1.123 1.132 1.141 1.152 1.162\n" + + " 100000: 10.000 1.099 1.107 1.115 1.123 1.132 1.141 1.150\n"; + assertEquals(expectedSkew, dumpProbability(10, 0.05)); + } + + /** + * This make a table showing how many more hits will be fetched as a factor of hits requested. + * It shows how it varies with probability and hits requested for a given number of partitions. + */ + private String dumpProbability(int numPartitions, double skewFactor) { + TopKEstimator estimator = new TopKEstimator(30, 0.9999, skewFactor); + int [] K = {10, 20, 40, 80, 100, 200, 400, 800, 1000, 2000, 4000, 8000, 10000, 20000, 40000, 80000, 100000}; + double [] P = {1.0, 0.9999, 0.99999, 0.999999, 0.9999999, 0.99999999, 0.999999999, 0.9999999999}; + int n = numPartitions; + StringBuilder sb = new StringBuilder(); + sb.append(String.format("Prob/Hits:")); + for (double p : P) { + sb.append(String.format(Locale.ENGLISH, " %1.10f", p)); + } + sb.append("\n"); + for (int k : K) { + sb.append(String.format(Locale.ENGLISH, "%9d:", k)); + for (double p : P) { + sb.append(String.format(Locale.ENGLISH, "%13.3f", (double)(estimator.estimateK(k, n, p)*n) / k)); + } + sb.append("\n"); + } + return sb.toString(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java index c07bf119782..c421e9523ed 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/rpc/RpcSearchInvokerTest.java @@ -19,15 +19,16 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** * @author ollivir */ public class RpcSearchInvokerTest { + @Test public void testProtobufSerialization() throws IOException { var compressionTypeHolder = new AtomicReference<CompressionType>(); @@ -35,18 +36,44 @@ public class RpcSearchInvokerTest { var lengthHolder = new AtomicInteger(); var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); - @SuppressWarnings("resource") - var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool); + var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool, 1000); Query q = new Query("search/?query=test&hits=10&offset=3"); - invoker.sendSearchRequest(q); + RpcSearchInvoker.RpcContext context = (RpcSearchInvoker.RpcContext) invoker.sendSearchRequest(q, null); + assertEquals(lengthHolder.get(), context.compressedPayload.uncompressedSize()); + assertSame(context.compressedPayload.data(), payloadHolder.get()); var bytes = mockPool.compressor().decompress(payloadHolder.get(), compressionTypeHolder.get(), lengthHolder.get()); var request = SearchProtocol.SearchRequest.newBuilder().mergeFrom(bytes).build(); - assertThat(request.getHits(), equalTo(10)); - assertThat(request.getOffset(), equalTo(3)); - assertThat(request.getQueryTreeBlob().size(), greaterThan(0)); + assertEquals(10, request.getHits()); + assertEquals(3, request.getOffset()); + assertTrue(request.getQueryTreeBlob().size() > 0); + + var invoker2 = new RpcSearchInvoker(mockSearcher(), new Node(8, "eight", 1), mockPool, 1000); + RpcSearchInvoker.RpcContext context2 = (RpcSearchInvoker.RpcContext)invoker2.sendSearchRequest(q, context); + assertSame(context, context2); + assertEquals(lengthHolder.get(), context.compressedPayload.uncompressedSize()); + assertSame(context.compressedPayload.data(), payloadHolder.get()); + } + + @Test + public void testProtobufSerializationWithMaxHitsSet() throws IOException { + int maxHits = 5; + var compressionTypeHolder = new AtomicReference<CompressionType>(); + var payloadHolder = new AtomicReference<byte[]>(); + var lengthHolder = new AtomicInteger(); + var mockClient = parameterCollectorClient(compressionTypeHolder, payloadHolder, lengthHolder); + var mockPool = new RpcResourcePool(ImmutableMap.of(7, mockClient.createConnection("foo", 123))); + var invoker = new RpcSearchInvoker(mockSearcher(), new Node(7, "seven", 1), mockPool, maxHits); + + Query q = new Query("search/?query=test&hits=10&offset=3"); + invoker.sendSearchRequest(q, null); + + var bytes = mockPool.compressor().decompress(payloadHolder.get(), compressionTypeHolder.get(), lengthHolder.get()); + var request = SearchProtocol.SearchRequest.newBuilder().mergeFrom(bytes).build(); + + assertEquals(maxHits, request.getHits()); } private Client parameterCollectorClient(AtomicReference<CompressionType> compressionTypeHolder, AtomicReference<byte[]> payloadHolder, @@ -91,4 +118,5 @@ public class RpcSearchInvokerTest { } }; } + } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 315a05ce14d..09024150a9a 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -8,20 +8,19 @@ import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.MockSearchCluster; +import com.yahoo.search.dispatch.TopKEstimator; import com.yahoo.search.result.ErrorMessage; -import org.jetbrains.annotations.NotNull; import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -30,101 +29,113 @@ import static org.junit.Assert.assertTrue; */ public class SearchClusterTest { - static class State { - class MyExecutor implements Executor { - private final List<Runnable> list = new ArrayList<>(); - @Override - public void execute(@NotNull Runnable command) { - list.add(command); - } - void run() { - for (Runnable runnable : list) { - runnable.run(); - } - list.clear(); - } - } + static class State implements AutoCloseable{ + final String clusterId; final int nodesPerGroup; final VipStatus vipStatus; final SearchCluster searchCluster; + final ClusterMonitor clusterMonitor; final List<AtomicInteger> numDocsPerNode; List<AtomicInteger> pingCounts; + State(String clusterId, int nodesPergroup, String ... nodeNames) { this(clusterId, nodesPergroup, Arrays.asList(nodeNames)); } - State(String clusterId, int nodesPergroup, List<String> nodeNames) { + + State(String clusterId, int nodesPerGroup, List<String> nodeNames) { this.clusterId = clusterId; - this.nodesPerGroup = nodesPergroup; - vipStatus = new VipStatus(new QrSearchersConfig.Builder().searchcluster(new QrSearchersConfig.Searchcluster.Builder().name(clusterId)).build(), new ClustersStatus()); + this.nodesPerGroup = nodesPerGroup; + vipStatus = new VipStatus(new QrSearchersConfig.Builder().searchcluster(new QrSearchersConfig.Searchcluster.Builder().name(clusterId)).build(), + new ClustersStatus()); numDocsPerNode = new ArrayList<>(nodeNames.size()); pingCounts = new ArrayList<>(nodeNames.size()); List<Node> nodes = new ArrayList<>(nodeNames.size()); for (String name : nodeNames) { - int key = nodes.size() % nodesPergroup; - int group = nodes.size() / nodesPergroup; + int key = nodes.size() % nodesPerGroup; + int group = nodes.size() / nodesPerGroup; nodes.add(new Node(key, name, group)); numDocsPerNode.add(new AtomicInteger(1)); pingCounts.add(new AtomicInteger(0)); } - searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPergroup, vipStatus); - } - void startMonitoring() { - searchCluster.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); + searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup, + vipStatus, new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); + clusterMonitor = new ClusterMonitor(searchCluster, false); + searchCluster.addMonitoring(clusterMonitor); } - static private int getMaxValue(List<AtomicInteger> list) { - int max = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() > max) { - max = v.get(); + + private int maxPingCount() { + int max = pingCounts.get(0).get(); + for (AtomicInteger count : pingCounts) { + if (count.get() > max) { + max = count.get(); } } return max; } - private static int getMinValue(List<AtomicInteger> list) { - int min = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() < min) { - min = v.get(); + + private int minPingCount() { + int min = pingCounts.get(0).get(); + for (AtomicInteger count : pingCounts) { + if (count.get() < min) { + min = count.get(); } } return min; } - private void waitAtLeast(int atLeast, List<AtomicInteger> list) { - while (getMinValue(list) < atLeast) { + + void waitOneFullPingRound() { + int minPingCount = minPingCount(); + int atLeast = maxPingCount() + 1; + while (minPingCount < atLeast) { ExecutorService executor = Executors.newCachedThreadPool(); - searchCluster.clusterMonitor().ping(executor); + clusterMonitor.ping(executor); executor.shutdown(); try { - executor.awaitTermination(60, TimeUnit.SECONDS); - } catch (InterruptedException e) {} + boolean completed = executor.awaitTermination(120, TimeUnit.SECONDS); + if ( ! completed ) + throw new IllegalStateException("Ping thread timed out"); + // Since a separate thread will be modifying values in pingCounts, we need to wait for the thread to + // finish before re-reading the minimum value + minPingCount = minPingCount(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } - void waitOneFullPingRound() { - waitAtLeast(getMaxValue(pingCounts) + 1, pingCounts); + + @Override + public void close() { + clusterMonitor.shutdown(); } + static class Factory implements PingFactory { - static class Pinger implements Callable<Pong> { + + static class PingJob implements Pinger { + private final AtomicInteger numDocs; private final AtomicInteger pingCount; - Pinger(AtomicInteger numDocs, AtomicInteger pingCount) { + private final PongHandler pongHandler; + PingJob(AtomicInteger numDocs, AtomicInteger pingCount, PongHandler pongHandler) { this.numDocs = numDocs; this.pingCount = pingCount; + this.pongHandler = pongHandler; } @Override - public Pong call() throws Exception { + public void ping() { int docs = numDocs.get(); - pingCount.incrementAndGet(); - return (docs < 0) + pongHandler.handle ((docs < 0) ? new Pong(ErrorMessage.createBackendCommunicationError("Negative numDocs = " + docs)) - : new Pong(docs); + : new Pong(docs)); + pingCount.incrementAndGet(); } } private final List<AtomicInteger> activeDocs; private final List<AtomicInteger> pingCounts; private final int numPerGroup; + Factory(int numPerGroup, List<AtomicInteger> activeDocs, List<AtomicInteger> pingCounts) { this.numPerGroup = numPerGroup; this.activeDocs = activeDocs; @@ -132,122 +143,139 @@ public class SearchClusterTest { } @Override - public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) { int index = node.group() * numPerGroup + node.key(); - return new Pinger(activeDocs.get(index), pingCounts.get(index)); + return new PingJob(activeDocs.get(index), pingCounts.get(index), pongHandler); } } + } @Test public void requireThatVipStatusIsDefaultDownButComesUpAfterPinging() { - State test = new State("cluster.1", 2, "a", "b"); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); + try (State test = new State("cluster.1", 2, "a", "b")) { + assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); + assertFalse(test.vipStatus.isInRotation()); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } } @Test public void requireThatZeroDocsAreFine() { - State test = new State("cluster.1", 2,"a", "b"); - test.startMonitoring(); - test.waitOneFullPingRound(); - - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - - test.numDocsPerNode.get(0).set(-1); - test.numDocsPerNode.get(1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(0).set(0); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); + try (State test = new State("cluster.1", 2, "a", "b")) { + test.waitOneFullPingRound(); + + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); + + test.numDocsPerNode.get(0).set(-1); + test.numDocsPerNode.get(1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(0); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } } @Test public void requireThatVipStatusIsDefaultDownWithLocalDispatch() { - State test = new State("cluster.1", 1, HostName.getLocalhost(), "b"); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); + try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "b")) { + assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); + assertFalse(test.vipStatus.isInRotation()); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } } @Test - public void requireThatVipStatusIsDefaultDownWithOnlySingleLocalDispatch() { - State test = new State("cluster.1", 1, HostName.getLocalhost()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - - assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(0).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); + public void requireThatVipStatusStaysUpWithLocalDispatchAndClusterSize1() { + try (State test = new State("cluster.1", 1, HostName.getLocalhost())) { + assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); + + assertFalse(test.vipStatus.isInRotation()); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(-1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } + } + + @Test + public void requireThatVipStatusIsDefaultDownWithLocalDispatchAndClusterSize2() { + try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "otherhost")) { + assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); + + assertFalse(test.vipStatus.isInRotation()); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + } } @Test public void requireThatVipStatusDownWhenLocalIsDown() { - State test = new State("cluster.1",1,HostName.getLocalhost(), "b"); - - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(0).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(0).set(1); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(1).set(-1); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(0).set(-1); - test.numDocsPerNode.get(1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(1).set(1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(0).set(1); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); + try (State test = new State("cluster.1",1,HostName.getLocalhost(), "b")) { + + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); + + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(0).set(1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(1).set(-1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(0).set(-1); + test.numDocsPerNode.get(1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(1).set(1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } } private void verifyThatVipStatusDownRequireAllNodesDown(int numGroups, int nodesPerGroup) { List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); - State test = new State("cluster.1", nodesPerGroup, nodeNames); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); + try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) { + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - for (int i=0; i < test.numDocsPerNode.size()-1; i++) { - test.numDocsPerNode.get(i).set(-1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + + for (int i = 0; i < test.numDocsPerNode.size() - 1; i++) { + test.numDocsPerNode.get(i).set(-1); + } + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(test.numDocsPerNode.size() - 1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); } - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(test.numDocsPerNode.size()-1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); } + @Test public void requireThatVipStatusDownRequireAllNodesDown() { verifyThatVipStatusDownRequireAllNodesDown(1,2); @@ -257,8 +285,8 @@ public class SearchClusterTest { static private List<String> generateNodeNames(int numGroups, int nodesPerGroup) { List<String> nodeNames = new ArrayList<>(numGroups*nodesPerGroup); for (int g = 0; g < numGroups; g++) { - for (int n=0; n < nodesPerGroup; n++) { - nodeNames.add(new StringBuilder("node.").append(g).append('.').append(n).toString()); + for (int n = 0; n < nodesPerGroup; n++) { + nodeNames.add("node." + g + '.' + n); } } return nodeNames; @@ -266,29 +294,45 @@ public class SearchClusterTest { private void verifyThatVipStatusUpRequireOnlyOneOnlineNode(int numGroups, int nodesPerGroup) { List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); - State test = new State("cluster.1", nodesPerGroup, nodeNames); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - - for (int i=0; i < test.numDocsPerNode.size()-1; i++) { - test.numDocsPerNode.get(i).set(-1); + + try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) { + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); + + for (int i = 0; i < test.numDocsPerNode.size() - 1; i++) { + test.numDocsPerNode.get(i).set(-1); + } + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(test.numDocsPerNode.size() - 1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(0).set(0); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); } - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(test.numDocsPerNode.size()-1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(0).set(0); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); } + @Test public void requireThatVipStatusUpRequireOnlyOneOnlineNode() { verifyThatVipStatusUpRequireOnlyOneOnlineNode(1, 2); verifyThatVipStatusUpRequireOnlyOneOnlineNode(3, 3); } + @Test + public void requireThatPingSequenceIsUpHeld() { + Node node = new Node(1, "n", 1); + assertEquals(1, node.createPingSequenceId()); + assertEquals(2, node.createPingSequenceId()); + assertEquals(0, node.getLastReceivedPongId()); + assertTrue(node.isLastReceivedPong(2)); + assertEquals(2, node.getLastReceivedPongId()); + assertFalse(node.isLastReceivedPong(1)); + assertFalse(node.isLastReceivedPong(2)); + assertTrue(node.isLastReceivedPong(3)); + assertEquals(3, node.getLastReceivedPongId()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java index 111b7a8eb69..65cb4dff1f8 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java @@ -201,13 +201,33 @@ public class FederationSearcherTestCase { } @Test - public void testPropertyPropagation() { - Result result = searchWithPropertyPropagation(PropagateSourceProperties.ALL); + public void testPropertyPropagation_native() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.NATIVE); assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); assertEquals("nalle", result.hits().get(0).getQuery().getPresentation().getSummary()); assertNull(result.hits().get(1).getQuery().getPresentation().getSummary()); + assertEquals(null, result.hits().get(0).getQuery().properties().get("custom")); + } + + @Test + public void testPropertyPropagation_every() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.EVERY); + + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + assertEquals("nalle", result.hits().get(0).getQuery().getPresentation().getSummary()); + assertEquals("foo", result.hits().get(0).getQuery().properties().get("customSourceProperty")); + assertEquals(null, result.hits().get(1).getQuery().properties().get("customSourceProperty")); + assertEquals(null, result.hits().get(0).getQuery().properties().get("custom.source.property")); + assertEquals("bar", result.hits().get(1).getQuery().properties().get("custom.source.property")); + assertEquals(13, result.hits().get(0).getQuery().properties().get("hits")); + assertEquals(1, result.hits().get(0).getQuery().properties().get("offset")); + assertEquals(10, result.hits().get(1).getQuery().properties().get("hits")); + assertEquals(0, result.hits().get(1).getQuery().properties().get("offset")); + + assertNull(result.hits().get(1).getQuery().getPresentation().getSummary()); } private Result searchWithPropertyPropagation(PropagateSourceProperties.Enum propagateSourceProperties) { @@ -215,7 +235,7 @@ public class FederationSearcherTestCase { addChained(new MockSearcher(), "mySource2"); Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher(propagateSourceProperties)); - Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle")); + Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle&source.mySource1.customSourceProperty=foo&source.mySource2.custom.source.property=bar&source.mySource1.hits=13&source.mySource1.offset=1")); Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); assertNull(result.hits().getError()); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java index 326e37ede38..6d9c2218022 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java @@ -15,7 +15,7 @@ import java.util.concurrent.TimeUnit; */ public class GroupingParserBenchmarkTest { - private static final int NUM_RUNS = 10;//000; + private static final int NUM_RUNS = 10; private static final Map<String, Long> PREV_RESULTS = new LinkedHashMap<>(); static { diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java index cd080405a7d..c6686471dc8 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -247,6 +247,7 @@ public class GroupingParserTestCase { assertParse("all(group(predefined(foo, bucket(1, 2), bucket(3, 4), bucket(5, 6))))"); assertParse("all(group(predefined(foo, bucket(1, 2), bucket(2, 3), bucket(3, 4))))"); assertParse("all(group(predefined(foo, bucket(-100, 0), bucket(0), bucket<0, 100))))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>)))"); assertParse("all(group(predefined(foo, bucket[1, 2>)))"); assertParse("all(group(predefined(foo, bucket[-1, 2>)))"); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java index e2f629604d7..84a6fc418a1 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -489,6 +489,7 @@ public class GroupingExecutorTestCase { */ @Test public void testRankProperties() { + final double delta = 0.000000001; Execution exc = newExecution(new GroupingExecutor()); { Query query = new Query("?query=foo"); @@ -496,21 +497,21 @@ public class GroupingExecutorTestCase { } { Query query = new Query("?query=foo&rankfeature.fieldMatch(foo)=2"); - assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals(2, query.getRanking().getFeatures().getDouble("fieldMatch(foo)").getAsDouble(), delta); exc.search(query); - assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals(2.0, query.getRanking().getFeatures().getDouble("fieldMatch(foo)").getAsDouble(), delta); } { Query query = new Query("?query=foo&rankfeature.query(now)=4"); - assertEquals("4", query.getRanking().getFeatures().get("query(now)")); + assertEquals(4, query.getRanking().getFeatures().getDouble("query(now)").getAsDouble(), delta); exc.search(query); - assertEquals("4", query.getRanking().getProperties().get("now").get(0)); + assertEquals("4.0", query.getRanking().getProperties().get("now").get(0)); } { Query query = new Query("?query=foo&rankfeature.$bar=8"); - assertEquals("8", query.getRanking().getFeatures().get("$bar")); + assertEquals(8, query.getRanking().getFeatures().getDouble("$bar").getAsDouble(), delta); exc.search(query); - assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + assertEquals("8.0", query.getRanking().getProperties().get("bar").get(0)); } { Query query = new Query("?query=foo&rankproperty.bar=8"); @@ -520,12 +521,12 @@ public class GroupingExecutorTestCase { } { Query query = new Query("?query=foo&rankfeature.fieldMatch(foo)=2&rankfeature.query(now)=4&rankproperty.bar=8"); - assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); - assertEquals("4", query.getRanking().getFeatures().get("query(now)")); + assertEquals(2, query.getRanking().getFeatures().getDouble("fieldMatch(foo)").getAsDouble(), delta); + assertEquals(4, query.getRanking().getFeatures().getDouble("query(now)").getAsDouble(), delta); assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); exc.search(query); - assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); - assertEquals("4", query.getRanking().getProperties().get("now").get(0)); + assertEquals(2, query.getRanking().getFeatures().getDouble("fieldMatch(foo)").getAsDouble(), delta); + assertEquals("4.0", query.getRanking().getProperties().get("now").get(0)); assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); } } diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 02e2152d7c9..272092b6fc0 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -12,7 +12,7 @@ import com.yahoo.net.HostName; import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; import com.yahoo.slime.Inspector; -import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.slime.SlimeUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.After; diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg index 96843d78aae..915da8dc037 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg @@ -1,4 +1,4 @@ -handler[7] +handler[8] handler[0].id com.yahoo.search.handler.SearchHandler handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningHandler handler[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningAsyncHandler @@ -6,3 +6,4 @@ handler[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingHandle handler[4].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingAsyncHandler handler[5].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingHandler handler[6].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingAsyncHandler +handler[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java new file mode 100644 index 00000000000..39d4fec2716 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java @@ -0,0 +1,28 @@ +package com.yahoo.search.query.profile.compiled; + +import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class CompiledQueryProfileRegistryTest { + + @Test + public void registry_can_be_created_from_config() { + var config = new QueryProfilesConfig.Builder() + .queryprofile(new QueryProfilesConfig.Queryprofile.Builder() + .id("profile1") + .property(new QueryProfilesConfig.Queryprofile.Property.Builder() + .name("hits") + .value("5"))) + .build(); + + var registry = new CompiledQueryProfileRegistry(config); + var profile1 = registry.findQueryProfile("profile1"); + assertEquals("5", profile1.get("hits")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java index d77dc3e9939..a1fba8e07f1 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java @@ -1,27 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.config.test; -import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.io.IOUtils; import com.yahoo.search.Query; import com.yahoo.search.query.profile.QueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.query.profile.config.QueryProfilesConfig.Queryprofile; import com.yahoo.search.test.QueryTestCase; -import com.yahoo.vespa.config.ConfigPayload; -import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; -import java.io.File; -import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; + /** * @author bratseth */ @@ -138,20 +136,22 @@ public class QueryProfileConfigurationTestCase { @Test public void testVariant2ConfigurationThroughQueryLookup() { + final double delta = 0.0000001; + QueryProfileConfigurer configurer= new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants2.cfg"); CompiledQueryProfileRegistry registry = configurer.getCurrentRegistry().compile(); Query query = new Query(QueryTestCase.httpEncode("?query=heh&queryProfile=multi&myindex=default&myquery=lo ve&tracelevel=5"), registry.findQueryProfile("multi")); - assertEquals("love",query.properties().get("model.queryString")); - assertEquals("default",query.properties().get("model.defaultIndex")); + assertEquals("love", query.properties().get("model.queryString")); + assertEquals("default", query.properties().get("model.defaultIndex")); - assertEquals("-20",query.properties().get("ranking.features.query(scorelimit)")); - assertEquals("-20",query.getRanking().getFeatures().get("query(scorelimit)")); + assertEquals(-20.0, query.properties().get("ranking.features.query(scorelimit)")); + assertEquals(-20.0, query.getRanking().getFeatures().getDouble("query(scorelimit)").getAsDouble(), delta); query.properties().set("rankfeature.query(scorelimit)", -30); - assertEquals("-30",query.properties().get("ranking.features.query(scorelimit)")); - assertEquals("-30",query.getRanking().getFeatures().get("query(scorelimit)")); + assertEquals(-30.0, query.properties().get("ranking.features.query(scorelimit)")); + assertEquals(-30, query.getRanking().getFeatures().getDouble("query(scorelimit)").getAsDouble(), delta); } private void assertGet(String expectedValue,String parameter,String[] dimensionValues,QueryProfile profile) { diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java index 02aa95be510..445073ced3a 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.search.query.profile.config.test; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.processing.execution.Execution; import com.yahoo.processing.request.CompoundName; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; @@ -32,6 +31,17 @@ import static org.junit.Assert.fail; public class XmlReadingTestCase { @Test + public void testInheritance() { + QueryProfileRegistry registry = + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/inheritance"); + + CompiledQueryProfile cProfile = registry.getComponent("child").compile(null); + Query q = new Query("?query=foo", cProfile); + assertEquals("a.b-parent", q.properties().getString("a.b")); + assertEquals("d-parent", q.properties().getString("d")); + } + + @Test public void testValid() { QueryProfileRegistry registry= new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/validxml"); @@ -299,67 +309,70 @@ public class XmlReadingTestCase { String queryString="tiled?query=india&queryProfile=myprofile&source.common.intl=tw&source.common.mode=adv"; Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET), registry.getComponent("myprofile")); - for (Map.Entry e : query.properties().listProperties().entrySet()) - System.out.println(e); assertEquals("news",query.properties().listProperties().get("source.common.provider")); assertEquals("news",query.properties().get("source.common.provider")); } @Test public void testNewsCase1() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase1").compile(); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase1").compile(); Query query; - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); - assertEquals("0.0",query.properties().get("ranking.features.b")); - assertEquals("0.0",query.properties().listProperties().get("ranking.features.b")); - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); - assertEquals("0.1",query.properties().get("ranking.features.b")); - assertEquals("0.1",query.properties().listProperties().get("ranking.features.b")); + query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET), + registry.getComponent("default")); + assertEquals(0.0, query.properties().get("ranking.features.b")); + assertEquals("0.0", query.properties().listProperties().get("ranking.features.b")); + query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET), + registry.getComponent("default")); + assertEquals(0.1, query.properties().get("ranking.features.b")); + assertEquals("0.1", query.properties().listProperties().get("ranking.features.b")); } @Test public void testNewsCase2() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase2").compile(); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase2").compile(); Query query; - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); - assertEquals("0.0",query.properties().get("a.features.b")); - assertEquals("0.0",query.properties().listProperties().get("a.features.b")); - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); - assertEquals("0.1",query.properties().get("a.features.b")); - assertEquals("0.1",query.properties().listProperties().get("a.features.b")); + query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET), + registry.getComponent("default")); + assertEquals("0.0", query.properties().get("a.features.b")); + assertEquals("0.0", query.properties().listProperties().get("a.features.b")); + query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET), + registry.getComponent("default")); + assertEquals("0.1", query.properties().get("a.features.b")); + assertEquals("0.1", query.properties().listProperties().get("a.features.b")); } @Test public void testNewsCase3() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase3").compile(); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase3").compile(); - Query query; - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + Query query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET), + registry.getComponent("default")); assertEquals("0.0",query.properties().get("a.features")); - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET), + registry.getComponent("default")); assertEquals("0.1",query.properties().get("a.features")); } - // Should cause an exception on the first line as we are trying to create a profile setting an illegal value in "ranking" @Test public void testNewsCase4() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase4").compile(); - - Query query; - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); - assertEquals("0.0",query.properties().get("ranking.features")); - query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); - assertEquals("0.1",query.properties().get("ranking.features")); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase4").compile(); + + Query query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET), + registry.getComponent("default")); + assertEquals(0.0, query.properties().get("ranking.features.foo")); + query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET), + registry.getComponent("default")); + assertEquals(0.1, query.properties().get("ranking.features.foo")); } @Test public void testVersionRefs() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/versionrefs").compile(); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/versionrefs").compile(); - Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); - assertEquals("MyProfile:1.0.2",query.properties().get("profile1.name")); + Query query = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), registry.getComponent("default")); + assertEquals("MyProfile:1.0.2", query.properties().get("profile1.name")); } @Test @@ -368,59 +381,82 @@ public class XmlReadingTestCase { { // Original reference - Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); - assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile1",query.properties().get("profileRef.name")); - assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + Query query = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), + registry.getComponent("default")); + assertEquals(null, query.properties().get("profileRef")); + assertEquals("MyProfile1", query.properties().get("profileRef.name")); + assertEquals("myProfile1Only", query.properties().get("profileRef.myProfile1Only")); assertNull(query.properties().get("profileRef.myProfile2Only")); } { // Overridden reference - Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); + Query query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); - assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); + assertEquals("myProfile2Only", query.properties().get("profileRef.myProfile2Only")); assertNull(query.properties().get("profileRef.myProfile1Only")); // later assignment - query.properties().set("profileRef.name","newName"); - assertEquals("newName",query.properties().get("profileRef.name")); + query.properties().set("profileRef.name", "newName"); + assertEquals("newName", query.properties().get("profileRef.name")); // ...will not impact others - query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); + query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), + registry.getComponent("default")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); } } @Test public void testRefOverrideTyped() { - CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); { // Original reference - Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); - assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile1",query.properties().get("profileRef.name")); - assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + Query query = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), registry.getComponent("default")); + assertEquals(null, query.properties().get("profileRef")); + assertEquals("MyProfile1", query.properties().get("profileRef.name")); + assertEquals("myProfile1Only", query.properties().get("profileRef.myProfile1Only")); assertNull(query.properties().get("profileRef.myProfile2Only")); } { // Overridden reference - Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals(null,query.properties().get("profileRef")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); - assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + Query query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals(null, query.properties().get("profileRef")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); + assertEquals("myProfile2Only", query.properties().get("profileRef.myProfile2Only")); assertNull(query.properties().get("profileRef.myProfile1Only")); // later assignment - query.properties().set("profileRef.name","newName"); - assertEquals("newName",query.properties().get("profileRef.name")); + query.properties().set("profileRef.name", "newName"); + assertEquals("newName", query.properties().get("profileRef.name")); // ...will not impact others - query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); - assertEquals("MyProfile2",query.properties().get("profileRef.name")); + query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default")); + assertEquals("MyProfile2", query.properties().get("profileRef.name")); } } + @Test + public void testTensorTypes() { + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/tensortypes").compile(); + + QueryProfileType type1 = registry.getTypeRegistry().getComponent("type1"); + assertEquals("tensor<float>(x[1])", type1.getFieldType(new CompoundName("ranking.features.query(tensor_1)")).stringValue()); + assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_2)"))); + assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_3)"))); + + QueryProfileType type2 = registry.getTypeRegistry().getComponent("type2"); + assertNull(type2.getFieldType(new CompoundName("ranking.features.query(tensor_1)"))); + assertEquals("tensor<float>(x[2])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_2)")).stringValue()); + assertEquals("tensor<float>(x[3])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_3)")).stringValue()); + + Query queryProfile1 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile1")); + assertEquals("Is received as a tensor tensor", "tensor<float>(x[1]):[1.2]", queryProfile1.properties().get("ranking.features.query(tensor_1)").toString()); + + Query queryProfile2 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile2")); + assertEquals("Is received as a string", "[1.200]", queryProfile2.properties().get("ranking.features.query(tensor_1)").toString()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/child.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/child.xml new file mode 100644 index 00000000000..64dd3b787ac --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/child.xml @@ -0,0 +1,6 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="child" inherits="parent"> + <field name="a.b.c">a.b.c-child</field> + <field name="d.e.f">d.e.f-child</field> +</query-profile> + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/parent.xml new file mode 100644 index 00000000000..b3443fab646 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/inheritance/parent.xml @@ -0,0 +1,7 @@ +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent"> + <field name="a.b">a.b-parent</field> + <field name="a.b.c">a.b.c-parent</field> + <field name="d">d-parent</field> + <field name="d.e.f">d.e.f-parent</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml index 07e0f9dda31..1366de00813 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml @@ -9,7 +9,7 @@ <query-profile for="parent" inherits="parent" /> <query-profile for="parent,child" > - <field name="ranking.features">0.1</field> + <field name="ranking.features.foo">0.1</field> </query-profile> </query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml index d35642b9ddd..9fe0ee85dae 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml @@ -1,5 +1,5 @@ <?xml version="1.0"?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <query-profile id="parent"> - <field name="ranking.features">0.0</field> + <field name="ranking.features.foo">0.0</field> </query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml new file mode 100644 index 00000000000..000fd3e1c5b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml @@ -0,0 +1,2 @@ +<query-profile id="profile1" type="type1"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml new file mode 100644 index 00000000000..f6539da23e8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml @@ -0,0 +1,2 @@ +<query-profile id="profile2" type="type2"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml new file mode 100644 index 00000000000..3dfaab9c5f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml @@ -0,0 +1,3 @@ +<query-profile-type id="type1"> + <field name="ranking.features.query(tensor_1)" type="tensor<float>(x[1])" /> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml new file mode 100644 index 00000000000..ed7cf23e464 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml @@ -0,0 +1,4 @@ +<query-profile-type id="type2"> + <field name="ranking.features.query(tensor_2)" type="tensor<float>(x[2])" /> + <field name="ranking.features.query(tensor_3)" type="tensor<float>(x[3])" /> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg index a047ae1cb73..04dcbb22d5d 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg @@ -10,3 +10,4 @@ components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileInt components[4].id com.yahoo.search.handler.SearchHandler components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack components[6].id com.yahoo.search.searchchain.ExecutionFactory +components[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg index ef9d4490a77..b5bc450ec17 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg @@ -10,3 +10,4 @@ components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileInt components[4].id com.yahoo.search.handler.SearchHandler components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack components[6].id com.yahoo.search.searchchain.ExecutionFactory +components[7].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index 46efb736918..eb1584efe84 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -326,14 +326,14 @@ public class QueryProfileTestCase { assertEquals("mormor-model.b", annetBarnMap.get("venn.model.b")); } - /** Tests that dots are followed when setting overridability */ + /** Dots are followed when setting overridability */ @Test public void testInstanceOverridable() { QueryProfile profile = new QueryProfile("root/unoverridableIndex"); profile.set("model.defaultIndex","default", null); profile.setOverridable("model.defaultIndex", false,null); - assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue()); + assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null)); // Parameters should be ignored Query query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title", Method.GET), profile.compile(null)); @@ -345,7 +345,7 @@ public class QueryProfileTestCase { assertEquals("de", query.getModel().getLanguage().languageCode()); } - /** Tests that dots are followed when setting overridability...also with variants */ + /** Dots are followed when setting overridability, also with variants */ @Test public void testInstanceOverridableWithVariants() { QueryProfile profile = new QueryProfile("root/unoverridableIndex"); @@ -504,7 +504,8 @@ public class QueryProfileTestCase { p.set("a","a-value", null); p.set("a.b","a.b-value", null); Map<String, Object> values = p.compile(null).listValues("a"); - assertEquals(1, values.size()); + assertEquals(2, values.size()); + p.set("a","a-value", null); assertEquals("a.b-value", values.get("b")); } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java index 7dc6eb3d8aa..b875c66735b 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java @@ -230,7 +230,6 @@ public class MandatoryTestCase { defaultProfile.setType(fixture.rootType); QueryProfile mandatoryProfile = new QueryProfile("mandatory"); - mandatoryProfile.setType(fixture.rootType); mandatoryProfile.setType(fixture.mandatoryType); fixture.registry.register(defaultProfile); @@ -249,7 +248,6 @@ public class MandatoryTestCase { defaultProfile.setType(fixture.rootType); QueryProfile mandatoryProfile = new QueryProfile("mandatory"); - mandatoryProfile.setType(fixture.rootType); mandatoryProfile.addInherited(defaultProfile); // The single difference from the test above mandatoryProfile.setType(fixture.mandatoryType); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index c05c3589a30..ecebbead866 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.search.query.profile.types.test; import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.prelude.query.QueryException; import com.yahoo.tensor.Tensor; @@ -338,20 +339,22 @@ public class QueryProfileTypeTestCase { */ @Test public void testTypedAssignmentOfQueryProfileReferencesNonStrictThroughQuery() { - QueryProfile profile=new QueryProfile("test"); + QueryProfile profile = new QueryProfile("test"); profile.setType(type); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(user); - newUser.set("myUserString","newUserValue1", registry); - newUser.set("myUserInteger",845, registry); + newUser.set("myUserString", "newUserValue1", registry); + newUser.set("myUserInteger", 845, registry); registry.register(profile); registry.register(newUser); CompiledQueryProfileRegistry cRegistry = registry.compile(); CompiledQueryProfile cprofile = cRegistry.getComponent("test"); - Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cprofile); + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + cprofile); assertEquals(0, query.errors().size()); @@ -365,13 +368,13 @@ public class QueryProfileTypeTestCase { */ @Test public void testTypedAssignmentOfQueryProfileReferencesStrictThroughQuery() { - QueryProfile profile=new QueryProfile("test"); + QueryProfile profile = new QueryProfile("test"); profile.setType(typeStrict); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(userStrict); - newUser.set("myUserString","newUserValue1", registry); - newUser.set("myUserInteger",845, registry); + newUser.set("myUserString", "newUserValue1", registry); + newUser.set("myUserInteger", 845, registry); registry.register(profile); registry.register(newUser); @@ -381,11 +384,11 @@ public class QueryProfileTypeTestCase { Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); assertEquals(0, query.errors().size()); - assertEquals("newUserValue1",query.properties().get("myUserQueryProfile.myUserString")); - assertEquals(845,query.properties().get("myUserQueryProfile.myUserInteger")); + assertEquals("newUserValue1", query.properties().get("myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("myUserQueryProfile.myUserInteger")); try { - query.properties().set("myUserQueryProfile.someKey","value"); + query.properties().set("myUserQueryProfile.someKey", "value"); fail("Should not be allowed to set this"); } catch (IllegalArgumentException e) { @@ -397,7 +400,7 @@ public class QueryProfileTypeTestCase { @Test public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException { - QueryProfile profile=new QueryProfile("test"); + QueryProfile profile = new QueryProfile("test"); profile.setType(type); registry.register(profile); @@ -405,7 +408,8 @@ public class QueryProfileTypeTestCase { String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; Query query = new Query(HttpRequest.createTestRequest("?" + encode("ranking.features.query(myTensor1)") + "=" + encode(tensorString), - com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); + com.yahoo.jdisc.http.HttpRequest.Method.GET), + cRegistry.getComponent("test")); assertEquals(0, query.errors().size()); assertEquals(Tensor.from(tensorString), query.properties().get("ranking.features.query(myTensor1)")); assertEquals(Tensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); @@ -417,26 +421,23 @@ public class QueryProfileTypeTestCase { @Test public void testIllegalStrictAssignmentFromRequest() { - QueryProfile profile=new QueryProfile("test"); + QueryProfile profile = new QueryProfile("test"); profile.setType(typeStrict); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(userStrict); profile.set("myUserQueryProfile", newUser, registry); try { - new Query( - HttpRequest.createTestRequest( - "?myUserQueryProfile.nondeclared=someValue", - com.yahoo.jdisc.http.HttpRequest.Method.GET), - profile.compile(null)); + new Query(HttpRequest.createTestRequest("?myUserQueryProfile.nondeclared=someValue", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + profile.compile(null)); fail("Above statement should throw"); } catch (QueryException e) { // As expected. - assertThat( - Exceptions.toMessageString(e), - containsString("Could not set 'myUserQueryProfile.nondeclared' to 'someValue': 'nondeclared' is not declared in query profile type 'userStrict', and the type is strict")); + assertThat(Exceptions.toMessageString(e), + containsString("Could not set 'myUserQueryProfile.nondeclared' to 'someValue': 'nondeclared' is not declared in query profile type 'userStrict', and the type is strict")); } } @@ -447,25 +448,25 @@ public class QueryProfileTypeTestCase { */ @Test public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { - QueryProfile topMap=new QueryProfile("topMap"); + QueryProfile topMap = new QueryProfile("topMap"); - QueryProfile subMap=new QueryProfile("topSubMap"); - topMap.set("subMap",subMap, registry); + QueryProfile subMap = new QueryProfile("topSubMap"); + topMap.set("subMap", subMap, registry); - QueryProfile test=new QueryProfile("test"); + QueryProfile test = new QueryProfile("test"); test.setType(type); - subMap.set("typeProfile",test, registry); + subMap.set("typeProfile", test, registry); - QueryProfile myUser=new QueryProfile("myUser"); + QueryProfile myUser = new QueryProfile("myUser"); myUser.setType(user); - myUser.set("myUserString","userValue1", registry); - myUser.set("myUserInteger",442, registry); - test.set("myUserQueryProfile",myUser, registry); + myUser.set("myUserString", "userValue1", registry); + myUser.set("myUserInteger", 442, registry); + test.set("myUserQueryProfile", myUser, registry); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(user); - newUser.set("myUserString","newUserValue1", registry); - newUser.set("myUserInteger",845, registry); + newUser.set("myUserString", "newUserValue1", registry); + newUser.set("myUserInteger", 845, registry); registry.register(topMap); registry.register(subMap); @@ -474,7 +475,9 @@ public class QueryProfileTypeTestCase { registry.register(newUser); CompiledQueryProfileRegistry cRegistry = registry.compile(); - Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + cRegistry.getComponent("topMap")); assertEquals(0, query.errors().size()); @@ -487,25 +490,25 @@ public class QueryProfileTypeTestCase { */ @Test public void testAnonTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { - QueryProfile topMap=new QueryProfile("topMap"); + QueryProfile topMap = new QueryProfile("topMap"); - QueryProfile subMap=new QueryProfile("topSubMap"); - topMap.set("subMap",subMap, registry); + QueryProfile subMap = new QueryProfile("topSubMap"); + topMap.set("subMap", subMap, registry); - QueryProfile test=new QueryProfile("test"); + QueryProfile test = new QueryProfile("test"); test.setType(type); - subMap.set("typeProfile",test, registry); + subMap.set("typeProfile", test, registry); - QueryProfile myUser=new QueryProfile("myUser"); + QueryProfile myUser = new QueryProfile("myUser"); myUser.setType(user); - myUser.set("myUserString","userValue1", registry); - myUser.set("myUserInteger",442, registry); - test.set("myQueryProfile",myUser, registry); + myUser.set("myUserString", "userValue1", registry); + myUser.set("myUserInteger", 442, registry); + test.set("myQueryProfile", myUser, registry); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(user); - newUser.set("myUserString","newUserValue1", registry); - newUser.set("myUserInteger",845, registry); + newUser.set("myUserString", "newUserValue1", registry); + newUser.set("myUserInteger", 845, registry); registry.register(topMap); registry.register(subMap); @@ -526,14 +529,14 @@ public class QueryProfileTypeTestCase { */ @Test public void testSettingValueInStrictTypeNestedUnderUntypedMaps() { - QueryProfile topMap=new QueryProfile("topMap"); + QueryProfile topMap = new QueryProfile("topMap"); - QueryProfile subMap=new QueryProfile("topSubMap"); - topMap.set("subMap",subMap, registry); + QueryProfile subMap = new QueryProfile("topSubMap"); + topMap.set("subMap", subMap, registry); - QueryProfile test=new QueryProfile("test"); + QueryProfile test = new QueryProfile("test"); test.setType(typeStrict); - subMap.set("typeProfile",test, registry); + subMap.set("typeProfile", test, registry); registry.register(topMap); registry.register(subMap); @@ -542,16 +545,14 @@ public class QueryProfileTypeTestCase { try { new Query( - HttpRequest.createTestRequest( - "?subMap.typeProfile.someValue=value", - com.yahoo.jdisc.http.HttpRequest.Method.GET), + HttpRequest.createTestRequest("?subMap.typeProfile.someValue=value", + com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); fail("Above statement should throw"); } catch (QueryException e) { // As expected. - assertThat( - Exceptions.toMessageString(e), - containsString("Could not set 'subMap.typeProfile.someValue' to 'value': 'someValue' is not declared in query profile type 'testtypeStrict', and the type is strict")); + assertThat(Exceptions.toMessageString(e), + containsString("Could not set 'subMap.typeProfile.someValue' to 'value': 'someValue' is not declared in query profile type 'testtypeStrict', and the type is strict")); } } @@ -562,19 +563,19 @@ public class QueryProfileTypeTestCase { */ @Test public void testTypedSettingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { - QueryProfile topMap=new QueryProfile("topMap"); + QueryProfile topMap = new QueryProfile("topMap"); - QueryProfile subMap=new QueryProfile("topSubMap"); + QueryProfile subMap = new QueryProfile("topSubMap"); topMap.set("subMap",subMap, registry); - QueryProfile test=new QueryProfile("test"); + QueryProfile test = new QueryProfile("test"); test.setType(type); subMap.set("typeProfile",test, registry); - QueryProfile newUser=new QueryProfile("newUser"); + QueryProfile newUser = new QueryProfile("newUser"); newUser.setType(user); - newUser.set("myUserString","newUserValue1", registry); - newUser.set("myUserInteger",845, registry); + newUser.set("myUserString", "newUserValue1", registry); + newUser.set("myUserInteger", 845, registry); registry.register(topMap); registry.register(subMap); @@ -582,13 +583,66 @@ public class QueryProfileTypeTestCase { registry.register(newUser); CompiledQueryProfileRegistry cRegistry = registry.compile(); - Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + cRegistry.getComponent("topMap")); assertEquals(0, query.errors().size()); assertEquals("newUserValue1", query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserString")); assertEquals(845, query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserInteger")); } + @Test + public void testNestedTypeName() { + ComponentId.resetGlobalCountersForTests(); + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType type = new QueryProfileType("testType"); + registry.getTypeRegistry().register(type); + type.addField(new FieldDescription("ranking.features.query(embedding_profile)", + "tensor<float>(model{},x[128])"), + registry.getTypeRegistry()); + QueryProfile test = new QueryProfile("test"); + registry.register(test); + test.setType(type); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo", cRegistry.getComponent("test")); + + // With a prefix we're not in the built-in type space + query.properties().set("prefix.ranking.foo", 0.1); + assertEquals(0.1, query.properties().get("prefix.ranking.foo")); + } + + @Test + public void testNestedTypeNameUsingBuiltInTypes() { + ComponentId.resetGlobalCountersForTests(); + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType type = new QueryProfileType("testType"); + type.inherited().add(Query.getArgumentType()); // Include native type checking + registry.getTypeRegistry().register(type); + type.addField(new FieldDescription("ranking.features.query(embedding_profile)", + "tensor<float>(model{},x[128])"), + registry.getTypeRegistry()); + QueryProfile test = new QueryProfile("test"); + registry.register(test); + test.setType(type); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo", cRegistry.getComponent("test")); + + // Cannot set a property in a strict built-in type + try { + query.properties().set("ranking.foo", 0.1); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("'foo' is not a valid property in 'ranking'. See the query api for valid keys starting by 'ranking'.", + e.getCause().getMessage()); + } + + // With a prefix we're not in the built-in type space + query.properties().set("prefix.ranking.foo", 0.1); + assertEquals(0.1, query.properties().get("prefix.ranking.foo")); + } + private void assertWrongType(QueryProfile profile,String typeName,String name,Object value) { try { profile.set(name,value, registry); diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java index d4847e91987..18929d75c7f 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java @@ -16,52 +16,54 @@ import static org.junit.Assert.assertEquals; */ public class ParametersTestCase { + private final double delta = 0.000001; + @Test public void testSettingRankProperty() { - Query query=new Query("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)"); - assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + Query query = new Query("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)"); + assertEquals("[(a:1,b:2)]", query.getRanking().getProperties().get("dotProduct.X").toString()); } @Test public void testSettingRankPropertyAsAlias() { - Query query=new Query("?query=test&rankproperty.dotProduct.X=(a:1,b:2)"); - assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + Query query = new Query("?query=test&rankproperty.dotProduct.X=(a:1,b:2)"); + assertEquals("[(a:1,b:2)]", query.getRanking().getProperties().get("dotProduct.X").toString()); } @Test public void testSettingRankFeature() { - Query query=new Query("?query=test&ranking.features.matches=3"); - assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + Query query = new Query("?query=test&ranking.features.matches=3"); + assertEquals(3, query.getRanking().getFeatures().getDouble("matches").getAsDouble(), delta); } @Test public void testSettingRankFeatureAsAlias() { - Query query=new Query("?query=test&rankfeature.matches=3"); - assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + Query query = new Query("?query=test&rankfeature.matches=3"); + assertEquals(3, query.getRanking().getFeatures().getDouble("matches").getAsDouble(), delta); } @Test public void testSettingRankPropertyWithQueryProfile() { - Query query=new Query(HttpRequest.createTestRequest("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); - assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + Query query = new Query(HttpRequest.createTestRequest("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); + assertEquals("[(a:1,b:2)]", query.getRanking().getProperties().get("dotProduct.X").toString()); } @Test public void testSettingRankPropertyAsAliasWithQueryProfile() { - Query query=new Query(HttpRequest.createTestRequest("?query=test&rankproperty.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); - assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + Query query = new Query(HttpRequest.createTestRequest("?query=test&rankproperty.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); + assertEquals("[(a:1,b:2)]", query.getRanking().getProperties().get("dotProduct.X").toString()); } @Test public void testSettingRankFeatureWithQueryProfile() { - Query query=new Query(HttpRequest.createTestRequest("?query=test&ranking.features.matches=3", Method.GET), createProfile()); - assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + Query query = new Query(HttpRequest.createTestRequest("?query=test&ranking.features.matches=3", Method.GET), createProfile()); + assertEquals(3, query.getRanking().getFeatures().getDouble("matches").getAsDouble(), delta); } @Test public void testSettingRankFeatureAsAliasWithQueryProfile() { - Query query=new Query(HttpRequest.createTestRequest("?query=test&rankfeature.matches=3", Method.GET), createProfile()); - assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + Query query = new Query(HttpRequest.createTestRequest("?query=test&rankfeature.matches=3", Method.GET), createProfile()); + assertEquals(3, query.getRanking().getFeatures().getDouble("matches").getAsDouble(), delta); } public CompiledQueryProfile createProfile() { diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java index 8aff81a90db..94e780f6e70 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java @@ -34,6 +34,7 @@ public class RankFeaturesTestCase { } @Test + @SuppressWarnings("deprecation") public void requireThatRankFeaturesUsingDoubleAndDoubleToStringEncodeTheSameWay() { RankFeatures withDouble = new RankFeatures(); withDouble.put("query(myDouble)", 3.8); diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java index e2f73617eb6..b617300fb2b 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java @@ -13,37 +13,39 @@ import static org.junit.Assert.assertFalse; */ public class RankingTestCase { - /** tests setting rank feature values */ + private static final double delta = 0.00000001; + + /** Tests setting rank feature values */ @Test public void testRankFeatures() { // Check initializing from query Query query = new Query("?query=test&ranking.features.query(name)=0.1&ranking.features.fieldMatch(foo)=0.2"); - assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); - assertEquals("0.2", query.getRanking().getFeatures().get("fieldMatch(foo)")); - assertEquals("{\"query(name)\":\"0.1\",\"fieldMatch(foo)\":\"0.2\"}", query.getRanking().getFeatures().toString()); + assertEquals(0.1, query.getRanking().getFeatures().getDouble("query(name)").getAsDouble(), delta); + assertEquals(0.2, query.getRanking().getFeatures().getDouble("fieldMatch(foo)").getAsDouble(), delta); + assertEquals("{\"query(name)\":0.1,\"fieldMatch(foo)\":0.2}", query.getRanking().getFeatures().toString()); // Test cloning Query clone = query.clone(); - assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); - assertEquals("0.2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals(0.1, query.getRanking().getFeatures().getDouble("query(name)").getAsDouble(), delta); + assertEquals(0.2, query.getRanking().getFeatures().getDouble("fieldMatch(foo)").getAsDouble(), delta); // Check programmatic setting + that the clone really has a separate object assertFalse(clone.getRanking().getFeatures() == query.getRanking().getFeatures()); clone.properties().set("ranking.features.query(name)","0.3"); - assertEquals("0.3", clone.getRanking().getFeatures().get("query(name)")); - assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); + assertEquals(0.3, clone.getRanking().getFeatures().getDouble("query(name)").getAsDouble(), delta); + assertEquals(0.1, query.getRanking().getFeatures().getDouble("query(name)").getAsDouble(), delta); // Check getting - assertEquals("0.3",clone.properties().get("ranking.features.query(name)")); + assertEquals(0.3, clone.properties().getDouble("ranking.features.query(name)"), 0.0000001); // Check map access assertEquals(2, query.getRanking().getFeatures().asMap().size()); - assertEquals("0.2", query.getRanking().getFeatures().asMap().get("fieldMatch(foo)")); - query.getRanking().getFeatures().asMap().put("fieldMatch(foo)", "0.3"); - assertEquals("0.3", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals(0.2, query.getRanking().getFeatures().asMap().get("fieldMatch(foo)")); + query.getRanking().getFeatures().asMap().put("fieldMatch(foo)", 0.3); + assertEquals(0.3, query.getRanking().getFeatures().getDouble("fieldMatch(foo)").getAsDouble(), delta); } - //This test is order dependent. Fix this!! + // This test is order dependent. Fix this!! @Test public void test_setting_rank_feature_values() { // Check initializing from query @@ -73,7 +75,7 @@ public class RankingTestCase { /** Test setting sorting to null does not cause an exception. */ @Test public void testResetSorting() { - Query q=new Query(); + Query q = new Query(); q.getRanking().setSorting((Sorting)null); q.getRanking().setSorting((String)null); } @@ -82,13 +84,13 @@ public class RankingTestCase { @Test public void testFeatureOverride() { Query query = new Query("?query=abc&featureoverride.something=2"); - assertEquals("2", query.getRanking().getFeatures().get("something")); + assertEquals(2, query.getRanking().getFeatures().getDouble("something").getAsDouble(), 0.0000001); } @Test public void testStructuredRankProperty() { Query query = new Query("?query=abc&rankproperty.distanceToPath(gps_position).path=(0,0,10,0,10,5,20,5)"); - assertEquals("(0,0,10,0,10,5,20,5)", query.getRanking().getProperties().get("distanceToPath(gps_position).path").get(0).toString()); + assertEquals("(0,0,10,0,10,5,20,5)", query.getRanking().getProperties().get("distanceToPath(gps_position).path").get(0)); } } diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java index a57ed07017f..5b16802a65e 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java @@ -18,13 +18,29 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import java.io.*; -import java.util.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.hamcrest.CoreMatchers.*; -import static org.junit.Assert.*; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertThat; /** * @author bratseth @@ -32,8 +48,8 @@ import static org.junit.Assert.*; */ public class SearchChainConfigurerTestCase { - private static Random random = new Random(1); - private static String topCfgDir = System.getProperty("java.io.tmpdir") + File.separator + + private static final Random random = new Random(1); + private static final String topCfgDir = System.getProperty("java.io.tmpdir") + File.separator + "SearchChainConfigurerTestCase" + File.separator; private static final String testDir = "src/test/java/com/yahoo/search/searchchain/config/test/"; @@ -132,7 +148,7 @@ public class SearchChainConfigurerTestCase { * that does not contain any bootstrap configs. */ @Test - public void testSearcherConfigUpdate() throws IOException, InterruptedException { + public void testSearcherConfigUpdate() throws IOException { File cfgDir = getCfgDir(); copyFile(testDir + "handlers.cfg", cfgDir + "/handlers.cfg"); copyFile(testDir + "qr-search.cfg", cfgDir + "/qr-search.cfg"); @@ -171,9 +187,9 @@ public class SearchChainConfigurerTestCase { // Searchers with unchanged config (or that takes no config) are the same as before. Searcher s = searchers.getComponent(DeclaredTestSearcher.class.getName()); - assertThat((DeclaredTestSearcher)s, sameInstance(noConfigSearcher)); + assertThat(s, sameInstance(noConfigSearcher)); s = searchers.getComponent(StringSearcher.class.getName()); - assertThat((StringSearcher)s, sameInstance(stringSearcher)); + assertThat(s, sameInstance(stringSearcher)); configurer.shutdown(); cleanup(cfgDir); @@ -219,7 +235,7 @@ public class SearchChainConfigurerTestCase { assertThat(getSearchChainRegistryFrom(configurer).getSearcherRegistry(), not(searchers)); searchers = getSearchChainRegistryFrom(configurer).getSearcherRegistry(); assertThat(searchers.getComponentCount(), is(3)); - assertThat((IntSearcher)searchers.getComponent(IntSearcher.class.getName()), sameInstance(intSearcher)); + assertThat(searchers.getComponent(IntSearcher.class.getName()), sameInstance(intSearcher)); assertThat(searchers.getComponent(ConfigurableSearcher.class.getName()), instanceOf(ConfigurableSearcher.class)); assertThat(searchers.getComponent(DeclaredTestSearcher.class.getName()), instanceOf(DeclaredTestSearcher.class)); assertThat(searchers.getComponent(StringSearcher.class.getName()), nullValue()); @@ -326,7 +342,7 @@ public class SearchChainConfigurerTestCase { if (append) { Pattern p = Pattern.compile("^[a-z]+" + "\\[\\d+\\]\\.id (.+)"); BufferedReader reader = new BufferedReader(new InputStreamReader( - new FileInputStream(new File(componentsFile)), "UTF-8")); + new FileInputStream(new File(componentsFile)), StandardCharsets.UTF_8)); while ((line = reader.readLine()) != null) { Matcher m = p.matcher(line); if (m.matches() && !m.group(1).equals(HandlersConfigurerDi.RegistriesHack.class.getName())) { @@ -337,7 +353,7 @@ public class SearchChainConfigurerTestCase { reader.close(); } BufferedReader reader = new BufferedReader(new InputStreamReader( - new FileInputStream(new File(configFile)), "UTF-8")); + new FileInputStream(new File(configFile)), StandardCharsets.UTF_8)); Pattern component = Pattern.compile("^" + componentType + "\\[\\d+\\]\\.id (.+)"); while ((line = reader.readLine()) != null) { Matcher m = component.matcher(line); @@ -353,7 +369,7 @@ public class SearchChainConfigurerTestCase { buf.append("components[").append(i++).append("].id ").append(ExecutionFactory.class.getName()).append("\n"); buf.insert(0, "components["+i+"]\n"); - Writer writer = new OutputStreamWriter(new FileOutputStream(new File(componentsFile)), "UTF-8"); + Writer writer = new OutputStreamWriter(new FileOutputStream(new File(componentsFile)), StandardCharsets.UTF_8); writer.write(buf.toString()); writer.flush(); writer.close(); diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg index ad20005e7ad..53811bdf536 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg @@ -1,2 +1,3 @@ -handler[1] +handler[2] handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg index ad20005e7ad..53811bdf536 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg @@ -1,2 +1,3 @@ -handler[1] +handler[2] handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg index 8a985f92d10..e1d5c418c6a 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg @@ -22,3 +22,4 @@ components[8].classId com.yahoo.search.searchchain.config.test.twosearchers.Mult components[8].bundle twosearchers components[9].id com.yahoo.search.handler.SearchHandler components[10].id com.yahoo.container.handler.config.HandlersConfigurerDi$RegistriesHack +components[11].id com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 0cbf3a6f92c..c49603737a6 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -1,14 +1,11 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.prelude.searcher; +package com.yahoo.search.searchers; import com.google.common.util.concurrent.MoreExecutors; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.config.subscription.RawSource; -import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; @@ -20,15 +17,11 @@ import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.Result; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchers.ValidateNearestNeighborSearcher; import com.yahoo.search.yql.YqlParser; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; -import java.util.*; - import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -100,7 +93,7 @@ public class ValidateNearestNeighborTestCase { } private String makeQuery(String attributeTensor, String queryTensor) { - return "select * from sources * where [{\"targetNumHits\":1}]nearestNeighbor(" + attributeTensor + ", " + queryTensor + ");"; + return "select * from sources * where [{\"targetHits\":1}]nearestNeighbor(" + attributeTensor + ", " + queryTensor + ");"; } @Test @@ -139,12 +132,24 @@ public class ValidateNearestNeighborTestCase { assertEquals(ErrorMessage.createIllegalQuery(message), r.hits().getError()); } + static String desc(String field, String qt, int th, String errmsg) { + StringBuilder r = new StringBuilder(); + r.append("NEAREST_NEIGHBOR {"); + r.append("field=").append(field); + r.append(",queryTensorName=").append(qt); + r.append(",hnsw.exploreAdditionalHits=0"); + r.append(",approximate=true"); + r.append(",targetHits=").append(th); + r.append("} ").append(errmsg); + return r.toString(); + } + @Test public void testMissingTargetNumHits() { String q = "select * from sources * where nearestNeighbor(dvector,qvector);"; Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=0} has invalid targetNumHits", r); + assertErrMsg(desc("dvector", "qvector", 0, "has invalid targetHits 0: Must be >= 1"), r); } @Test @@ -152,16 +157,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("dvector", "foo"); Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=foo,targetNumHits=1} query tensor not found", r); - } - - @Test - public void testQueryTensorWrongType() { - String q = makeQuery("dvector", "qvector"); - Result r = doSearch(searcher, q, "tensor string"); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} query tensor should be a tensor, was: class java.lang.String", r); - r = doSearch(searcher, q, null); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} query tensor should be a tensor, was: null", r); + assertErrMsg(desc("dvector", "foo", 1, "requires a tensor rank feature query(foo) but this is not present"), r); } @Test @@ -169,7 +165,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("dvector", "qvector"); Tensor t = makeTensor(tt_dense_dvector_2, 2); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} field type tensor(x[3]) does not match query tensor type tensor(x[2])", r); + assertErrMsg(desc("dvector", "qvector", 1, "field type tensor(x[3]) does not match query type tensor(x[2])"), r); } @Test @@ -177,7 +173,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("foo", "qvector"); Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=foo,queryTensorName=qvector,targetNumHits=1} field is not an attribute", r); + assertErrMsg(desc("foo", "qvector", 1, "field is not an attribute"), r); } @Test @@ -185,7 +181,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("simple", "qvector"); Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=simple,queryTensorName=qvector,targetNumHits=1} field is not a tensor", r); + assertErrMsg(desc("simple", "qvector", 1, "field is not a tensor"), r); } @Test @@ -193,7 +189,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("sparse", "qvector"); Tensor t = makeTensor(tt_sparse_vector_x); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=sparse,queryTensorName=qvector,targetNumHits=1} tensor type tensor(x{}) is not a dense vector", r); + assertErrMsg(desc("sparse", "qvector", 1, "tensor type tensor(x{}) is not a dense vector"), r); } @Test @@ -201,14 +197,14 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("matrix", "qvector"); Tensor t = makeMatrix(tt_dense_matrix_xy); Result r = doSearch(searcher, q, t); - assertErrMsg("NEAREST_NEIGHBOR {field=matrix,queryTensorName=qvector,targetNumHits=1} tensor type tensor(x[3],y[1]) is not a dense vector", r); + assertErrMsg(desc("matrix", "qvector", 1, "tensor type tensor(x[3],y[1]) is not a dense vector"), r); } - private static Result doSearch(ValidateNearestNeighborSearcher searcher, String yqlQuery, Object qTensor) { + private static Result doSearch(ValidateNearestNeighborSearcher searcher, String yqlQuery, Tensor qTensor) { QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery)); Query query = new Query(); query.getModel().getQueryTree().setRoot(queryTree.getRoot()); - query.getRanking().getProperties().put("qvector", qTensor); + query.getRanking().getFeatures().put("query(qvector)", qTensor); SearchDefinition searchDefinition = new SearchDefinition("document"); IndexFacts indexFacts = new IndexFacts(new IndexModel(searchDefinition)); Execution.Context context = new Execution.Context(null, indexFacts, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java index dbeced57c52..aa507d38be5 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java @@ -5,6 +5,7 @@ import static org.junit.Assert.*; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import org.junit.After; import org.junit.Before; @@ -23,50 +24,50 @@ import com.yahoo.text.Utf8; /** * Functional test for InputCheckingSearcher. * - * @author <a href="mailto:[email protected]">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class InputCheckingSearcherTestCase { Execution execution; @Before - public void setUp() throws Exception { + public void setUp() { execution = new Execution(new Chain<Searcher>(new InputCheckingSearcher(MetricReceiver.nullImplementation)), - Execution.Context.createContextStub(new IndexFacts())); + Execution.Context.createContextStub(new IndexFacts())); } @After - public void tearDown() throws Exception { + public void tearDown() { execution = null; } @Test - public final void testCommonCase() { + public void testCommonCase() { Result r = execution.search(new Query("/search/?query=three+blind+mice")); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButAsciiOnly() { + public void candidateButAsciiOnly() { Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a")); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButValid() throws UnsupportedEncodingException { + public void candidateButValid() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { + public void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8"))); assertNull(r.hits().getErrorHit()); } @Test - public final void testDoubleEncoded() throws UnsupportedEncodingException { + public void testDoubleEncoded() throws UnsupportedEncodingException { String rawQuery = "å å å å å å"; byte[] encodedOnce = Utf8.toBytes(rawQuery); char[] secondEncodingBuffer = new char[encodedOnce.length]; @@ -74,33 +75,42 @@ public class InputCheckingSearcherTestCase { secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF); } String query = new String(secondEncodingBuffer); - Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8"))); + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, StandardCharsets.UTF_8))); assertEquals(1, r.hits().getErrorHit().errors().size()); } @Test - public final void testRepeatedConsecutiveTermsInPhrase() { - Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + public void testRepeatedConsecutiveTermsInPhrase() { + Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); + assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); assertNull(r.hits().getErrorHit()); } + @Test - public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { - Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + public void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { + Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22")); assertNotNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); + assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); + r = execution.search(new Query("/search/?query=%22a.b.0.0.0.1.0.0.0.c%22")); assertNull(r.hits().getErrorHit()); } + @Test - public final void testThatMaxRepeatedTermsInPhraseIs10() { - Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a")); + public void testThatMaxRepeatedTermsInPhraseIs10() { + Result r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a%22")); assertNull(r.hits().getErrorHit()); - r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a")); + r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a%22")); assertNotNull(r.hits().getErrorHit()); + assertEquals("Phrase contains more than 10 occurrences of term 'a' in phrase : \"0 a 1 a 2 a 3 a 4 a 5 a 6 a 7 a 8 a 9 a 10 a\"", + r.hits().getErrorHit().errorIterator().next().getDetailedMessage()); } + } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 84565472820..3a67245e912 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.test; -import com.google.common.collect.ImmutableList; import com.yahoo.component.chain.Chain; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; @@ -17,6 +16,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.IndexedItem; @@ -31,12 +31,10 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.grouping.GroupingQueryParser; -import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.SessionId; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.result.Hit; @@ -49,23 +47,19 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -240,10 +234,10 @@ public class QueryTestCase { @Test public void test_that_cloning_preserves_timeout() { Query original = new Query(); - original.setTimeout(9876l); + original.setTimeout(9876L); Query clone = original.clone(); - assertThat(clone.getTimeout(), is(9876l)); + assertEquals(9876L, clone.getTimeout()); } @Test @@ -297,9 +291,7 @@ public class QueryTestCase { fail("Above statement should throw"); } catch (QueryException e) { // As expected. - assertThat( - Exceptions.toMessageString(e), - containsString("Could not set 'timeout' to 'nalle': Error parsing 'nalle': Invalid number 'nalle'")); + assertTrue(Exceptions.toMessageString(e).contains("Could not set 'timeout' to 'nalle': Error parsing 'nalle': Invalid number 'nalle'")); } } @@ -356,6 +348,61 @@ public class QueryTestCase { } @Test + public void testQueryProfileClearAndSet() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("b", "b-value", null); + Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + assertEquals("b-value", q.properties().get("b")); + assertContains(q.properties().listProperties("b"), "b-value"); + + q.properties().set("b", null, null); + assertContains(q.properties().listProperties("b"), (Object)null); + + q.properties().set("b", "b-value", null); + assertEquals("b-value", q.properties().get("b")); + assertContains(q.properties().listProperties("b"), "b-value"); + } + + @Test + public void testQueryProfileClearValue() { + QueryProfile profile = new QueryProfile("myProfile"); + profile.set("a", "a-value", null); + profile.set("b", "b-value", null); + profile.set("b.c", "b.c-value", null); + profile.set("b.d", "b.d-value", null); + Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); + assertEquals("a-value", q.properties().get("a")); + assertEquals("b-value", q.properties().get("b")); + assertEquals("b.c-value", q.properties().get("b.c")); + assertEquals("b.d-value", q.properties().get("b.d")); + assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value"); + + q.properties().set("a", null, null); + assertEquals(null, q.properties().get("a")); + + q.properties().set("b", null, null); + assertEquals(null, q.properties().get("b")); + assertEquals("b.c-value", q.properties().get("b.c")); + assertEquals("b.d-value", q.properties().get("b.d")); + assertContains(q.properties().listProperties("b"), null, "b.c-value", "b.d-value"); + + q.properties().set("b", "b-value", null); + q.properties().set("b.e", "b.e-value", null); + q.properties().set("b.f", "b.f-value", null); + assertEquals("b-value", q.properties().get("b")); + assertEquals("b.e-value", q.properties().get("b.e")); + assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value", "b.e-value", "b.f-value"); + + q.properties().clearAll("b"); + assertEquals(null, q.properties().get("b")); + assertEquals(null, q.properties().get("b.c")); + assertEquals(null, q.properties().get("b.d")); + assertEquals(null, q.properties().get("b.e")); + assertEquals(null, q.properties().get("b.f")); + assertContains(q.properties().listProperties("b"), (Object)null); + } + + @Test public void testNotEqual() { Query q = new Query("/?query=something+test&nocache"); Query p = new Query("/?query=something+test"); @@ -895,12 +942,12 @@ public class QueryTestCase { @Test public void testImplicitPhraseIsDefault() { Query query = new Query(httpEncode("?query=it's fine")); - assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString()); + assertEquals("AND (SAND it s) fine", query.getModel().getQueryTree().toString()); } @Test public void testImplicitPhrase() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -910,12 +957,12 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND myfield:'it s' myfield:fine", query.getModel().getQueryTree().toString()); + assertEquals("AND myfield:'it s' myfield:\"a b\" myfield:c", query.getModel().getQueryTree().toString()); } @Test public void testImplicitAnd() { - Query query = new Query(httpEncode("?query=myfield:it's myfield:fine")); + Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c")); SearchDefinition test = new SearchDefinition("test"); Index myField = new Index("myfield"); @@ -925,7 +972,57 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("AND (SAND myfield:it myfield:s) myfield:fine", query.getModel().getQueryTree().toString()); + assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString()); + // 'it' and 's' should have connectivity 1 + AndItem root = (AndItem)query.getModel().getQueryTree().getRoot(); + AndSegmentItem sand = (AndSegmentItem)root.getItem(0); + WordItem it = (WordItem)sand.getItem(0); + assertEquals("it", it.getWord()); + WordItem s = (WordItem)sand.getItem(1); + assertEquals("s", s.getWord()); + assertEquals(s, it.getConnectedItem()); + assertEquals(1.0, it.getConnectivity(), 0.00000001); + } + + @Test + public void testImplicitAndConnectivity() { + SearchDefinition test = new SearchDefinition("test"); + Index myField = new Index("myfield"); + myField.addCommand("phrase-segmenting false"); + test.addIndex(myField); + IndexModel indexModel = new IndexModel(test); + + { + Query query = new Query(httpEncode("?query=myfield:b.c.d")); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + assertEquals("AND myfield:b myfield:c myfield:d", query.getModel().getQueryTree().toString()); + AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); + WordItem b = (WordItem) root.getItem(0); + WordItem c = (WordItem) root.getItem(1); + WordItem d = (WordItem) root.getItem(2); + assertEquals(c, b.getConnectedItem()); + assertEquals(1.0, b.getConnectivity(), 0.00000001); + assertEquals(d, c.getConnectedItem()); + assertEquals(1.0, c.getConnectivity(), 0.00000001); + } + + { + Query query = new Query(httpEncode("?query=myfield:a myfield:b.c.d myfield:e")); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); + assertEquals("AND myfield:a myfield:b myfield:c myfield:d myfield:e", query.getModel().getQueryTree().toString()); + AndItem root = (AndItem) query.getModel().getQueryTree().getRoot(); + WordItem a = (WordItem) root.getItem(0); + WordItem b = (WordItem) root.getItem(1); + WordItem c = (WordItem) root.getItem(2); + WordItem d = (WordItem) root.getItem(3); + WordItem e = (WordItem) root.getItem(4); + assertNull(a.getConnectedItem()); + assertEquals(c, b.getConnectedItem()); + assertEquals(1.0, b.getConnectivity(), 0.00000001); + assertEquals(d, c.getConnectedItem()); + assertEquals(1.0, c.getConnectivity(), 0.00000001); + assertNull(d.getConnectedItem()); + } } @Test @@ -940,7 +1037,7 @@ public class QueryTestCase { IndexModel indexModel = new IndexModel(test); query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel)))); - assertEquals("myfield:\"it s fine\"", query.getModel().getQueryTree().toString()); + assertEquals("myfield:\"'it s' fine\"", query.getModel().getQueryTree().toString()); } @Test @@ -996,6 +1093,19 @@ public class QueryTestCase { assertEquals(expectedDetectionText, mockLinguistics.detector.lastDetectionText); } + private void assertContains(Map<String, Object> properties, Object ... expectedValues) { + if (expectedValues == null) { + assertEquals(1, properties.size()); + assertTrue("Contains value null", properties.containsValue(null)); + } + else { + assertEquals(properties + " contains values " + Arrays.toString(expectedValues), + expectedValues.length, properties.size()); + for (Object expectedValue : expectedValues) + assertTrue("Contains value " + expectedValue, properties.containsValue(expectedValue)); + } + } + /** A linguistics instance which records the last language detection text passed to it */ private static class MockLinguistics extends SimpleLinguistics { diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index 1106d8c3999..f8e930fa19d 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -119,6 +119,14 @@ public class VespaSerializerTestCase { } @Test + public void testGeoLocation() { + parseAndConfirm("geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\")"); + parseAndConfirm("geoLocation(headquarters, 37.41638, -122.024683, \"180.0 deg\")"); + parseAndConfirm("geoLocation(home, -17.0, 42.0, \"0.0 deg\")"); + parseAndConfirm("geoLocation(workplace, -12.0, -34.0, \"-1.0 deg\")"); + } + + @Test public void testNear() { parseAndConfirm("title contains near(\"a\", \"b\")"); parseAndConfirm("title contains ([{\"distance\": 50}]near(\"a\", \"b\"))"); @@ -128,6 +136,9 @@ public class VespaSerializerTestCase { public void testNearestNeighbor() { parseAndConfirm("[{\"label\": \"foo\", \"targetNumHits\": 1000}]nearestNeighbor(semantic_embedding, my_property)"); parseAndConfirm("[{\"targetNumHits\": 42}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{\"targetNumHits\": 1, \"hnsw.exploreAdditionalHits\": 76}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{\"targetNumHits\": 2, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)"); + parseAndConfirm("[{\"targetNumHits\": 3, \"hnsw.exploreAdditionalHits\": 67, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 5eb1f3e3de1..62a9e27cd96 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -130,13 +130,13 @@ public class YqlParserTestCase { @Test public void testComplexExpression() { - String queryTreeYql = "rank((((filter contains ([{\"origin\": {\"original\": \"filter:VideoAdsCappingTestCPM\", \"offset\": 7, \"length\": 22}, \"normalizeCase\": false, \"id\": 1}]\"videoadscappingtestcpm\") AND hasRankRestriction contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}]\"0\") AND ((objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}]\"install_app\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}]\"default\")) AND !(objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}]\"install_app\"))) AND advt_age = ([{\"id\": 9}]2147483647) AND advt_gender contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}]\"all\") AND advt_all_segments = ([{\"id\": 11}]2147483647) AND advt_keywords contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}]\"all\") AND advMobilePlatform = ([{\"id\": 13}]2147483647) AND advMobileDeviceType = ([{\"id\": 14}]2147483647) AND advMobileCon = ([{\"id\": 15}]2147483647) AND advMobileOSVersions = ([{\"id\": 16}]2147483647) AND advCarrier = ([{\"id\": 17}]2147483647) AND ([{\"id\": 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}]\"adv_tuesday\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}]\"all\")) AND isAppReengagementAd = ([{\"id\": 23}]0) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}]\"default\") AND serveWithPromotionOnly = ([{\"id\": 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{\"id\": 27}]0) AND budgetResellerThrottleRateFilter = ([{\"id\": 28}]0) AND (isMystiqueRequired = ([{\"id\": 29}]0) OR (isMystiqueRequired = ([{\"id\": 30}]1) AND useBcFactorFilter = ([{\"id\": 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{\"id\": 32}]55) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}]\"default\"))) AND !(useBcFactorFilter = ([{\"id\": 34}]1)) OR ((useBcFactorFilter = ([{\"id\": 35}]1) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}]\"default\") AND (bcFactorTiers = ([{\"id\": 38}]127) OR bcFactorTiers = ([{\"id\": 39}]0)) AND ((firstPriceEnforced = ([{\"id\": 40}]0) AND (secondPriceEnforced = ([{\"id\": 41}]1) OR isPrivateDeal = ([{\"id\": 42}]0) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}]\"default\")) AND !(bcActiveTier = ([{\"id\": 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{\"id\": 45}]18)))) AND !(isOutOfDailyBudget = ([{\"id\": 37}]1))) AND testCreative = ([{\"id\": 46}]0) AND advt_geo = ([{\"id\": 47}]2147483647) AND ((adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}]\"strm_video\") AND isPortraitVideo = ([{\"id\": 49}]0)) OR adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}]\"stream_ad\")) AND ((isCPM = ([{\"id\": 51}]0) AND isOCPC = ([{\"id\": 52}]0) AND isECPC = ([{\"id\": 53}]0) AND ((priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}]\"cpcv\") AND bid >= ([{\"id\": 55}]0.005)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}]\"cpv\") AND bid >= ([{\"id\": 57}]0.01)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}]\"cpc\") AND bid >= ([{\"id\": 59}]0.05)) OR (objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}]\"promote_content\") AND bid >= ([{\"id\": 61}]0.01)) OR hasFloorPriceUsd = ([{\"id\": 62}]1))) OR isECPC = ([{\"id\": 63}]1) OR (isCPM = ([{\"id\": 64}]1) AND isOCPM = ([{\"id\": 65}]0) AND (([{\"id\": 66}]range(bid, 0.25, Infinity)) OR hasFloorPriceUsd = ([{\"id\": 67}]1)))) AND start_date <= ([{\"id\": 68}]1572976776299L) AND end_date >= ([{\"id\": 69}]1572976776299L))) AND !(isHoldoutAd = ([{\"id\": 25}]1))) AND !((disclaimerExtensionsTypes contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}]\"pharma\") OR ([{\"id\": 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{\"id\": 72}]1) OR blacklist_section_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}]\"223\") OR blacklist_publisher_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}]\"223\") OR blacklist_site_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}]\"223\"))), [{\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1}), [{\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1}), [{\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1}), [{\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1}), [{\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1}), [{\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1}), (bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{\"id\": 88, \"weight\": 1}]223), [{\"id\": 89, \"label\": \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})) limit 0 timeout 3980 | all(group(adTypeForGrouping) each(group(advertiser_id) max(11) output(count() as(groupingCounter)) each(max(1) each(output(summary())))))"; + String queryTreeYql = "rank((((filter contains ([{\"origin\": {\"original\": \"filter:VideoAdsCappingTestCPM\", \"offset\": 7, \"length\": 22}, \"normalizeCase\": false, \"id\": 1}]\"videoadscappingtestcpm\") AND hasRankRestriction contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}]\"0\") AND ((objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}]\"install_app\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}]\"default\")) AND !(objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}]\"install_app\"))) AND advt_age = ([{\"id\": 9}]2147483647) AND advt_gender contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}]\"all\") AND advt_all_segments = ([{\"id\": 11}]2147483647) AND advt_keywords contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}]\"all\") AND advMobilePlatform = ([{\"id\": 13}]2147483647) AND advMobileDeviceType = ([{\"id\": 14}]2147483647) AND advMobileCon = ([{\"id\": 15}]2147483647) AND advMobileOSVersions = ([{\"id\": 16}]2147483647) AND advCarrier = ([{\"id\": 17}]2147483647) AND ([{\"id\": 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}]\"adv_tuesday\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}]\"all\")) AND isAppReengagementAd = ([{\"id\": 23}]0) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}]\"default\") AND serveWithPromotionOnly = ([{\"id\": 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{\"id\": 27}]0) AND budgetResellerThrottleRateFilter = ([{\"id\": 28}]0) AND (isMystiqueRequired = ([{\"id\": 29}]0) OR (isMystiqueRequired = ([{\"id\": 30}]1) AND useBcFactorFilter = ([{\"id\": 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{\"id\": 32}]55) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}]\"default\"))) AND !(useBcFactorFilter = ([{\"id\": 34}]1)) OR ((useBcFactorFilter = ([{\"id\": 35}]1) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}]\"default\") AND (bcFactorTiers = ([{\"id\": 38}]127) OR bcFactorTiers = ([{\"id\": 39}]0)) AND ((firstPriceEnforced = ([{\"id\": 40}]0) AND (secondPriceEnforced = ([{\"id\": 41}]1) OR isPrivateDeal = ([{\"id\": 42}]0) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}]\"default\")) AND !(bcActiveTier = ([{\"id\": 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{\"id\": 45}]18)))) AND !(isOutOfDailyBudget = ([{\"id\": 37}]1))) AND testCreative = ([{\"id\": 46}]0) AND advt_geo = ([{\"id\": 47}]2147483647) AND ((adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}]\"strm_video\") AND isPortraitVideo = ([{\"id\": 49}]0)) OR adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}]\"stream_ad\")) AND ((isCPM = ([{\"id\": 51}]0) AND isOCPC = ([{\"id\": 52}]0) AND isECPC = ([{\"id\": 53}]0) AND ((priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}]\"cpcv\") AND bid >= ([{\"id\": 55}]0.005)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}]\"cpv\") AND bid >= ([{\"id\": 57}]0.01)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}]\"cpc\") AND bid >= ([{\"id\": 59}]0.05)) OR (objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}]\"promote_content\") AND bid >= ([{\"id\": 61}]0.01)) OR hasFloorPriceUsd = ([{\"id\": 62}]1))) OR isECPC = ([{\"id\": 63}]1) OR (isCPM = ([{\"id\": 64}]1) AND isOCPM = ([{\"id\": 65}]0) AND (([{\"id\": 66}]range(bid, 0.25, Infinity)) OR hasFloorPriceUsd = ([{\"id\": 67}]1)))) AND start_date <= ([{\"id\": 68}]1572976776299L) AND end_date >= ([{\"id\": 69}]1572976776299L))) AND !(isHoldoutAd = ([{\"id\": 25}]1))) AND !((disclaimerExtensionsTypes contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}]\"pharma\") OR ([{\"id\": 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{\"id\": 72}]1) OR blocked_section_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}]\"223\") OR blocked_publisher_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}]\"223\") OR blocked_site_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}]\"223\"))), [{\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1}), [{\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1}), [{\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1}), [{\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1}), [{\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1}), [{\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1}), (bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{\"id\": 88, \"weight\": 1}]223), [{\"id\": 89, \"label\": \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})) limit 0 timeout 3980 | all(group(adTypeForGrouping) each(group(advertiser_id) max(11) output(count() as(groupingCounter)) each(max(1) each(output(summary())))))"; QueryTree parsed = assertParse("select * from sources * where " + queryTreeYql + ";", - "RANK (+(+(AND filter:VideoAdsCappingTestCPM hasRankRestriction:0 (OR (AND objective:install_app availableExtendedFields:cpiparams) (AND availableExtendedFields:appinstallinfo availableExtendedFields:appmetroplexinfo) (+dummyField:default -objective:install_app)) advt_age:2147483647 advt_gender:all advt_all_segments:2147483647 advt_keywords:all advMobilePlatform:2147483647 advMobileDeviceType:2147483647 advMobileCon:2147483647 advMobileOSVersions:2147483647 advCarrier:2147483647 WEIGHTEDSET advt_supply{[1]:\"site223\",[1]:\"pub223\",[1]:\"all\",[1]:\"sec223\"} (OR advt_day_parting:adv_tuesday!1 advt_day_parting:adv_tuesday_17!1 advt_day_parting:adv_tuesday_17_forty_five!1 advt_day_parting:all) isAppReengagementAd:0 dummyField:default serveWithPromotionOnly:0 budgetAdvertiserThrottleRateFilter:0 budgetResellerThrottleRateFilter:0 (OR isMystiqueRequired:0 (AND isMystiqueRequired:1 useBcFactorFilter:1)) (OR (+(AND budgetCampaignThrottleRateBits:55 dummyField:default) -useBcFactorFilter:1) (+(AND useBcFactorFilter:1 dummyField:default (OR bcFactorTiers:127 bcFactorTiers:0) (OR (AND firstPriceEnforced:0 (OR secondPriceEnforced:1 isPrivateDeal:0 (+dummyField:default -bcActiveTier:0))) mystiqueCampaignThrottleRateBits:18)) -isOutOfDailyBudget:1)) testCreative:0 advt_geo:2147483647 (OR (AND adType:strm_video isPortraitVideo:0) adType:stream_ad) (OR (AND isCPM:0 isOCPC:0 isECPC:0 (OR (AND priceType:cpcv bid:[0.005;]) (AND priceType:cpv bid:[0.01;]) (AND priceType:cpc bid:[0.05;]) (AND objective:promote_content bid:[0.01;]) hasFloorPriceUsd:1)) isECPC:1 (AND isCPM:1 isOCPM:0 (OR bid:[0.25;] hasFloorPriceUsd:1))) start_date:[;1572976776299] end_date:[1572976776299;]) -isHoldoutAd:1) -(OR disclaimerExtensionsTypes:pharma WEIGHTEDSET exclusion_advt_supply{[1]:\"extsite223\",[1]:\"site223\",[1]:\"pub223\",[1]:\"sec223\"} isPersonalized:1 blacklist_section_ids:223 blacklist_publisher_ids:223 blacklist_site_ids:223)) DOTPRODUCT ocpc_max_cpc{[1]:\"0\"} DOTPRODUCT ocpc_min_cpc{[1]:\"0\"} DOTPRODUCT ocpc_max_alpha{[1]:\"0\"} DOTPRODUCT ocpc_min_alpha{[1]:\"0\"} DOTPRODUCT ocpc_alpha_0{[1]:\"0\"} DOTPRODUCT ocpc_alpha_1{[1]:\"0\"} (OR bidAdjustmentDayParting:adv_tuesday!1 bidAdjustmentDayParting:adv_tuesday_17!1 bidAdjustmentDayParting:adv_tuesday_17_forty_five!1 bidAdjustmentDayPartingForCostCap:adv_tuesday!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17_forty_five!1) bidAdjustmentForCpi:223!1 DOTPRODUCT boostingForBackfill{[1000]:\"priority\"}"); + "RANK (+(+(AND filter:VideoAdsCappingTestCPM hasRankRestriction:0 (OR (AND objective:install_app availableExtendedFields:cpiparams) (AND availableExtendedFields:appinstallinfo availableExtendedFields:appmetroplexinfo) (+dummyField:default -objective:install_app)) advt_age:2147483647 advt_gender:all advt_all_segments:2147483647 advt_keywords:all advMobilePlatform:2147483647 advMobileDeviceType:2147483647 advMobileCon:2147483647 advMobileOSVersions:2147483647 advCarrier:2147483647 WEIGHTEDSET advt_supply{[1]:\"site223\",[1]:\"pub223\",[1]:\"all\",[1]:\"sec223\"} (OR advt_day_parting:adv_tuesday!1 advt_day_parting:adv_tuesday_17!1 advt_day_parting:adv_tuesday_17_forty_five!1 advt_day_parting:all) isAppReengagementAd:0 dummyField:default serveWithPromotionOnly:0 budgetAdvertiserThrottleRateFilter:0 budgetResellerThrottleRateFilter:0 (OR isMystiqueRequired:0 (AND isMystiqueRequired:1 useBcFactorFilter:1)) (OR (+(AND budgetCampaignThrottleRateBits:55 dummyField:default) -useBcFactorFilter:1) (+(AND useBcFactorFilter:1 dummyField:default (OR bcFactorTiers:127 bcFactorTiers:0) (OR (AND firstPriceEnforced:0 (OR secondPriceEnforced:1 isPrivateDeal:0 (+dummyField:default -bcActiveTier:0))) mystiqueCampaignThrottleRateBits:18)) -isOutOfDailyBudget:1)) testCreative:0 advt_geo:2147483647 (OR (AND adType:strm_video isPortraitVideo:0) adType:stream_ad) (OR (AND isCPM:0 isOCPC:0 isECPC:0 (OR (AND priceType:cpcv bid:[0.005;]) (AND priceType:cpv bid:[0.01;]) (AND priceType:cpc bid:[0.05;]) (AND objective:promote_content bid:[0.01;]) hasFloorPriceUsd:1)) isECPC:1 (AND isCPM:1 isOCPM:0 (OR bid:[0.25;] hasFloorPriceUsd:1))) start_date:[;1572976776299] end_date:[1572976776299;]) -isHoldoutAd:1) -(OR disclaimerExtensionsTypes:pharma WEIGHTEDSET exclusion_advt_supply{[1]:\"extsite223\",[1]:\"site223\",[1]:\"pub223\",[1]:\"sec223\"} isPersonalized:1 blocked_section_ids:223 blocked_publisher_ids:223 blocked_site_ids:223)) DOTPRODUCT ocpc_max_cpc{[1]:\"0\"} DOTPRODUCT ocpc_min_cpc{[1]:\"0\"} DOTPRODUCT ocpc_max_alpha{[1]:\"0\"} DOTPRODUCT ocpc_min_alpha{[1]:\"0\"} DOTPRODUCT ocpc_alpha_0{[1]:\"0\"} DOTPRODUCT ocpc_alpha_1{[1]:\"0\"} (OR bidAdjustmentDayParting:adv_tuesday!1 bidAdjustmentDayParting:adv_tuesday_17!1 bidAdjustmentDayParting:adv_tuesday_17_forty_five!1 bidAdjustmentDayPartingForCostCap:adv_tuesday!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17_forty_five!1) bidAdjustmentForCpi:223!1 DOTPRODUCT boostingForBackfill{[1000]:\"priority\"}"); String serializedQueryTreeYql = VespaSerializer.serialize(parsed); // Note: All the details here are not verified - assertEquals("rank((((filter contains ([{\"normalizeCase\": false, \"id\": 1}]\"VideoAdsCappingTestCPM\") AND hasRankRestriction contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}]\"0\") AND ((objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}]\"install_app\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}]\"default\")) AND !(objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}]\"install_app\"))) AND advt_age = ([{\"id\": 9}]2147483647) AND advt_gender contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}]\"all\") AND advt_all_segments = ([{\"id\": 11}]2147483647) AND advt_keywords contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}]\"all\") AND advMobilePlatform = ([{\"id\": 13}]2147483647) AND advMobileDeviceType = ([{\"id\": 14}]2147483647) AND advMobileCon = ([{\"id\": 15}]2147483647) AND advMobileOSVersions = ([{\"id\": 16}]2147483647) AND advCarrier = ([{\"id\": 17}]2147483647) AND ([{\"id\": 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}]\"adv_tuesday\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}]\"all\")) AND isAppReengagementAd = ([{\"id\": 23}]0) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}]\"default\") AND serveWithPromotionOnly = ([{\"id\": 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{\"id\": 27}]0) AND budgetResellerThrottleRateFilter = ([{\"id\": 28}]0) AND (isMystiqueRequired = ([{\"id\": 29}]0) OR (isMystiqueRequired = ([{\"id\": 30}]1) AND useBcFactorFilter = ([{\"id\": 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{\"id\": 32}]55) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}]\"default\"))) AND !(useBcFactorFilter = ([{\"id\": 34}]1)) OR ((useBcFactorFilter = ([{\"id\": 35}]1) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}]\"default\") AND (bcFactorTiers = ([{\"id\": 38}]127) OR bcFactorTiers = ([{\"id\": 39}]0)) AND ((firstPriceEnforced = ([{\"id\": 40}]0) AND (secondPriceEnforced = ([{\"id\": 41}]1) OR isPrivateDeal = ([{\"id\": 42}]0) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}]\"default\")) AND !(bcActiveTier = ([{\"id\": 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{\"id\": 45}]18)))) AND !(isOutOfDailyBudget = ([{\"id\": 37}]1))) AND testCreative = ([{\"id\": 46}]0) AND advt_geo = ([{\"id\": 47}]2147483647) AND ((adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}]\"strm_video\") AND isPortraitVideo = ([{\"id\": 49}]0)) OR adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}]\"stream_ad\")) AND ((isCPM = ([{\"id\": 51}]0) AND isOCPC = ([{\"id\": 52}]0) AND isECPC = ([{\"id\": 53}]0) AND ((priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}]\"cpcv\") AND bid >= ([{\"id\": 55}]0.005)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}]\"cpv\") AND bid >= ([{\"id\": 57}]0.01)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}]\"cpc\") AND bid >= ([{\"id\": 59}]0.05)) OR (objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}]\"promote_content\") AND bid >= ([{\"id\": 61}]0.01)) OR hasFloorPriceUsd = ([{\"id\": 62}]1))) OR isECPC = ([{\"id\": 63}]1) OR (isCPM = ([{\"id\": 64}]1) AND isOCPM = ([{\"id\": 65}]0) AND ([{\"id\": 66}]range(bid, 0.25, Infinity) OR hasFloorPriceUsd = ([{\"id\": 67}]1)))) AND start_date <= ([{\"id\": 68}]1572976776299L) AND end_date >= ([{\"id\": 69}]1572976776299L))) AND !(isHoldoutAd = ([{\"id\": 25}]1))) AND !((disclaimerExtensionsTypes contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}]\"pharma\") OR ([{\"id\": 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{\"id\": 72}]1) OR blacklist_section_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}]\"223\") OR blacklist_publisher_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}]\"223\") OR blacklist_site_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}]\"223\"))), ([{\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1})), ([{\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1})), ([{\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1})), ([{\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1})), ([{\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1})), ([{\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1})), (bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{\"id\": 88, \"weight\": 1}]223), ([{\"id\": 89, \"label\": \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})))", + assertEquals("rank((((filter contains ([{\"normalizeCase\": false, \"id\": 1}]\"VideoAdsCappingTestCPM\") AND hasRankRestriction contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}]\"0\") AND ((objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}]\"install_app\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}]\"cpiparams\")) OR (availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}]\"appinstallinfo\") AND availableExtendedFields contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}]\"appmetroplexinfo\")) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}]\"default\")) AND !(objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}]\"install_app\"))) AND advt_age = ([{\"id\": 9}]2147483647) AND advt_gender contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}]\"all\") AND advt_all_segments = ([{\"id\": 11}]2147483647) AND advt_keywords contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}]\"all\") AND advMobilePlatform = ([{\"id\": 13}]2147483647) AND advMobileDeviceType = ([{\"id\": 14}]2147483647) AND advMobileCon = ([{\"id\": 15}]2147483647) AND advMobileOSVersions = ([{\"id\": 16}]2147483647) AND advCarrier = ([{\"id\": 17}]2147483647) AND ([{\"id\": 18}]weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}]\"adv_tuesday\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}]\"adv_tuesday_17\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}]\"all\")) AND isAppReengagementAd = ([{\"id\": 23}]0) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}]\"default\") AND serveWithPromotionOnly = ([{\"id\": 26}]0) AND budgetAdvertiserThrottleRateFilter = ([{\"id\": 27}]0) AND budgetResellerThrottleRateFilter = ([{\"id\": 28}]0) AND (isMystiqueRequired = ([{\"id\": 29}]0) OR (isMystiqueRequired = ([{\"id\": 30}]1) AND useBcFactorFilter = ([{\"id\": 31}]1))) AND (((budgetCampaignThrottleRateBits = ([{\"id\": 32}]55) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}]\"default\"))) AND !(useBcFactorFilter = ([{\"id\": 34}]1)) OR ((useBcFactorFilter = ([{\"id\": 35}]1) AND dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}]\"default\") AND (bcFactorTiers = ([{\"id\": 38}]127) OR bcFactorTiers = ([{\"id\": 39}]0)) AND ((firstPriceEnforced = ([{\"id\": 40}]0) AND (secondPriceEnforced = ([{\"id\": 41}]1) OR isPrivateDeal = ([{\"id\": 42}]0) OR (dummyField contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}]\"default\")) AND !(bcActiveTier = ([{\"id\": 44}]0)))) OR mystiqueCampaignThrottleRateBits = ([{\"id\": 45}]18)))) AND !(isOutOfDailyBudget = ([{\"id\": 37}]1))) AND testCreative = ([{\"id\": 46}]0) AND advt_geo = ([{\"id\": 47}]2147483647) AND ((adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}]\"strm_video\") AND isPortraitVideo = ([{\"id\": 49}]0)) OR adType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}]\"stream_ad\")) AND ((isCPM = ([{\"id\": 51}]0) AND isOCPC = ([{\"id\": 52}]0) AND isECPC = ([{\"id\": 53}]0) AND ((priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}]\"cpcv\") AND bid >= ([{\"id\": 55}]0.005)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}]\"cpv\") AND bid >= ([{\"id\": 57}]0.01)) OR (priceType contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}]\"cpc\") AND bid >= ([{\"id\": 59}]0.05)) OR (objective contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}]\"promote_content\") AND bid >= ([{\"id\": 61}]0.01)) OR hasFloorPriceUsd = ([{\"id\": 62}]1))) OR isECPC = ([{\"id\": 63}]1) OR (isCPM = ([{\"id\": 64}]1) AND isOCPM = ([{\"id\": 65}]0) AND ([{\"id\": 66}]range(bid, 0.25, Infinity) OR hasFloorPriceUsd = ([{\"id\": 67}]1)))) AND start_date <= ([{\"id\": 68}]1572976776299L) AND end_date >= ([{\"id\": 69}]1572976776299L))) AND !(isHoldoutAd = ([{\"id\": 25}]1))) AND !((disclaimerExtensionsTypes contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}]\"pharma\") OR ([{\"id\": 71}]weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ([{\"id\": 72}]1) OR blocked_section_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}]\"223\") OR blocked_publisher_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}]\"223\") OR blocked_site_ids contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}]\"223\"))), ([{\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}]dotProduct(ocpc_max_cpc, {\"0\": 1})), ([{\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}]dotProduct(ocpc_min_cpc, {\"0\": 1})), ([{\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}]dotProduct(ocpc_max_alpha, {\"0\": 1})), ([{\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}]dotProduct(ocpc_min_alpha, {\"0\": 1})), ([{\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}]dotProduct(ocpc_alpha_0, {\"0\": 1})), ([{\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}]dotProduct(ocpc_alpha_1, {\"0\": 1})), (bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}]\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}]\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}]\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ([{\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}]\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ([{\"id\": 88, \"weight\": 1}]223), ([{\"id\": 89, \"label\": \"boostingForBackfill\"}]dotProduct(boostingForBackfill, {\"priority\": 1000})))", serializedQueryTreeYql); } @@ -513,7 +513,7 @@ public class YqlParserTestCase { public void testWand() { assertParse("select foo from bar where wand(description, {\"a\":1, \"b\":2});", "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}"); - assertParse("select foo from bar where [ {\"scoreThreshold\": 13.3, \"targetNumHits\": 7, " + + assertParse("select foo from bar where [ {\"scoreThreshold\": 13.3, \"targetHits\": 7, " + "\"thresholdBoostFactor\": 2.3} ]wand(description, {\"a\":1, \"b\":2});", "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}"); } @@ -548,11 +548,31 @@ public class YqlParserTestCase { } @Test + public void testGeoLocation() { + assertParse("select foo from bar where geoLocation(workplace, 63.418417, 10.433033, \"0.5 deg\");", + "GEO_LOCATION workplace:(2,10433033,63418417,500000,0,1,0,1921876103)"); + assertParse("select foo from bar where geoLocation(headquarters, \"37.416383\", \"-122.024683\", \"100 miles\");", + "GEO_LOCATION headquarters:(2,-122024683,37416383,1450561,0,1,0,3411238761)"); + assertParse("select foo from bar where geoLocation(home, \"E10.433033\", \"N63.418417\", \"5km\");", + "GEO_LOCATION home:(2,10433033,63418417,45066,0,1,0,1921876103)"); + + assertParseFail("select foo from bar where geoLocation(qux, 1, 2);", + new IllegalArgumentException("Expected 4 arguments, got 3.")); + assertParseFail("select foo from bar where geoLocation(qux, 2.0, \"N5.0\", \"0.5 deg\");", + new IllegalArgumentException( + "Invalid geoLocation coordinates 'Latitude: 2.0 degrees' and 'Latitude: 5.0 degrees'")); + assertParse("select foo from bar where geoLocation(workplace, -12, -34, \"-77 d\");", + "GEO_LOCATION workplace:(2,-34000000,-12000000,-1,0,1,0,4201111954)"); + } + + @Test public void testNearestNeighbor() { assertParse("select foo from bar where nearestNeighbor(semantic_embedding, my_vector);", - "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,targetNumHits=0}"); - assertParse("select foo from bar where [{\"targetNumHits\": 37}]nearestNeighbor(semantic_embedding, my_vector);", - "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,targetNumHits=37}"); + "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=0}"); + assertParse("select foo from bar where [{\"targetHits\": 37}]nearestNeighbor(semantic_embedding, my_vector);", + "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=37}"); + assertParse("select foo from bar where [{\"approximate\": false, \"hnsw.exploreAdditionalHits\": 8, \"targetHits\": 3}]nearestNeighbor(semantic_embedding, my_vector);", + "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=8,approximate=false,targetHits=3}"); } @Test @@ -595,7 +615,7 @@ public class YqlParserTestCase { public void testWeakAnd() { assertParse("select foo from bar where weakAnd(a contains \"A\", b contains \"B\");", "WAND(100) a:A b:B"); - assertParse("select foo from bar where [{\"targetNumHits\": 37}]weakAnd(a contains \"A\", " + + assertParse("select foo from bar where [{\"targetHits\": 37}]weakAnd(a contains \"A\", " + "b contains \"B\");", "WAND(37) a:A b:B"); diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 261069ea1c3..f297fd69f24 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -473,7 +473,7 @@ public class SelectTestCase { public void testWand() { assertParse("{ \"wand\": [\"description\", { \"a\": 1, \"b\": 2 }] }", "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}"); - assertParse("{ \"wand\": { \"children\": [\"description\", { \"a\": 1, \"b\": 2 }], \"attributes\": { \"scoreThreshold\": 13.3, \"targetNumHits\": 7, \"thresholdBoostFactor\": 2.3 } } }", + assertParse("{ \"wand\": { \"children\": [\"description\", { \"a\": 1, \"b\": 2 }], \"attributes\": { \"scoreThreshold\": 13.3, \"targetHits\": 7, \"thresholdBoostFactor\": 2.3 } } }", "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}"); } @@ -522,10 +522,31 @@ public class SelectTestCase { } @Test + public void testGeoLocation() { + assertParse("{ \"geoLocation\": [ \"workplace\", 63.418417, 10.433033, \"0.5 deg\" ] }", + "GEO_LOCATION workplace:(2,10433033,63418417,500000,0,1,0,1921876103)"); + assertParse("{ \"geoLocation\": [ \"headquarters\", \"37.416383\", \"-122.024683\", \"100 miles\" ] }", + "GEO_LOCATION headquarters:(2,-122024683,37416383,1450561,0,1,0,3411238761)"); + assertParse("{ \"geoLocation\": [ \"home\", \"E10.433033\", \"N63.418417\", \"5km\" ] }", + "GEO_LOCATION home:(2,10433033,63418417,45066,0,1,0,1921876103)"); + assertParse("{ \"geoLocation\": [ \"workplace\", -12.0, -34.0, \"-77 deg\" ] }", + "GEO_LOCATION workplace:(2,-34000000,-12000000,-1,0,1,0,4201111954)"); + } + + @Test + public void testNearestNeighbor() { + assertParse("{ \"nearestNeighbor\": [ \"f1field\", \"q2prop\" ] }", + "NEAREST_NEIGHBOR {field=f1field,queryTensorName=q2prop,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=0}"); + + assertParse("{ \"nearestNeighbor\": { \"children\" : [ \"f3field\", \"q4prop\" ], \"attributes\" : {\"targetHits\": 37} }}", + "NEAREST_NEIGHBOR {field=f3field,queryTensorName=q4prop,hnsw.exploreAdditionalHits=0,approximate=true,targetHits=37}"); + } + + @Test public void testWeakAnd() { assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", "WAND(100) a:A b:B"); - assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetNumHits\": 37} }}", + assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetHits\": 37} }}", "WAND(37) a:A b:B"); QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}"); @@ -652,7 +673,6 @@ public class SelectTestCase { assertGrouping(expected, parseGrouping(grouping)); } - @Test public void testMultipleGroupings() { String grouping = "[ { \"all\" : { \"group\" : \"a\", \"each\" : { \"output\" : \"count()\"}}}, { \"all\" : { \"group\" : \"b\", \"each\" : { \"output\" : \"count()\"}}} ]"; @@ -661,6 +681,20 @@ public class SelectTestCase { assertGrouping(expected, parseGrouping(grouping)); } + @Test + public void testGroupingWithPredefinedBuckets() { + String grouping = "[ { \"all\" : { \"group\" : { \"predefined\" : [ \"foo\", { \"bucket\": [1,2]}, { \"bucket\": [3,4]} ] } } } ]"; + String expected = "[[]all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>)))]"; + assertGrouping(expected, parseGrouping(grouping)); + } + + @Test + public void testMultipleOutputs() { + String grouping = "[ { \"all\" : { \"group\" : \"b\", \"each\" : {\"output\": [ \"count()\", \"avg(foo)\" ] } } } ]"; + String expected = "[[]all(group(b) each(output(count(), avg(foo))))]"; + assertGrouping(expected, parseGrouping(grouping)); + } + //------------------------------------------------------------------- Other tests @Test @@ -711,6 +745,7 @@ public class SelectTestCase { assertEquals("all(group(time.dayofmonth(a)) each(output(count())))", query.getSelect().getGrouping().get(0).toString()); Query clone = query.clone(); + assertEquals(clone.getSelect().getGroupingExpressionString(), query.getSelect().getGroupingExpressionString()); assertNotSame(query.getSelect(), clone.getSelect()); assertNotSame(query.getSelect().getGrouping(), clone.getSelect().getGrouping()); assertNotSame(query.getSelect().getGrouping().get(0), clone.getSelect().getGrouping().get(0)); @@ -719,8 +754,15 @@ public class SelectTestCase { assertEquals(query.getSelect().getGroupingString(), clone.getSelect().getGroupingString()); assertEquals(query.getSelect().getGrouping().get(0).toString(), clone.getSelect().getGrouping().get(0).toString()); assertEquals(query.getSelect().getGrouping().get(1).toString(), clone.getSelect().getGrouping().get(1).toString()); + } + @Test + public void testCloneWithGroupingExpressionString() { + Query query = new Query(); + query.getSelect().setGroupingExpressionString("all(group(foo) each(output(count())))"); + Query clone = query.clone(); + assertEquals(clone.getSelect().getGroupingExpressionString(), query.getSelect().getGroupingExpressionString()); } //------------------------------------------------------------------- Assert methods @@ -763,7 +805,6 @@ public class SelectTestCase { } private List<VespaGroupingStep> parseGrouping(String grouping) { - return parser.getGroupingSteps(grouping); } |