diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo')
11 files changed, 224 insertions, 95 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java new file mode 100644 index 00000000000..27045629780 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/BoolItem.java @@ -0,0 +1,102 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import java.nio.ByteBuffer; + +/** + * A true/false term suitable for searching bool indexes. + */ +public class BoolItem extends TermItem { + + private boolean value; + + public BoolItem(boolean value) { + this(value, ""); + } + + public BoolItem(boolean value, String indexName) { this(value, indexName, false); } + + public BoolItem(boolean value, String indexName, boolean isFromQuery) { + super(indexName, isFromQuery, new Substring(String.valueOf(value))); + this.value = value; + } + + /** Returns ItemType.WORD as we do not want a string binding from the parsed query to index types */ + @Override + public ItemType getItemType() { return ItemType.WORD; } + + @Override + public String getName() { return "BOOL"; } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); // takes care of index bytes + putString(stringValue(), buffer); + } + + public boolean value() { return value; } + + /** Returns "true" or "false" */ + @Override + public String stringValue() { return value ? "true" : "false"; } + + public void setValue(boolean value) { + this.value = value; + } + + /** + * Sets the value from a string + * + * @param stringValue "true" or "false" + * @throws IllegalArgumentException if the given value is not equal to "true" nor "false" (ignoring case) + */ + @Override + public void setValue(String stringValue) { + this.value = toBoolean(stringValue); + } + + private boolean toBoolean(String stringValue) { + switch (stringValue.toLowerCase()) { + case "true" : return true; + case "false" : return false; + default: throw new IllegalArgumentException("Expected 'true' or 'false', got '" + stringValue + "'"); + } + } + + /** Returns the same as stringValue */ + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public boolean isStemmed() { return false; } + + /** Word items uses a empty heading instead of "WORD " */ + @Override + protected void appendHeadingString(StringBuilder buffer) {} + + @Override + public int hashCode() { + return Boolean.hashCode(value); + } + + @Override + public boolean equals(Object object) { + if ( ! super.equals(object)) return false; + + BoolItem other = (BoolItem) object; // Ensured by superclass + return this.value == other.value; + } + + @Override + public int getNumWords() { return 1; } + + @Override + public String getIndexedString() { return stringValue(); } + + /** Returns true if this consists of regular word characters. Returns false if this represents a "special token" */ + @Override + public boolean isWords() { return false; } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java index 2e24937856c..714e8f9cb5e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java @@ -56,17 +56,6 @@ public class IntItem extends TermItem { expression = toExpression(from, to, 0); } - /** Returns the simplest expression matching this */ - private String toExpression(Limit from, Limit to, int hitLimit) { - if (from.equals(to) && hitLimit == 0) return from.number().toString(); - - String expression = from.toRangeStart() + ";" + to.toRangeEnd(); - if (hitLimit == 0) return expression; - - // Insert ;hitLimit at the end inside the brackets - return expression.substring(0, expression.length()-1) + ";" + hitLimit + expression.substring(expression.length()-1); - } - public IntItem(String expression) { this(expression, ""); } @@ -91,8 +80,19 @@ public class IntItem extends TermItem { this.expression = toExpression(from, to, hitLimit); } + /** Returns the simplest expression matching this */ + private String toExpression(Limit from, Limit to, int hitLimit) { + if (from.equals(to) && hitLimit == 0) return from.number().toString(); + + String expression = from.toRangeStart() + ";" + to.toRangeEnd(); + if (hitLimit == 0) return expression; + + // Insert ;hitLimit at the end inside the brackets + return expression.substring(0, expression.length()-1) + ";" + hitLimit + expression.substring(expression.length()-1); + } + /** Sets limit and flip them if "from" is greater than "to" */ - private final void setLimits(Limit from, Limit to) { + private void setLimits(Limit from, Limit to) { if (from.number().doubleValue() > to.number().doubleValue()) { this.from = to; this.to = from; @@ -188,8 +188,7 @@ public class IntItem extends TermItem { * <code>from</code> are returned, and if this number is negative the hits * closest to <code>to</code> are returned. * - * @param hitLimit - * number of hits to match for this operator + * @param hitLimit number of hits to match for this operator */ public final void setHitLimit(int hitLimit) { this.hitLimit = hitLimit; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java index 2535619f193..f230c76c954 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleIndexedItem.java @@ -54,17 +54,15 @@ public abstract class SimpleIndexedItem extends SimpleTaggableItem implements In } } + @Override public boolean equals(Object object) { - if (!super.equals(object)) { - return false; - } + if (!super.equals(object)) return false; + IndexedItem other = (IndexedItem) object; // Ensured by superclass - if (!this.index.equals(other.getIndexName())) { - return false; - } - return true; + return this.index.equals(other.getIndexName()); } + @Override public int hashCode() { return super.hashCode() + 113 * index.hashCode(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Substring.java b/container-search/src/main/java/com/yahoo/prelude/query/Substring.java index 10ea6f85b5f..ab3e28fc259 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Substring.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Substring.java @@ -19,14 +19,21 @@ public class Substring { /** The string this is a substring of */ public final String string; - public Substring(int start, int end,String string) { + /** Creates a substring which is identical to the string containing it */ + public Substring(String string) { + this.start = 0; + this.end = string.length(); + this.string=string; + } + + public Substring(int start, int end, String string) { this.start = start; this.end = end; this.string=string; } public String getValue() { - return string.substring(start,end); + return string.substring(start, end); } /** Returns the entire string this is a substring of. The start and end offsets are into this string. */ diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java index 764fb2b1118..8fdf147ce0b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermItem.java @@ -8,8 +8,7 @@ import java.nio.ByteBuffer; /** - * Superclass of "leaf" conditions containing a single entity which is either matched in - * a field or not. + * Superclass of "leaf" conditions containing a single entity which is either matched in a field or not. * * @author bratseth * @author havardpe @@ -116,8 +115,7 @@ public abstract class TermItem extends SimpleIndexedItem implements BlockItem { * but for historical reasons that is not the case. This method has nothing * to do with Unicode normalization. * - * @param normalizable - * set to true if accent removal can/should be performed + * @param normalizable set to true if accent removal can/should be performed */ public void setNormalizable(boolean normalizable) { this.normalizable = normalizable; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java index 0eb37373e1f..a0467703b13 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java @@ -6,7 +6,7 @@ package com.yahoo.prelude.query; * A term type enumeration * * @author bratseth - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class TermType { @@ -62,10 +62,9 @@ public class TermType { */ public Item createItemClass() { try { - return instanceClass.newInstance(); + return instanceClass.getDeclaredConstructor().newInstance(); } catch (Exception e) { - throw new RuntimeException("Could not create an instance for item " - + this, e); + throw new RuntimeException("Could not create an instance for item " + this, e); } } @@ -73,16 +72,15 @@ public class TermType { return sign; } + @Override public boolean equals(Object o) { - if (!(o instanceof TermType)) { - return false; - } + if ( ! (o instanceof TermType)) return false; TermType other = (TermType) o; - return name.equals(other.name); } + @Override public int hashCode() { return name.hashCode(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java index fb0a2418d37..72ba012ab07 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ToolBox.java @@ -6,12 +6,13 @@ import com.google.common.annotations.Beta; /** * Query tree helper methods and factories. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ @Beta public final class ToolBox { public static abstract class QueryVisitor { + /** * Called for each item in the query tree given to * {@link ToolBox#visit(QueryVisitor, Item)}. Return true to visit the @@ -29,6 +30,7 @@ public final class ToolBox { * visit() if there are no sub-items or visit() returned false. */ public abstract void onExit(); + } public static void visit(QueryVisitor visitor, Item item) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java index 39573e4d71f..be56c95a6af 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WordItem.java @@ -33,14 +33,6 @@ public class WordItem extends TermItem { private boolean lowercased = false; - public ItemType getItemType() { - return ItemType.WORD; - } - - public String getName() { - return "WORD"; - } - public WordItem(String word) { this(word, ""); } @@ -70,12 +62,21 @@ public class WordItem extends TermItem { setWord(word); } + public ItemType getItemType() { + return ItemType.WORD; + } + + public String getName() { + return "WORD"; + } + public void setWord(String word) { Validator.ensureNotNull("The word of a word item", word); Validator.ensureNonEmpty("The word of a word item", word); this.word = word; } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); // takes care of index bytes putString(getEncodedWord(), buffer); @@ -112,6 +113,7 @@ public class WordItem extends TermItem { return word; } + @Override public boolean isStemmed() { return stemmed; } public void setStemmed(boolean stemmed) { this.stemmed = stemmed; } @@ -168,6 +170,7 @@ public class WordItem extends TermItem { } /** Returns true if this consists of regular word characters. Returns false if this represents a "special token" */ + @Override public boolean isWords() { return words; } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 1c2f409d792..36a04bb86e8 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -776,11 +776,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * different from default, while linguistics metadata are not added. * * @return a valid YQL+ query string or a human readable error message - * @see Query#yqlRepresentation(Tuple2, boolean) + * @see Query#yqlRepresentation(boolean) */ public String yqlRepresentation() { try { - return yqlRepresentation(null, true); + return yqlRepresentation(true); } catch (NullItemException e) { return "Query currently a placeholder, NullItem encountered."; } catch (RuntimeException e) { @@ -799,23 +799,22 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } } + /** @deprecated remove the ignored segmenterVersion argument from invocations */ + @Deprecated // TODO: Remove on Vespa 8 + public String yqlRepresentation(Tuple2<String, Version> segmenterVersion, boolean includeHitsAndOffset) { + return yqlRepresentation(includeHitsAndOffset); + } + /** * Serialize this query as YQL+. This will create a string representation * which should always be legal YQL+. If a problem occurs, a * RuntimeException is thrown. * - * @param segmenterVersion - * linguistics metadata used in federation, set to null if the - * annotation is not necessary - * @param includeHitsAndOffset - * whether to include hits and offset parameters converted to a - * offset/limit slice + * @param includeHitsAndOffset whether to include hits and offset parameters converted to a offset/limit slice * @return a valid YQL+ query string * @throws RuntimeException if there is a problem serializing the query tree */ - public String yqlRepresentation(@Nullable Tuple2<String, Version> segmenterVersion, boolean includeHitsAndOffset) { - String q = VespaSerializer.serialize(this); - + public String yqlRepresentation(boolean includeHitsAndOffset) { Set<String> sources = getModel().getSources(); Set<String> fields = getPresentation().getSummaryFields(); StringBuilder yql = new StringBuilder("select "); @@ -834,31 +833,20 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { commaSeparated(yql, sources); } yql.append(" where "); - if (segmenterVersion != null) { - yql.append("[{\"segmenter\": {\"version\": \"") - .append(segmenterVersion.second.toString()) - .append("\", \"backend\": \"") - .append(segmenterVersion.first).append("\"}}]("); - } - yql.append(q); - if (segmenterVersion != null) { - yql.append(')'); - } + yql.append(VespaSerializer.serialize(this)); if (getRanking().getSorting() != null && getRanking().getSorting().fieldOrders().size() > 0) { serializeSorting(yql); } if (includeHitsAndOffset) { if (getOffset() != 0) { - yql.append(" limit ") - .append(Integer.toString(getHits() + getOffset())) - .append(" offset ") - .append(Integer.toString(getOffset())); + yql.append(" limit ").append(getHits() + getOffset()) + .append(" offset ").append(getOffset()); } else if (getHits() != 10) { - yql.append(" limit ").append(Integer.toString(getHits())); + yql.append(" limit ").append(getHits()); } } if (getTimeout() != defaultTimeout) { - yql.append(" timeout ").append(Long.toString(getTimeout())); + yql.append(" timeout ").append(getTimeout()); } yql.append(';'); return yql.toString(); @@ -871,29 +859,41 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (yql.length() > initLen) { yql.append(", "); } - final Class<? extends AttributeSorter> sorterType = f.getSorter() - .getClass(); + Class<? extends AttributeSorter> sorterType = f.getSorter().getClass(); if (sorterType == Sorting.RawSorter.class) { - yql.append("[{\"").append(YqlParser.SORTING_FUNCTION) - .append("\": \"").append(Sorting.RAW).append("\"}]"); + yql.append("[{\"") + .append(YqlParser.SORTING_FUNCTION) + .append("\": \"") + .append(Sorting.RAW) + .append("\"}]"); } else if (sorterType == Sorting.LowerCaseSorter.class) { - yql.append("[{\"").append(YqlParser.SORTING_FUNCTION) - .append("\": \"").append(Sorting.LOWERCASE) - .append("\"}]"); + yql.append("[{\"") + .append(YqlParser.SORTING_FUNCTION) + .append("\": \"") + .append(Sorting.LOWERCASE) + .append("\"}]"); } else if (sorterType == Sorting.UcaSorter.class) { Sorting.UcaSorter uca = (Sorting.UcaSorter) f.getSorter(); String ucaLocale = uca.getLocale(); Sorting.UcaSorter.Strength ucaStrength = uca.getStrength(); - yql.append("[{\"").append(YqlParser.SORTING_FUNCTION) - .append("\": \"").append(Sorting.UCA).append("\""); + yql.append("[{\"") + .append(YqlParser.SORTING_FUNCTION) + .append("\": \"") + .append(Sorting.UCA) + .append("\""); if (ucaLocale != null) { - yql.append(", \"").append(YqlParser.SORTING_LOCALE) - .append("\": \"").append(ucaLocale).append('"'); + yql.append(", \"") + .append(YqlParser.SORTING_LOCALE) + .append("\": \"") + .append(ucaLocale) + .append('"'); } if (ucaStrength != Sorting.UcaSorter.Strength.UNDEFINED) { - yql.append(", \"").append(YqlParser.SORTING_STRENGTH) - .append("\": \"").append(ucaStrength.name()) - .append('"'); + yql.append(", \"") + .append(YqlParser.SORTING_STRENGTH) + .append("\": \"") + .append(ucaStrength.name()) + .append('"'); } yql.append("}]"); } @@ -906,8 +906,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns the context of this query, possibly creating it if missing. Returns the context, or null */ public QueryContext getContext(boolean create) { - if (context==null && create) - context=new QueryContext(getTraceLevel(),this); + if (context == null && create) + context = new QueryContext(getTraceLevel(),this); return context; } @@ -920,7 +920,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns whether the given query is equal to this */ @Override public boolean equals(Object other) { - if (this==other) return true; + if (this == other) return true; if ( ! (other instanceof Query)) return false; Query q = (Query) other; diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index ac1c2ee4a6c..c27c047c899 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -61,6 +61,7 @@ import java.util.Map.Entry; import com.google.common.collect.ImmutableMap; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.DotProductItem; import com.yahoo.prelude.query.EquivItem; import com.yahoo.prelude.query.ExactStringItem; @@ -102,9 +103,10 @@ import edu.umd.cs.findbugs.annotations.NonNull; /** * Serialize Vespa query trees to YQL+ strings. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class VespaSerializer { + // TODO refactor, too much copy/paste private static class AndSegmentSerializer extends Serializer { @@ -270,6 +272,7 @@ public class VespaSerializer { } private static class NotSerializer extends Serializer { + @Override void onExit(StringBuilder destination, Item item) { destination.append(')'); @@ -298,10 +301,8 @@ public class VespaSerializer { @Override boolean serialize(StringBuilder destination, Item item) { - throw new NullItemException( - "NullItem encountered in query tree." - + " This is usually a symptom of an invalid query or an error" - + " in a query transformer."); + throw new NullItemException("NullItem encountered in query tree. This is usually a symptom of an invalid " + + "query or an error in a query transformer."); } } @@ -402,6 +403,21 @@ public class VespaSerializer { } } + private static class BoolSerializer extends Serializer { + + @Override + void onExit(StringBuilder destination, Item item) { } + + @Override + boolean serialize(StringBuilder destination, Item item) { + BoolItem intItem = (BoolItem) item; + destination.append(normalizeIndexName(intItem.getIndexName())).append(" = "); + destination.append(((BoolItem) item).stringValue()); + return false; + } + + } + private static class RegExpSerializer extends Serializer { @Override @@ -419,6 +435,7 @@ public class VespaSerializer { } private static class ONearSerializer extends Serializer { + @Override void onExit(StringBuilder destination, Item item) { } @@ -452,6 +469,7 @@ public class VespaSerializer { } private static class OrSerializer extends Serializer { + @Override void onExit(StringBuilder destination, Item item) { destination.append(')'); @@ -1071,6 +1089,7 @@ public class VespaSerializer { dispatchBuilder.put(EquivItem.class, new EquivSerializer()); dispatchBuilder.put(ExactStringItem.class, new WordSerializer()); dispatchBuilder.put(IntItem.class, new NumberSerializer()); + dispatchBuilder.put(BoolItem.class, new BoolSerializer()); dispatchBuilder.put(MarkerWordItem.class, new WordSerializer()); // gotcha dispatchBuilder.put(NearItem.class, new NearSerializer()); dispatchBuilder.put(NotItem.class, new NotSerializer()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 5928dd23b28..af095fefc1c 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -26,6 +26,7 @@ import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BoolItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.DotProductItem; import com.yahoo.prelude.query.EquivItem; @@ -947,8 +948,10 @@ public class YqlParser implements Parser { String value = fetchConditionWord(ast); TermItem item; - if (value.equals("true") || value.equals("false")) - item = new WordItem(value, fetchConditionIndex(ast)); + if (value.equals("true")) { + item = new BoolItem(true, fetchConditionIndex(ast)); + } else if (value.equals("false")) + item = new BoolItem(false, fetchConditionIndex(ast)); else item = new IntItem(value, fetchConditionIndex(ast)); |