diff options
author | Jon Bratseth <bratseth@oath.com> | 2020-09-03 23:46:50 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-09-03 23:46:50 +0200 |
commit | 9a735030cd58e7f7ce7c2cd9bcaae121089e6ee7 (patch) | |
tree | de78463f39e6fc888dc785f65da2f170b173500a /container-search | |
parent | 2a238d333516e658eb96f02af2ae27d91ac4bd1a (diff) | |
parent | 79224ea5f1d35892cb8e5380693f4f350924bdfe (diff) |
Merge pull request #14273 from vespa-engine/bratseth/continuation-references
Continuation references
Diffstat (limited to 'container-search')
6 files changed, 84 insertions, 28 deletions
diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index dfa9f2e93e1..d680ea1b91e 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -646,6 +646,7 @@ constantExpression : scalar_literal | constantMapExpression | constantArray + | parameter ; constantArray diff --git a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java index d7ee3fedfc9..b74101fb83d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java @@ -8,7 +8,7 @@ import com.yahoo.search.grouping.vespa.ContinuationDecoder; * subsequently be sent back along with the original request to navigate across a large result set. It is an opaque * data object that is not intended to be human readable.</p> * - * <p>To render a Cookie within a result set, you simply need to call {@link #toString()}.</p> + * <p>To render a continuation within a result set, you simply need to call {@link #toString()}.</p> * * @author Simon Thoresen Hult */ @@ -18,8 +18,8 @@ public abstract class Continuation { public static final String PREV_PAGE = "prev"; public static final String THIS_PAGE = "this"; - public static Continuation fromString(String str) { - return ContinuationDecoder.decode(str); + public static Continuation fromString(String string) { + return ContinuationDecoder.decode(string); } /** Returns a deep copy of this */ diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java index e1a222d6bc0..15781060d7f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import java.util.Arrays; + /** * @author Simon Thoresen Hult */ @@ -33,7 +35,8 @@ class IntegerDecoder { if (c >= CHAR_MIN && c <= CHAR_MAX) { return (0xF & (c - CHAR_MIN)); } else { - throw new NumberFormatException(String.valueOf(c)); + throw new NumberFormatException("Expected a char in " + Arrays.toString(IntegerEncoder.CHARS) + + " but was '" + c + "'"); } } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index 234558f5fd2..bcd5822a6f6 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -987,7 +987,7 @@ final class ProgramParser { for (Field_defContext rulenode : fieldDefs) { // FIELD // expression alias_def? - OperatorNode<ExpressionOperator> expr = convertExpr((ExpressionContext)rulenode.getChild(0), scope); + OperatorNode<ExpressionOperator> expr = convertExpr(rulenode.getChild(0), scope); String aliasName = null; if (rulenode.getChildCount() > 1) { @@ -1018,9 +1018,8 @@ final class ProgramParser { .constantMapExpression(), scope); OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild), ExpressionOperator.VESPA_GROUPING, secondChild.getText()); - List<String> names = (List<String>) annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation - .getArgument(1); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); for (int i = 0; i < names.size(); ++i) { expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); } @@ -1147,8 +1146,8 @@ final class ProgramParser { AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope); - List<String> names = (List<String>) annotation.getArgument(0); - List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation.getArgument(1); + List<String> names = annotation.getArgument(0); + List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1); for (int i = 0; i < names.size(); ++i) { expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); } @@ -1375,9 +1374,9 @@ final class ProgramParser { case yqlplusParser.STRING: return StringUnescaper.unquote(text); case yqlplusParser.TRUE: - return Boolean.valueOf(true); + return true; case yqlplusParser.FALSE: - return Boolean.valueOf(false); + return false; case yqlplusParser.LONG_INT: return Long.parseLong(text.substring(0, text.length()-1)); default: @@ -1391,21 +1390,24 @@ final class ProgramParser { return node.getArgument(0); case MAP: { ImmutableMap.Builder<String, Object> map = ImmutableMap.builder(); - List<String> names = (List<String>) node.getArgument(0); - List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(1); + List<String> names = node.getArgument(0); + List<OperatorNode<ExpressionOperator>> exprs = node.getArgument(1); for (int i = 0; i < names.size(); ++i) { map.put(names.get(i), readConstantExpression(exprs.get(i))); } return map.build(); } case ARRAY: { - List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(0); + List<OperatorNode<ExpressionOperator>> exprs = node.getArgument(0); ImmutableList.Builder<Object> lst = ImmutableList.builder(); for (OperatorNode<ExpressionOperator> expr : exprs) { lst.add(readConstantExpression(expr)); } return lst.build(); } + case VARREF: { + return node; // must be dereferenced in YqlParser when we have userQuery + } default: throw new ProgramCompileException(node.getLocation(), "Internal error: Unknown constant expression type: " + node.getOperator()); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 3875018609e..739aae0e277 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -810,10 +810,11 @@ public class YqlParser implements Parser { OperatorNode<ExpressionOperator> groupingAst = ast.<List<OperatorNode<ExpressionOperator>>> getArgument(2).get(0); GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.<String> getArgument(0)); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); - List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, + List<Object> continuations = getAnnotation(groupingAst, "continuations", List.class, Collections.emptyList(), "grouping continuations"); - for (String continuation : continuations) { - groupingStep.continuations().add(Continuation.fromString(continuation)); + + for (Object continuation : continuations) { + groupingStep.continuations().add(Continuation.fromString(dereference(continuation))); } groupingSteps.add(groupingStep); ast = ast.getArgument(0); @@ -822,6 +823,18 @@ public class YqlParser implements Parser { return ast; } + private String dereference(Object constantOrVarref) { + if (constantOrVarref instanceof OperatorNode) { + OperatorNode<?> varref = (OperatorNode<?>)constantOrVarref; + Preconditions.checkState(userQuery != null, + "properties must be available when trying to fetch user input"); + return userQuery.properties().getString(varref.getArgument(0, String.class)); + } + else { + return constantOrVarref.toString(); + } + } + private OperatorNode<?> fetchSorting(OperatorNode<?> ast) { if (ast.getOperator() != SequenceOperator.SORT) return ast; @@ -1270,7 +1283,7 @@ public class YqlParser implements Parser { equiv.setIndexName(field); for (OperatorNode<ExpressionOperator> arg : args) { switch (arg.getOperator()) { - case LITERAL: + case LITERAL: case VARREF: equiv.addItem(instantiateWordItem(field, arg, equiv.getClass())); break; case CALL: @@ -1279,7 +1292,7 @@ public class YqlParser implements Parser { break; default: throw newUnexpectedArgumentException(arg.getOperator(), - ExpressionOperator.CALL, ExpressionOperator.LITERAL); + ExpressionOperator.CALL, ExpressionOperator.LITERAL, ExpressionOperator.VARREF); } } return leafStyleSettings(ast, equiv); @@ -1340,7 +1353,8 @@ public class YqlParser implements Parser { } private Item instantiateWordItem(String field, - OperatorNode<ExpressionOperator> ast, Class<?> parent, + OperatorNode<ExpressionOperator> ast, + Class<?> parent, SegmentWhen segmentPolicy) { String wordData = getStringContents(ast); return instantiateWordItem(field, wordData, ast, parent, segmentPolicy, null, decideParsingLanguage(ast, wordData)); @@ -1356,7 +1370,8 @@ public class YqlParser implements Parser { // which always expands first, but not using getIndex, which performs checks that doesn't always work private Item instantiateWordItem(String field, String rawWord, - OperatorNode<ExpressionOperator> ast, Class<?> parent, + OperatorNode<ExpressionOperator> ast, + Class<?> parent, SegmentWhen segmentPolicy, Boolean exactMatch, Language language) { @@ -1750,7 +1765,15 @@ public class YqlParser implements Parser { Object value = ast.getAnnotation(key); for (Iterator<OperatorNode<?>> i = annotationStack.iterator(); value == null && considerParents && i.hasNext();) { - value = i.next().getAnnotation(key); + OperatorNode node = i.next(); + if (node.getOperator() == ExpressionOperator.VARREF) { + Preconditions.checkState(userQuery != null, + "properties must be available when trying to fetch user input"); + value = userQuery.properties().getString(ast.getArgument(0, String.class)); + } + else { + value = node.getAnnotation(key); + } } if (value == null) return defaultValue; Preconditions.checkArgument(expectedClass.isInstance(value), diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index 75517a25909..06160e1c6d5 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -15,6 +15,8 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; +import java.util.Arrays; + import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; /** @@ -204,13 +206,40 @@ public class UserInputTestCase { assertEquals("select * from sources * where year > 1980;", query.yqlRepresentation()); } + @Test + public void testReferenceInContinuation() { + URIBuilder builder = searchUri(); + builder.setParameter("continuation", "BCBCBCBEBG"); + builder.setParameter("yql", + "select * from sources * where myfield contains 'token'" + + "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())));"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where myfield contains \"token\" | [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())));", query.yqlRepresentation()); + } + + @Test + public void testReferenceInEquiv() { + URIBuilder builder = searchUri(); + builder.setParameter("term", "A"); + builder.setParameter("yql", + "select foo from bar where fieldName contains equiv(@term,'B');"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select foo from bar where fieldName contains equiv(\"A\", \"B\");", query.yqlRepresentation()); + } + private Query searchAndAssertNoErrors(URIBuilder builder) { Query query = new Query(builder.toString()); Result r = execution.search(query); - assertNull(r.hits().getError()); + assertNull(stackTraceIfAny(r), r.hits().getError()); return query; } + private String stackTraceIfAny(Result r) { + if (r.hits().getError() == null) return ""; + if (r.hits().getError().getCause() == null) return ""; + return Arrays.toString(r.hits().getError().getCause().getStackTrace()); + } + private URIBuilder searchUri() { URIBuilder builder = new URIBuilder(); builder.setPath("search/"); @@ -220,8 +249,7 @@ public class UserInputTestCase { @Test public void testEmptyUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", - "select * from sources * where userInput(\"\");"); + builder.setParameter("yql", "select * from sources * where userInput(\"\");"); assertQueryFails(builder); } @@ -229,8 +257,7 @@ public class UserInputTestCase { public void testEmptyUserInputFromQueryProperty() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", - "select * from sources * where userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where userInput(@foo);"); assertQueryFails(builder); } |