From 842478eca170b3cc04408d80acafb252d7c19c26 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 3 Sep 2020 14:43:55 +0200 Subject: Continuation references --- .../main/antlr4/com/yahoo/search/yql/yqlplus.g4 | 1 + .../com/yahoo/search/grouping/Continuation.java | 6 ++--- .../search/grouping/vespa/IntegerDecoder.java | 5 +++- .../com/yahoo/search/yql/MinimalQueryInserter.java | 1 + .../java/com/yahoo/search/yql/ProgramParser.java | 24 ++++++++++-------- .../main/java/com/yahoo/search/yql/YqlParser.java | 29 +++++++++++++++++++--- .../com/yahoo/search/yql/UserInputTestCase.java | 27 ++++++++++++++++---- 7 files changed, 69 insertions(+), 24 deletions(-) diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index dfa9f2e93e1..d680ea1b91e 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -646,6 +646,7 @@ constantExpression : scalar_literal | constantMapExpression | constantArray + | parameter ; constantArray diff --git a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java index d7ee3fedfc9..b74101fb83d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java @@ -8,7 +8,7 @@ import com.yahoo.search.grouping.vespa.ContinuationDecoder; * subsequently be sent back along with the original request to navigate across a large result set. It is an opaque * data object that is not intended to be human readable.

* - *

To render a Cookie within a result set, you simply need to call {@link #toString()}.

+ *

To render a continuation within a result set, you simply need to call {@link #toString()}.

* * @author Simon Thoresen Hult */ @@ -18,8 +18,8 @@ public abstract class Continuation { public static final String PREV_PAGE = "prev"; public static final String THIS_PAGE = "this"; - public static Continuation fromString(String str) { - return ContinuationDecoder.decode(str); + public static Continuation fromString(String string) { + return ContinuationDecoder.decode(string); } /** Returns a deep copy of this */ diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java index e1a222d6bc0..15781060d7f 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import java.util.Arrays; + /** * @author Simon Thoresen Hult */ @@ -33,7 +35,8 @@ class IntegerDecoder { if (c >= CHAR_MIN && c <= CHAR_MAX) { return (0xF & (c - CHAR_MIN)); } else { - throw new NumberFormatException(String.valueOf(c)); + throw new NumberFormatException("Expected a char in " + Arrays.toString(IntegerEncoder.CHARS) + + " but was '" + c + "'"); } } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index 209340e0a99..29490c9ae58 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -93,6 +93,7 @@ public class MinimalQueryInserter extends Searcher { Parsable parsable = Parsable.fromQueryModel(query.getModel()).setQuery(query.properties().getString(YQL)); newTree = parser.parse(parsable); } catch (RuntimeException e) { + e.printStackTrace(); return new Result(query, ErrorMessage.createInvalidQueryParameter("Could not instantiate query from YQL", e)); } if (parser.getOffset() != null) { diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java index 234558f5fd2..bcd5822a6f6 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java @@ -987,7 +987,7 @@ final class ProgramParser { for (Field_defContext rulenode : fieldDefs) { // FIELD // expression alias_def? - OperatorNode expr = convertExpr((ExpressionContext)rulenode.getChild(0), scope); + OperatorNode expr = convertExpr(rulenode.getChild(0), scope); String aliasName = null; if (rulenode.getChildCount() > 1) { @@ -1018,9 +1018,8 @@ final class ProgramParser { .constantMapExpression(), scope); OperatorNode expr = OperatorNode.create(toLocation(scope, secondChild), ExpressionOperator.VESPA_GROUPING, secondChild.getText()); - List names = (List) annotation.getArgument(0); - List> annotates = (List>) annotation - .getArgument(1); + List names = annotation.getArgument(0); + List> annotates = annotation.getArgument(1); for (int i = 0; i < names.size(); ++i) { expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); } @@ -1147,8 +1146,8 @@ final class ProgramParser { AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation(); OperatorNode annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope); OperatorNode expr = convertExpr(parseTree.getChild(1), scope); - List names = (List) annotation.getArgument(0); - List> annotates = (List>) annotation.getArgument(1); + List names = annotation.getArgument(0); + List> annotates = annotation.getArgument(1); for (int i = 0; i < names.size(); ++i) { expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i))); } @@ -1375,9 +1374,9 @@ final class ProgramParser { case yqlplusParser.STRING: return StringUnescaper.unquote(text); case yqlplusParser.TRUE: - return Boolean.valueOf(true); + return true; case yqlplusParser.FALSE: - return Boolean.valueOf(false); + return false; case yqlplusParser.LONG_INT: return Long.parseLong(text.substring(0, text.length()-1)); default: @@ -1391,21 +1390,24 @@ final class ProgramParser { return node.getArgument(0); case MAP: { ImmutableMap.Builder map = ImmutableMap.builder(); - List names = (List) node.getArgument(0); - List> exprs = (List>) node.getArgument(1); + List names = node.getArgument(0); + List> exprs = node.getArgument(1); for (int i = 0; i < names.size(); ++i) { map.put(names.get(i), readConstantExpression(exprs.get(i))); } return map.build(); } case ARRAY: { - List> exprs = (List>) node.getArgument(0); + List> exprs = node.getArgument(0); ImmutableList.Builder lst = ImmutableList.builder(); for (OperatorNode expr : exprs) { lst.add(readConstantExpression(expr)); } return lst.build(); } + case VARREF: { + return node; // must be dereferenced in YqlParser when we have userQuery + } default: throw new ProgramCompileException(node.getLocation(), "Internal error: Unknown constant expression type: " + node.getOperator()); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 3875018609e..3880ad594e3 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -810,10 +810,11 @@ public class YqlParser implements Parser { OperatorNode groupingAst = ast.>> getArgument(2).get(0); GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst. getArgument(0)); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); - List continuations = getAnnotation(groupingAst, "continuations", List.class, + List continuations = getAnnotation(groupingAst, "continuations", List.class, Collections.emptyList(), "grouping continuations"); - for (String continuation : continuations) { - groupingStep.continuations().add(Continuation.fromString(continuation)); + + for (Object continuation : continuations) { + groupingStep.continuations().add(Continuation.fromString(dereference(continuation))); } groupingSteps.add(groupingStep); ast = ast.getArgument(0); @@ -822,6 +823,18 @@ public class YqlParser implements Parser { return ast; } + private String dereference(Object constantOrVarref) { + if (constantOrVarref instanceof OperatorNode) { + OperatorNode varref = (OperatorNode)constantOrVarref; + Preconditions.checkState(userQuery != null, + "properties must be available when trying to fetch user input"); + return userQuery.properties().getString(varref.getArgument(0, String.class)); + } + else { + return constantOrVarref.toString(); + } + } + private OperatorNode fetchSorting(OperatorNode ast) { if (ast.getOperator() != SequenceOperator.SORT) return ast; @@ -1750,7 +1763,15 @@ public class YqlParser implements Parser { Object value = ast.getAnnotation(key); for (Iterator> i = annotationStack.iterator(); value == null && considerParents && i.hasNext();) { - value = i.next().getAnnotation(key); + OperatorNode node = i.next(); + if (node.getOperator() == ExpressionOperator.VARREF) { + Preconditions.checkState(userQuery != null, + "properties must be available when trying to fetch user input"); + value = userQuery.properties().getString(ast.getArgument(0, String.class)); + } + else { + value = node.getAnnotation(key); + } } if (value == null) return defaultValue; Preconditions.checkArgument(expectedClass.isInstance(value), diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index 75517a25909..6433270d691 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -15,6 +15,8 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; +import java.util.Arrays; + import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; /** @@ -204,13 +206,30 @@ public class UserInputTestCase { assertEquals("select * from sources * where year > 1980;", query.yqlRepresentation()); } + @Test + public void testReferenceInContinuation() { + URIBuilder builder = searchUri(); + builder.setParameter("continuation", "BCBCBCBEBG"); + builder.setParameter("yql", + "select * from sources * where myfield contains 'token'" + + "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())));"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where myfield contains \"token\" | [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())));", query.yqlRepresentation()); + } + private Query searchAndAssertNoErrors(URIBuilder builder) { Query query = new Query(builder.toString()); Result r = execution.search(query); - assertNull(r.hits().getError()); + assertNull(stackTraceIfAny(r), r.hits().getError()); return query; } + private String stackTraceIfAny(Result r) { + if (r.hits().getError() == null) return ""; + if (r.hits().getError().getCause() == null) return ""; + return Arrays.toString(r.hits().getError().getCause().getStackTrace()); + } + private URIBuilder searchUri() { URIBuilder builder = new URIBuilder(); builder.setPath("search/"); @@ -220,8 +239,7 @@ public class UserInputTestCase { @Test public void testEmptyUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", - "select * from sources * where userInput(\"\");"); + builder.setParameter("yql", "select * from sources * where userInput(\"\");"); assertQueryFails(builder); } @@ -229,8 +247,7 @@ public class UserInputTestCase { public void testEmptyUserInputFromQueryProperty() { URIBuilder builder = searchUri(); builder.setParameter("foo", ""); - builder.setParameter("yql", - "select * from sources * where userInput(@foo);"); + builder.setParameter("yql", "select * from sources * where userInput(@foo);"); assertQueryFails(builder); } -- cgit v1.2.3