Merge pull request #14273 from vespa-engine/bratseth/continuation-references

Continuation references

6 files changed, 84 insertions, 28 deletions

diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4
index dfa9f2e93e1..d680ea1b91e 100644
--- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4
+++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4
@@ -646,6 +646,7 @@ constantExpression
     : scalar_literal
     | constantMapExpression
     | constantArray
+    | parameter
     ;
 
 constantArray
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java
index d7ee3fedfc9..b74101fb83d 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/Continuation.java
@@ -8,7 +8,7 @@ import com.yahoo.search.grouping.vespa.ContinuationDecoder;
  * subsequently be sent back along with the original request to navigate across a large result set. It is an opaque
  * data object that is not intended to be human readable.</p>
  *
- * <p>To render a Cookie within a result set, you simply need to call {@link #toString()}.</p>
+ * <p>To render a continuation within a result set, you simply need to call {@link #toString()}.</p>
  *
  * @author Simon Thoresen Hult
  */
@@ -18,8 +18,8 @@ public abstract class Continuation {
     public static final String PREV_PAGE = "prev";
     public static final String THIS_PAGE = "this";
 
-    public static Continuation fromString(String str) {
-        return ContinuationDecoder.decode(str);
+    public static Continuation fromString(String string) {
+        return ContinuationDecoder.decode(string);
     }
 
     /** Returns a deep copy of this */
diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java
index e1a222d6bc0..15781060d7f 100644
--- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java
+++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/IntegerDecoder.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.grouping.vespa;
 
+import java.util.Arrays;
+
 /**
  * @author Simon Thoresen Hult
  */
@@ -33,7 +35,8 @@ class IntegerDecoder {
         if (c >= CHAR_MIN && c <= CHAR_MAX) {
             return (0xF & (c - CHAR_MIN));
         } else {
-            throw new NumberFormatException(String.valueOf(c));
+            throw new NumberFormatException("Expected a char in " + Arrays.toString(IntegerEncoder.CHARS) +
+                                            " but was '" + c + "'");
         }
     }
 }
diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
index 234558f5fd2..bcd5822a6f6 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramParser.java
@@ -987,7 +987,7 @@ final class ProgramParser {
         for (Field_defContext rulenode : fieldDefs) {
             // FIELD
                 // expression alias_def?
-                OperatorNode<ExpressionOperator> expr = convertExpr((ExpressionContext)rulenode.getChild(0), scope);
+                OperatorNode<ExpressionOperator> expr = convertExpr(rulenode.getChild(0), scope);
 
                 String aliasName = null;
                 if (rulenode.getChildCount() > 1) {
@@ -1018,9 +1018,8 @@ final class ProgramParser {
 	                            .constantMapExpression(), scope);
 	                    OperatorNode<ExpressionOperator> expr = OperatorNode.create(toLocation(scope, secondChild),
 	                            ExpressionOperator.VESPA_GROUPING, secondChild.getText());
-	                    List<String> names = (List<String>) annotation.getArgument(0);
-	                    List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation
-	                            .getArgument(1);
+	                    List<String> names = annotation.getArgument(0);
+	                    List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1);
 	                    for (int i = 0; i < names.size(); ++i) {
 	                        expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i)));
 	                    }
@@ -1147,8 +1146,8 @@ final class ProgramParser {
 			AnnotationContext annotateExpressionContext = ((AnnotateExpressionContext)parseTree).annotation();
 			OperatorNode<ExpressionOperator> annotation = convertExpr(annotateExpressionContext.constantMapExpression(), scope);
 			OperatorNode<ExpressionOperator> expr = convertExpr(parseTree.getChild(1), scope);
-			List<String> names = (List<String>) annotation.getArgument(0);
-			List<OperatorNode<ExpressionOperator>> annotates = (List<OperatorNode<ExpressionOperator>>) annotation.getArgument(1);
+			List<String> names = annotation.getArgument(0);
+			List<OperatorNode<ExpressionOperator>> annotates = annotation.getArgument(1);
 			for (int i = 0; i < names.size(); ++i) {
 				expr.putAnnotation(names.get(i), readConstantExpression(annotates.get(i)));
 			}
@@ -1375,9 +1374,9 @@ final class ProgramParser {
             case yqlplusParser.STRING:
                 return StringUnescaper.unquote(text);
             case yqlplusParser.TRUE:
-                return Boolean.valueOf(true);
+                return true;
             case yqlplusParser.FALSE:
-                return Boolean.valueOf(false);
+                return false;
             case yqlplusParser.LONG_INT:
                 return Long.parseLong(text.substring(0, text.length()-1));
             default:
@@ -1391,21 +1390,24 @@ final class ProgramParser {
                 return node.getArgument(0);
             case MAP: {
                 ImmutableMap.Builder<String, Object> map = ImmutableMap.builder();
-                List<String> names = (List<String>) node.getArgument(0);
-                List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(1);
+                List<String> names = node.getArgument(0);
+                List<OperatorNode<ExpressionOperator>> exprs = node.getArgument(1);
                 for (int i = 0; i < names.size(); ++i) {
                     map.put(names.get(i), readConstantExpression(exprs.get(i)));
                 }
                 return map.build();
             }
             case ARRAY: {
-                List<OperatorNode<ExpressionOperator>> exprs = (List<OperatorNode<ExpressionOperator>>) node.getArgument(0);
+                List<OperatorNode<ExpressionOperator>> exprs = node.getArgument(0);
                 ImmutableList.Builder<Object> lst = ImmutableList.builder();
                 for (OperatorNode<ExpressionOperator> expr : exprs) {
                     lst.add(readConstantExpression(expr));
                 }
                 return lst.build();
             }
+            case VARREF: {
+                return node; // must be dereferenced in YqlParser when we have userQuery
+            }
             default:
                 throw new ProgramCompileException(node.getLocation(), "Internal error: Unknown constant expression type: " + node.getOperator());
         }
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
index 3875018609e..739aae0e277 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
@@ -810,10 +810,11 @@ public class YqlParser implements Parser {
             OperatorNode<ExpressionOperator> groupingAst = ast.<List<OperatorNode<ExpressionOperator>>> getArgument(2).get(0);
             GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.<String> getArgument(0));
             VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation);
-            List<String> continuations = getAnnotation(groupingAst, "continuations", List.class,
+            List<Object> continuations = getAnnotation(groupingAst, "continuations", List.class,
                                                        Collections.emptyList(), "grouping continuations");
-            for (String continuation : continuations) {
-                groupingStep.continuations().add(Continuation.fromString(continuation));
+
+            for (Object continuation : continuations) {
+                groupingStep.continuations().add(Continuation.fromString(dereference(continuation)));
             }
             groupingSteps.add(groupingStep);
             ast = ast.getArgument(0);
@@ -822,6 +823,18 @@ public class YqlParser implements Parser {
         return ast;
     }
 
+    private String dereference(Object constantOrVarref) {
+        if (constantOrVarref instanceof OperatorNode) {
+            OperatorNode<?> varref = (OperatorNode<?>)constantOrVarref;
+            Preconditions.checkState(userQuery != null,
+                                     "properties must be available when trying to fetch user input");
+            return userQuery.properties().getString(varref.getArgument(0, String.class));
+        }
+        else {
+            return constantOrVarref.toString();
+        }
+    }
+
     private OperatorNode<?> fetchSorting(OperatorNode<?> ast) {
         if (ast.getOperator() != SequenceOperator.SORT) return ast;
 
@@ -1270,7 +1283,7 @@ public class YqlParser implements Parser {
         equiv.setIndexName(field);
         for (OperatorNode<ExpressionOperator> arg : args) {
             switch (arg.getOperator()) {
-                case LITERAL:
+                case LITERAL: case VARREF:
                     equiv.addItem(instantiateWordItem(field, arg, equiv.getClass()));
                     break;
                 case CALL:
@@ -1279,7 +1292,7 @@ public class YqlParser implements Parser {
                     break;
                 default:
                     throw newUnexpectedArgumentException(arg.getOperator(),
-                                                         ExpressionOperator.CALL, ExpressionOperator.LITERAL);
+                                                         ExpressionOperator.CALL, ExpressionOperator.LITERAL, ExpressionOperator.VARREF);
             }
         }
         return leafStyleSettings(ast, equiv);
@@ -1340,7 +1353,8 @@ public class YqlParser implements Parser {
     }
 
     private Item instantiateWordItem(String field,
-                                     OperatorNode<ExpressionOperator> ast, Class<?> parent,
+                                     OperatorNode<ExpressionOperator> ast,
+                                     Class<?> parent,
                                      SegmentWhen segmentPolicy) {
         String wordData = getStringContents(ast);
         return instantiateWordItem(field, wordData, ast, parent, segmentPolicy, null, decideParsingLanguage(ast, wordData));
@@ -1356,7 +1370,8 @@ public class YqlParser implements Parser {
     //       which always expands first, but not using getIndex, which performs checks that doesn't always work
     private Item instantiateWordItem(String field,
                                      String rawWord,
-                                     OperatorNode<ExpressionOperator> ast, Class<?> parent,
+                                     OperatorNode<ExpressionOperator> ast,
+                                     Class<?> parent,
                                      SegmentWhen segmentPolicy,
                                      Boolean exactMatch,
                                      Language language) {
@@ -1750,7 +1765,15 @@ public class YqlParser implements Parser {
         Object value = ast.getAnnotation(key);
         for (Iterator<OperatorNode<?>> i = annotationStack.iterator(); value == null
                                                                        && considerParents && i.hasNext();) {
-            value = i.next().getAnnotation(key);
+            OperatorNode node = i.next();
+            if (node.getOperator() == ExpressionOperator.VARREF) {
+                Preconditions.checkState(userQuery != null,
+                                         "properties must be available when trying to fetch user input");
+                value = userQuery.properties().getString(ast.getArgument(0, String.class));
+            }
+            else {
+                value = node.getAnnotation(key);
+            }
         }
         if (value == null) return defaultValue;
         Preconditions.checkArgument(expectedClass.isInstance(value),
diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java
index 75517a25909..06160e1c6d5 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java
@@ -15,6 +15,8 @@ import com.yahoo.search.Result;
 import com.yahoo.search.Searcher;
 import com.yahoo.search.searchchain.Execution;
 
+import java.util.Arrays;
+
 import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER;
 
 /**
@@ -204,13 +206,40 @@ public class UserInputTestCase {
         assertEquals("select * from sources * where year > 1980;", query.yqlRepresentation());
     }
 
+    @Test
+    public void testReferenceInContinuation() {
+        URIBuilder builder = searchUri();
+        builder.setParameter("continuation", "BCBCBCBEBG");
+        builder.setParameter("yql",
+                             "select * from sources * where myfield contains 'token'" +
+                             "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())));");
+        Query query = searchAndAssertNoErrors(builder);
+        assertEquals("select * from sources * where myfield contains \"token\" | [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())));", query.yqlRepresentation());
+    }
+
+    @Test
+    public void testReferenceInEquiv() {
+        URIBuilder builder = searchUri();
+        builder.setParameter("term", "A");
+        builder.setParameter("yql",
+                             "select foo from bar where fieldName contains equiv(@term,'B');");
+        Query query = searchAndAssertNoErrors(builder);
+        assertEquals("select foo from bar where fieldName contains equiv(\"A\", \"B\");", query.yqlRepresentation());
+    }
+
     private Query searchAndAssertNoErrors(URIBuilder builder) {
         Query query = new Query(builder.toString());
         Result r = execution.search(query);
-        assertNull(r.hits().getError());
+        assertNull(stackTraceIfAny(r), r.hits().getError());
         return query;
     }
 
+    private String stackTraceIfAny(Result r) {
+        if (r.hits().getError() == null) return "";
+        if (r.hits().getError().getCause() == null) return "";
+        return Arrays.toString(r.hits().getError().getCause().getStackTrace());
+    }
+
     private URIBuilder searchUri() {
         URIBuilder builder = new URIBuilder();
         builder.setPath("search/");
@@ -220,8 +249,7 @@ public class UserInputTestCase {
     @Test
     public void testEmptyUserInput() {
         URIBuilder builder = searchUri();
-        builder.setParameter("yql",
-                "select * from sources * where userInput(\"\");");
+        builder.setParameter("yql", "select * from sources * where userInput(\"\");");
         assertQueryFails(builder);
     }
 
@@ -229,8 +257,7 @@ public class UserInputTestCase {
     public void testEmptyUserInputFromQueryProperty() {
         URIBuilder builder = searchUri();
         builder.setParameter("foo", "");
-        builder.setParameter("yql",
-                "select * from sources * where userInput(@foo);");
+        builder.setParameter("yql", "select * from sources * where userInput(@foo);");
         assertQueryFails(builder);
     }