Merge branch 'master' into balder/reduce-timestamp-usage

7 files changed, 60 insertions, 25 deletions

diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json
index bde3b6abb6c..ff6c1ad0b9d 100644
--- a/searchlib/abi-spec.json
+++ b/searchlib/abi-spec.json
@@ -925,6 +925,8 @@
       "public final void labelAndDimensionValues(java.util.List)",
       "public final java.util.List valueAddress()",
       "public final com.yahoo.tensor.functions.Slice$DimensionValue dimensionValue(java.util.Optional)",
+      "public final java.lang.String label()",
+      "public final java.lang.String string()",
       "public void <init>(java.io.InputStream)",
       "public void <init>(java.io.InputStream, java.lang.String)",
       "public void ReInit(java.io.InputStream)",
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java
index 6200515462b..6e1cdf52158 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java
@@ -169,16 +169,24 @@ public class TensorFunctionNode extends CompositeNode {
 
             if (outermost instanceof ExpressionToStringContext) {
                 ExpressionToStringContext context = (ExpressionToStringContext)outermost;
-                return expression.toString(new StringBuilder(),
-                                           new ExpressionToStringContext(context.wrappedSerializationContext, c, context.path, context.parent),
-                                           context.path,
-                                           context.parent).toString();
+                ExpressionNode root = expression;
+                if (root instanceof CompositeNode && ! (root instanceof EmbracedNode) && ! isIdentifierReference(root))
+                    root = new EmbracedNode(root); // Output embraced if composite
+                return root.toString(new StringBuilder(),
+                                     new ExpressionToStringContext(context.wrappedSerializationContext, c, context.path, context.parent),
+                                     context.path,
+                                     context.parent).toString();
             }
             else {
                 return expression.toString();
             }
         }
 
+        private boolean isIdentifierReference(ExpressionNode node) {
+            if ( ! (node instanceof ReferenceNode)) return false;
+            return ((ReferenceNode)node).reference().isIdentifier();
+        }
+
     }
 
     /**
diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj
index e413e398183..c3d597fca46 100755
--- a/searchlib/src/main/javacc/RankingExpressionParser.jj
+++ b/searchlib/src/main/javacc/RankingExpressionParser.jj
@@ -894,7 +894,7 @@ void keyValue(TensorType type, java.util.Map cellMap) :
     ExpressionNode value;
 }
 {
-    label = tag() <COLON> value = expression()
+    label = label() <COLON> value = expression()
     { cellMap.put(TensorAddress.ofLabels(label), TensorFunctionNode.wrapScalar(value)); }
 }
 
@@ -904,7 +904,7 @@ void mixedBlock(TensorType type, List dimensionOrder, java.util.Map cellMap) :
     List cells;
 }
 {
-    label = tag() <COLON> cells = indexedTensorCells()
+    label = label() <COLON> cells = indexedTensorCells()
     { TensorFunctionNode.wrapScalarBlock(type, dimensionOrder, label, cells, cellMap); }
 }
 
@@ -952,7 +952,7 @@ void labelAndDimension(TensorAddress.Builder addressBuilder) :
     String dimension, label;
 }
 {
-    dimension = identifier() <COLON> label = tag()
+    dimension = identifier() <COLON> label = label()
     { addressBuilder.add(dimension, label); }
 }
 
@@ -1000,4 +1000,18 @@ Slice.DimensionValue dimensionValue(Optional dimensionName) :
     else
         return new Slice.DimensionValue(dimensionName, TensorFunctionNode.wrapScalar(value));
 }
+}
+
+String label() :
+{
+    String label;
+}
+{
+    ( label = tag() | label = string() )
+    { return label; }
+}
+
+String string() : {}
+{
+    <STRING> { return token.image.substring(1, token.image.length() - 1); }
 }
\ No newline at end of file
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
index b750a7607cc..ea09de32137 100755
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
@@ -150,11 +150,11 @@ public class RankingExpressionTestCase {
                             "map(constant(tensor0), f(a)(cos(a))) + l2_normalize(attribute(tensor1), x)");
         assertSerialization("join(reduce(join(reduce(join(constant(tensor0), attribute(tensor1), f(a,b)(a * b)), sum, x), attribute(tensor1), f(a,b)(a * b)), sum, y), query(tensor2), f(a,b)(a + b))", 
                             "xw_plus_b(matmul(constant(tensor0), attribute(tensor1), x), attribute(tensor1), query(tensor2), y)");
-        assertSerialization("tensor(x{}):{{x:a}:1 + 2 + 3,{x:b}:if (1 > 2, 3, 4),{x:c}:reduce(tensor0 * tensor1, sum)}",
+        assertSerialization("tensor(x{}):{{x:a}:(1 + 2 + 3),{x:b}:(if (1 > 2, 3, 4)),{x:c}:(reduce(tensor0 * tensor1, sum))}",
                             "tensor(x{}):{ {x:a}:1+2+3, {x:b}:if(1>2,3,4), {x:c}:sum(tensor0*tensor1) }");
         assertSerialization("tensor(x[3]):{{x:0}:1.0,{x:1}:2.0,{x:2}:3}",
                             "tensor(x[3]):[1.0, 2.0, 3]");
-        assertSerialization("tensor(x[3]):{{x:0}:1.0,{x:1}:reduce(tensor0 * tensor1, sum),{x:2}:3}",
+        assertSerialization("tensor(x[3]):{{x:0}:1.0,{x:1}:(reduce(tensor0 * tensor1, sum)),{x:2}:3}",
                             "tensor(x[3]):[1.0, sum(tensor0*tensor1), 3]");
     }
 
@@ -165,44 +165,44 @@ public class RankingExpressionTestCase {
         functions.add(new ExpressionFunction("tensorFunction", List.of(), new RankingExpression("tensor(x[3]):[1, 2, 3]")));
 
         // Getting a value from a tensor supplied by a function, inside a tensor generate function
-        assertSerialization(List.of("tensor(x[3])(rankingExpression(tensorFunction)[x])"),
+        assertSerialization(List.of("tensor(x[3])((rankingExpression(tensorFunction)[x]))"),
                             "tensor(x[3])(tensorFunction[x])",
                             functions, false);
 
         // Getting a value from a tensor supplied by a function, where the value index is supplied by a function, inside a tensor generate function, short form
-        assertSerialization(List.of("tensor(x[3])(rankingExpression(tensorFunction)[rankingExpression(scalarFunction)])"),
+        assertSerialization(List.of("tensor(x[3])((rankingExpression(tensorFunction)[(rankingExpression(scalarFunction))]))"),
                             "tensor(x[3])(tensorFunction[scalarFunction()])",
                             functions, false);
 
         // 'scalarFunction'  is interpreted as a label here since it is the short form of a mapped dimension
-        assertSerialization(List.of("tensor(x[3])(rankingExpression(tensorFunction){scalarFunction})"),
+        assertSerialization(List.of("tensor(x[3])((rankingExpression(tensorFunction){scalarFunction}))"),
                             "tensor(x[3])(tensorFunction{scalarFunction})",
                             functions, false);
 
         // Getting a value from a tensor supplied by a function, where the value index is supplied by a function, inside a tensor generate function, long form
-        assertSerialization(List.of("tensor(x[3])(rankingExpression(tensorFunction){x:rankingExpression(scalarFunction)})"),
+        assertSerialization(List.of("tensor(x[3])((rankingExpression(tensorFunction){x:(rankingExpression(scalarFunction))}))"),
                             "tensor(x[3])(tensorFunction{x:scalarFunction()})",
                             functions, false);
 
         // 'scalarFunction'  without parentheses is interpreted as a label instead of a reference to the function
-        assertSerialization(List.of("tensor(x[3])(rankingExpression(tensorFunction){x:scalarFunction})"),
+        assertSerialization(List.of("tensor(x[3])((rankingExpression(tensorFunction){x:scalarFunction}))"),
                             "tensor(x[3])(tensorFunction{x:scalarFunction})",
                             functions, false);
 
         // Accessing a function in a dynamic tensor, short form
-        assertSerialization(List.of("tensor(x[2]):{{x:0}:rankingExpression(scalarFunction),{x:1}:rankingExpression(scalarFunction)}"),
+        assertSerialization(List.of("tensor(x[2]):{{x:0}:(rankingExpression(scalarFunction)),{x:1}:(rankingExpression(scalarFunction))}"),
                             "tensor(x[2]):[scalarFunction(), scalarFunction()]",
                             functions, false);
 
         // Accessing a function in a dynamic tensor, long form
-        assertSerialization(List.of("tensor(x{}):{{x:foo}:rankingExpression(scalarFunction),{x:bar}:rankingExpression(scalarFunction)}"),
+        assertSerialization(List.of("tensor(x{}):{{x:foo}:(rankingExpression(scalarFunction)),{x:bar}:(rankingExpression(scalarFunction))}"),
                             "tensor(x{}):{{x:foo}:scalarFunction(), {x:bar}:scalarFunction()}",
                             functions, false);
 
         // Shadowing
-        assertSerialization(List.of("tensor(scalarFunction[1])(rankingExpression(tensorFunction){x:scalarFunction + rankingExpression(scalarFunction)})"),
+        assertSerialization(List.of("tensor(scalarFunction[1])((rankingExpression(tensorFunction){x:(scalarFunction + rankingExpression(scalarFunction))}))"),
                             "tensor(scalarFunction[1])(tensorFunction{x: scalarFunction + scalarFunction()})",
-                            functions, false);
+                            functions, true);
 
     }
 
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java
index 26861dd3cd6..ca2f6c6bbec 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java
@@ -324,7 +324,7 @@ public class EvaluationTestCase {
                                                                                 "{y:1}:((1+1)+a)}{y:1}" +
                                                            "}");
 
-        // tensor value
+        // tensor slice
         tester.assertEvaluates("3.0", "tensor0{x:1}", "{ {x:0}:1, {x:1}:3 }");
         tester.assertEvaluates("1.2", "tensor0{key:foo,x:0}", true, "{ {key:foo,x:0}:1.2, {key:bar,x:0}:3 }");
         tester.assertEvaluates("3.0", "tensor0{bar}", true, "{ {x:foo}:1, {x:bar}:3 }");
@@ -401,6 +401,9 @@ public class EvaluationTestCase {
         tester.assertEvaluates("tensor(x{}):{ {x:a}:6.0, {x:b}:4.0, {x:c}:14.0 }",
                                "tensor(x{}):{ {x:a}:1+2+3, {x:b}:if(1>2,3,4), {x:c}:sum(tensor0*tensor1) }",
                                "{ {x:0}:7 }", "tensor(x{}):{ {x:0}:2 }");
+        tester.assertEvaluates("tensor(x{}):{ {x:a}:6.0, {x:b}:4.0, {x:'--'}:14.0 }",
+                               "tensor(x{}):{ a:1+2+3, b:if(1>2,3,4), '--':sum(tensor0*tensor1) }",
+                               "{ {x:0}:7 }", "tensor(x{}):{ {x:0}:2 }");
         tester.assertEvaluates("tensor<float>(d0[1],x[3]):[[1.0, 0.5, 0.25]]",
                                "tensor<float>(d0[1],x[3]):[[one,one_half,a_quarter]]");
         tester.assertEvaluates("tensor(x[2],y[3]):[[1.0, 0.5, 0.25],[0.25, 0.5, 1.0]]",
@@ -410,10 +413,10 @@ public class EvaluationTestCase {
         tester.assertEvaluates("tensor(x{},y[2]):{a:[1.0, 0.5], b:[0.25, 2]}",
                                "tensor(x{},y[2]):{a:[one, one_half], b:[a_quarter, 2]}");
         tester.assertEvaluates("tensor(key{},x[2],y[3]):{key1:[[1.0, 0.5, 0.25],[0.25, 0.5, 1.0]]," +
-                               "                                       key2:[[1.0, 2.0, 3.00],[4.00, 5.0, 6.0]]}",
+                               "                                       'key2.[]':[[1.0, 2.0, 3.00],[4.00, 5.0, 6.0]]}",
                                "tensor(key{},x[2],y[3]):{key1:[[one,one_half,a_quarter],[a_quarter,one_half,one]]," +
-                               "                                        key2:[[1,2,3],[4,5,6]]}");
-        tester.assertEvaluates("tensor(x{}):{{x:a}:1, {x:b}:-2, {x:cee}:0.5}", "tensor(x{}):{a:1, b:-2, cee:one_half}");
+                               "                                        'key2.[]':[[1,2,3],[4,5,6]]}");
+        tester.assertEvaluates("tensor(x{}):{{x:a}:1, {x:'\"'}:-2, {x:\"'\"}:0.5}", "tensor(x{}):{a:1, '\"':-2, \"'\":one_half}");
 
         // Opposite order in the expression:
         // - indexed
diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp
index 244a4e26554..896a917d6e3 100644
--- a/searchlib/src/tests/fef/fef_test.cpp
+++ b/searchlib/src/tests/fef/fef_test.cpp
@@ -68,14 +68,23 @@ TEST("test TermFieldMatchDataAppend")
     tmd.appendPosition(pos);
     EXPECT_EQUAL(2u, tmd.size());
     EXPECT_EQUAL(2u, tmd.capacity());
+    uint32_t resizeCount(0);
+    const TermFieldMatchDataPosition * prev = tmd.begin();
     for (size_t i(2); i < std::numeric_limits<uint16_t>::max(); i++) {
         EXPECT_EQUAL(i, tmd.size());
         EXPECT_EQUAL(std::min(size_t(std::numeric_limits<uint16_t>::max()), vespalib::roundUp2inN(i)), tmd.capacity());
         tmd.appendPosition(pos);
+        const TermFieldMatchDataPosition * cur = tmd.begin();
+        if (cur != prev) {
+            prev = cur;
+            resizeCount++;
+        }
     }
+    EXPECT_EQUAL(15u, resizeCount);
     EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.size());
     EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.capacity());
     tmd.appendPosition(pos);
+    EXPECT_EQUAL(prev, tmd.begin());
     EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.size());
     EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.capacity());
 }
diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
index 4de84866edf..781135bf246 100644
--- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
+++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.cpp
@@ -142,6 +142,7 @@ TermFieldMatchData::allocateVector()
 void
 TermFieldMatchData::appendPositionToAllocatedVector(const TermFieldMatchDataPosition &pos)
 {
+    if (__builtin_expect(_sz >= MAX_ELEMS, false)) return;
     assert(allocated());
     if (__builtin_expect(_sz >= _data._positions._allocated, false)) {
         resizePositionVector(_sz*2);
@@ -149,9 +150,7 @@ TermFieldMatchData::appendPositionToAllocatedVector(const TermFieldMatchDataPosi
     if (__builtin_expect(pos.getElementLen() > _data._positions._maxElementLength, false)) {
         _data._positions._maxElementLength = pos.getElementLen();
     }
-    if (__builtin_expect(_sz < MAX_ELEMS, true)) {
-        _data._positions._positions[_sz++] = pos;
-    }
+    _data._positions._positions[_sz++] = pos;
 }
 
 }