summaryrefslogtreecommitdiffstats
path: root/eval
diff options
context:
space:
mode:
authorHåvard Pettersen <havardpe@oath.com>2019-12-06 10:22:04 +0000
committerHåvard Pettersen <havardpe@oath.com>2019-12-06 10:22:04 +0000
commite9f6c933f4ac542c77c12b54170055a85d2a8640 (patch)
treea47156a4c8c7d7b7d3b14e1569be47e44d5ea4f0 /eval
parent1cfdd7f98bf1a33e7edc6cfb646ee53d5654c7f4 (diff)
improve label parsing in ranking expressions
unquoted labels can now contain almost anything labels can now also be quoted (in tensor create/peek) NB: indexes in verbose tensor create cannot be quoted expressions in tensor peek must have () around them expression results are rounded before conversion the sign is kept when expression results are converted to labels trivial number expressions in tensor peek are converted to labels ... by the parser, to better inline bindings created by tensor lambdas verbatim tensor peek for indexed dimensions will fail type resolution ... when the label cannot be converted to a valid index
Diffstat (limited to 'eval')
-rw-r--r--eval/src/tests/eval/function/function_test.cpp72
-rw-r--r--eval/src/tests/eval/node_types/node_types_test.cpp4
-rw-r--r--eval/src/vespa/eval/eval/basic_nodes.cpp43
-rw-r--r--eval/src/vespa/eval/eval/basic_nodes.h4
-rw-r--r--eval/src/vespa/eval/eval/function.cpp70
-rw-r--r--eval/src/vespa/eval/eval/string_stuff.cpp42
-rw-r--r--eval/src/vespa/eval/eval/string_stuff.h5
-rw-r--r--eval/src/vespa/eval/eval/tensor_function.cpp6
-rw-r--r--eval/src/vespa/eval/eval/tensor_nodes.h2
-rw-r--r--eval/src/vespa/eval/eval/test/tensor_conformance.cpp8
10 files changed, 141 insertions, 115 deletions
diff --git a/eval/src/tests/eval/function/function_test.cpp b/eval/src/tests/eval/function/function_test.cpp
index 6b9b52900c7..7cfe54b4b95 100644
--- a/eval/src/tests/eval/function/function_test.cpp
+++ b/eval/src/tests/eval/function/function_test.cpp
@@ -828,11 +828,15 @@ TEST("require that tensor lambda can use non-dimension symbols") {
TEST("require that verbose tensor create can be parsed") {
auto dense = Function::parse("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}");
- auto sparse = Function::parse("tensor(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}");
- auto mixed = Function::parse("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}");
+ auto sparse1 = Function::parse("tensor(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}");
+ auto sparse2 = Function::parse("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}");
+ auto mixed1 = Function::parse("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}");
+ auto mixed2 = Function::parse("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}");
EXPECT_EQUAL("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}", dense.dump());
- EXPECT_EQUAL("tensor(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}", sparse.dump());
- EXPECT_EQUAL("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}", mixed.dump());
+ EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse1.dump());
+ EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse2.dump());
+ EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed1.dump());
+ EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed2.dump());
}
TEST("require that verbose tensor create can contain expressions") {
@@ -873,15 +877,24 @@ TEST("require that verbose tensor create detects non-numeric indexes for indexed
"[tensor(x[1]):{{x:]...[expected number]...[foo}:1}]"));
}
+TEST("require that verbose tensor create indexes cannot be quoted") {
+ TEST_DO(verify_error("tensor(x[1]):{{x:\"1\"}:1}",
+ "[tensor(x[1]):{{x:]...[expected number]...[\"1\"}:1}]"));
+}
+
//-----------------------------------------------------------------------------
TEST("require that convenient tensor create can be parsed") {
auto dense = Function::parse("tensor(x[3]):[1,2,3]");
- auto sparse = Function::parse("tensor(x{}):{a:1,b:2,c:3}");
- auto mixed = Function::parse("tensor(x{},y[2]):{a:[1,2]}");
+ auto sparse1 = Function::parse("tensor(x{}):{a:1,b:2,c:3}");
+ auto sparse2 = Function::parse("tensor(x{}):{\"a\":1,\"b\":2,\"c\":3}");
+ auto mixed1 = Function::parse("tensor(x{},y[2]):{a:[1,2]}");
+ auto mixed2 = Function::parse("tensor(x{},y[2]):{\"a\":[1,2]}");
EXPECT_EQUAL("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}", dense.dump());
- EXPECT_EQUAL("tensor(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}", sparse.dump());
- EXPECT_EQUAL("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}", mixed.dump());
+ EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse1.dump());
+ EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse2.dump());
+ EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed1.dump());
+ EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed2.dump());
}
TEST("require that convenient tensor create can contain expressions") {
@@ -893,11 +906,11 @@ TEST("require that convenient tensor create can contain expressions") {
TEST("require that convenient tensor create handles dimension order") {
auto mixed = Function::parse("tensor(y{},x[2]):{a:[1,2]}");
- EXPECT_EQUAL("tensor(x[2],y{}):{{x:0,y:a}:1,{x:1,y:a}:2}", mixed.dump());
+ EXPECT_EQUAL("tensor(x[2],y{}):{{x:0,y:\"a\"}:1,{x:1,y:\"a\"}:2}", mixed.dump());
}
TEST("require that convenient tensor create can be highly nested") {
- vespalib::string expect("tensor(a{},b{},c[1],d[1]):{{a:x,b:y,c:0,d:0}:5}");
+ vespalib::string expect("tensor(a{},b{},c[1],d[1]):{{a:\"x\",b:\"y\",c:0,d:0}:5}");
auto nested1 = Function::parse("tensor(a{},b{},c[1],d[1]):{x:{y:[[5]]}}");
auto nested2 = Function::parse("tensor(c[1],d[1],a{},b{}):[[{x:{y:5}}]]");
auto nested3 = Function::parse("tensor(a{},c[1],b{},d[1]): { x : [ { y : [ 5 ] } ] } ");
@@ -907,7 +920,7 @@ TEST("require that convenient tensor create can be highly nested") {
}
TEST("require that convenient tensor create can have multiple values on multiple levels") {
- vespalib::string expect("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2,{x:b,y:0}:3,{x:b,y:1}:4}");
+ vespalib::string expect("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2,{x:\"b\",y:0}:3,{x:\"b\",y:1}:4}");
auto fun1 = Function::parse("tensor(x{},y[2]):{a:[1,2],b:[3,4]}");
auto fun2 = Function::parse("tensor(y[2],x{}):[{a:1,b:3},{a:2,b:4}]");
auto fun3 = Function::parse("tensor(x{},y[2]): { a : [ 1 , 2 ] , b : [ 3 , 4 ] } ");
@@ -941,41 +954,44 @@ TEST("require that convenient tensor create detects under-specified cells") {
//-----------------------------------------------------------------------------
TEST("require that tensor peek can be parsed") {
- TEST_DO(verify_parse("t{x:1,y:foo}", "f(t)(t{x:1,y:foo})"));
+ TEST_DO(verify_parse("t{x:\"1\",y:\"foo\"}", "f(t)(t{x:\"1\",y:\"foo\"})"));
+ TEST_DO(verify_parse("t{x:1,y:foo}", "f(t)(t{x:\"1\",y:\"foo\"})"));
}
TEST("require that tensor peek can contain expressions") {
- TEST_DO(verify_parse("t{x:1+2,y:foo}", "f(t)(t{x:(1+2),y:foo})"));
- TEST_DO(verify_parse("t{x:1+bar,y:foo}", "f(t,bar)(t{x:(1+bar),y:foo})"));
- TEST_DO(verify_parse("t{x:1,y:foo+2}", "f(t,foo)(t{x:1,y:(foo+2)})"));
- TEST_DO(verify_parse("t{x:1,y:(foo)}", "f(t,foo)(t{x:1,y:(foo)})"));
+ TEST_DO(verify_parse("t{x:(1+2),y:1+2}", "f(t)(t{x:(1+2),y:\"1+2\"})"));
+ TEST_DO(verify_parse("t{x:(foo),y:foo}", "f(t,foo)(t{x:(foo),y:\"foo\"})"));
+ TEST_DO(verify_parse("t{x:(foo+2),y:foo+2}", "f(t,foo)(t{x:(foo+2),y:\"foo+2\"})"));
}
-TEST("require that trivial tensor peek string/number expressions are converted to verbatim labels") {
- TEST_DO(verify_parse("t{x:\"foo\"}", "f(t)(t{x:foo})"));
- TEST_DO(verify_parse("t{x:(\"foo\")}", "f(t)(t{x:foo})"));
- TEST_DO(verify_parse("t{x:5.5}", "f(t)(t{x:5})"));
- TEST_DO(verify_parse("t{x:(5.5)}", "f(t)(t{x:5})"));
+TEST("require that trivial tensor peek number expressions are converted to verbatim labels") {
+ TEST_DO(verify_parse("t{x:(5.7)}", "f(t)(t{x:\"6\"})"));
+ TEST_DO(verify_parse("t{x:(5.3)}", "f(t)(t{x:\"5\"})"));
+ TEST_DO(verify_parse("t{x:(-5.7)}", "f(t)(t{x:\"-6\"})"));
+ TEST_DO(verify_parse("t{x:(-5.3)}", "f(t)(t{x:\"-5\"})"));
}
TEST("require that tensor peek can contain extra whitespace") {
- TEST_DO(verify_parse(" t { x : 1 + bar , y : foo + 2 } ",
+ TEST_DO(verify_parse(" t { x : ( 1 + bar ) , y : ( foo + 2 ) } ",
"f(t,bar,foo)(t{x:(1+bar),y:(foo+2)})"));
-}
-
-TEST("require that converted tensor peek string expression must be valid identifier") {
- TEST_DO(verify_error("x{a:\"5.5\"}", "[x{a:\"5.5\"]...[invalid identifier: '5.5']...[}]"));
+ TEST_DO(verify_parse(" t { x : \"1 + bar\" , y : \"foo + 2\" } ",
+ "f(t)(t{x:\"1 + bar\",y:\"foo + 2\"})"));
}
TEST("require that empty tensor peek is not allowed") {
TEST_DO(verify_error("x{}", "[x{}]...[empty peek spec]...[]"));
}
+TEST("require that tensor peek empty label is not allowed") {
+ TEST_DO(verify_error("x{a:}", "[x{a:]...[missing label]...[}]"));
+ TEST_DO(verify_error("x{a:\"\"}", "[x{a:\"\"]...[missing label]...[}]"));
+}
+
//-----------------------------------------------------------------------------
TEST("require that nested tensor lambda using tensor peek can be parsed") {
- vespalib::string expect("tensor(x[2]):{{x:0}:tensor(y[2]):{{y:0}:((0+0)+a),{y:1}:((0+1)+a)}{y:0},"
- "{x:1}:tensor(y[2]):{{y:0}:((1+0)+a),{y:1}:((1+1)+a)}{y:1}}");
+ vespalib::string expect("tensor(x[2]):{{x:0}:tensor(y[2]):{{y:0}:((0+0)+a),{y:1}:((0+1)+a)}{y:\"0\"},"
+ "{x:1}:tensor(y[2]):{{y:0}:((1+0)+a),{y:1}:((1+1)+a)}{y:\"1\"}}");
EXPECT_EQUAL(Function::parse(expect).dump(), expect);
auto fun = Function::parse("tensor(x[2])(tensor(y[2])(x+y+a){y:(x)})");
EXPECT_EQUAL(fun.dump(), expect);
diff --git a/eval/src/tests/eval/node_types/node_types_test.cpp b/eval/src/tests/eval/node_types/node_types_test.cpp
index 97a80d9b4b3..6fbcea8a8a1 100644
--- a/eval/src/tests/eval/node_types/node_types_test.cpp
+++ b/eval/src/tests/eval/node_types/node_types_test.cpp
@@ -222,7 +222,7 @@ TEST("require that tensor peek resolves correct type") {
TEST_DO(verify("tensor(x{}){x:1}", "double"));
TEST_DO(verify("tensor(x{}){x:foo}", "double"));
TEST_DO(verify("tensor(x{}){x:(double)}", "double"));
- TEST_DO(verify("tensor(x{}){x:tensor(x[3])}", "error"));
+ TEST_DO(verify("tensor(x{}){x:(tensor(x[3]))}", "error"));
TEST_DO(verify("tensor(x{},y[3]){x:foo,y:2}", "double"));
TEST_DO(verify("tensor(x{},y[3]){x:foo}", "tensor(y[3])"));
TEST_DO(verify("tensor(x{},y[3]){y:2}", "tensor(x{})"));
@@ -233,7 +233,7 @@ TEST("require that tensor peek resolves correct type") {
TEST_DO(verify("tensor<float>(x{}){x:1}", "double"));
TEST_DO(verify("tensor<float>(x{}){x:foo}", "double"));
TEST_DO(verify("tensor<float>(x{}){x:(double)}", "double"));
- TEST_DO(verify("tensor<float>(x{}){x:tensor(x[3])}", "error"));
+ TEST_DO(verify("tensor<float>(x{}){x:(tensor(x[3]))}", "error"));
TEST_DO(verify("tensor<float>(x{},y[3]){x:foo,y:2}", "double"));
TEST_DO(verify("tensor<float>(x{},y[3]){x:foo}", "tensor<float>(y[3])"));
TEST_DO(verify("tensor<float>(x{},y[3]){y:2}", "tensor<float>(x{})"));
diff --git a/eval/src/vespa/eval/eval/basic_nodes.cpp b/eval/src/vespa/eval/eval/basic_nodes.cpp
index 473deca1117..1c46f2ee322 100644
--- a/eval/src/vespa/eval/eval/basic_nodes.cpp
+++ b/eval/src/vespa/eval/eval/basic_nodes.cpp
@@ -66,49 +66,6 @@ void Not ::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
void If ::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
void Error ::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
-vespalib::string
-String::dump(DumpContext &) const
-{
- vespalib::string str;
- str.push_back('"');
- for (uint32_t i = 0; i < _value.size(); ++i) {
- char c = _value[i];
- switch (c) {
- case '\\':
- str.append("\\\\");
- break;
- case '"':
- str.append("\\\"");
- break;
- case '\t':
- str.append("\\t");
- break;
- case '\n':
- str.append("\\n");
- break;
- case '\r':
- str.append("\\r");
- break;
- case '\f':
- str.append("\\f");
- break;
- default:
- if (static_cast<unsigned char>(c) >= 32 &&
- static_cast<unsigned char>(c) <= 126)
- {
- str.push_back(c);
- } else {
- const char *lookup = "0123456789abcdef";
- str.append("\\x");
- str.push_back(lookup[(c >> 4) & 0xf]);
- str.push_back(lookup[c & 0xf]);
- }
- }
- }
- str.push_back('"');
- return str;
-}
-
If::If(Node_UP cond_in, Node_UP true_expr_in, Node_UP false_expr_in, double p_true_in)
: _cond(std::move(cond_in)),
_true_expr(std::move(true_expr_in)),
diff --git a/eval/src/vespa/eval/eval/basic_nodes.h b/eval/src/vespa/eval/eval/basic_nodes.h
index 62af30fb71f..06c296fe753 100644
--- a/eval/src/vespa/eval/eval/basic_nodes.h
+++ b/eval/src/vespa/eval/eval/basic_nodes.h
@@ -124,7 +124,9 @@ public:
double get_const_value() const override { return hash(); }
const vespalib::string &value() const { return _value; }
uint32_t hash() const { return hash_code(_value.data(), _value.size()); }
- vespalib::string dump(DumpContext &ctx) const override;
+ vespalib::string dump(DumpContext &) const override {
+ return as_quoted_string(_value);
+ }
void accept(NodeVisitor &visitor) const override;
};
diff --git a/eval/src/vespa/eval/eval/function.cpp b/eval/src/vespa/eval/eval/function.cpp
index 593451d6838..baaaff25b34 100644
--- a/eval/src/vespa/eval/eval/function.cpp
+++ b/eval/src/vespa/eval/eval/function.cpp
@@ -363,8 +363,7 @@ int unhex(char c) {
return -1;
}
-void parse_string(ParseContext &ctx) {
- vespalib::string &str = ctx.scratch();
+void extract_quoted_string(ParseContext &ctx, vespalib::string &str) {
ctx.eat('"');
while (!ctx.eos() && ctx.get() != '"') {
if (ctx.get() == '\\') {
@@ -395,6 +394,11 @@ void parse_string(ParseContext &ctx) {
ctx.next();
}
ctx.eat('"');
+}
+
+void parse_string(ParseContext &ctx) {
+ vespalib::string &str = ctx.scratch();
+ extract_quoted_string(ctx, str);
ctx.push_expression(Node_UP(new nodes::String(str)));
}
@@ -445,14 +449,6 @@ bool is_ident(char c, bool first) {
(c == '$' && !first));
}
-bool is_ident(const vespalib::string &str) {
- bool result = str.empty() ? false : is_ident(str[0], true);
- for (size_t i = 1; result && (i < str.size()); ++i) {
- result &= is_ident(str[i], false);
- }
- return result;
-}
-
vespalib::string get_ident(ParseContext &ctx, bool allow_empty) {
ctx.skip_spaces();
vespalib::string ident;
@@ -480,6 +476,28 @@ size_t get_size_t(ParseContext &ctx) {
return atoi(num.c_str());
}
+bool is_label_end(char c) {
+ return (isspace(c) || (c == '\0') ||
+ (c == ':') || (c == ',') || (c == '}'));
+}
+
+vespalib::string get_label(ParseContext &ctx) {
+ ctx.skip_spaces();
+ vespalib::string label;
+ if (ctx.get() == '"') {
+ extract_quoted_string(ctx, label);
+ } else {
+ while (!is_label_end(ctx.get())) {
+ label.push_back(ctx.get());
+ ctx.next();
+ }
+ }
+ if (label.empty()) {
+ ctx.fail("missing label");
+ }
+ return label;
+}
+
void parse_if(ParseContext &ctx) {
Node_UP cond = get_expression(ctx);
ctx.eat(',');
@@ -625,7 +643,7 @@ TensorSpec::Address get_tensor_address(ParseContext &ctx, const ValueType &type)
ctx.skip_spaces();
ctx.eat(':');
if (dim.is_mapped()) {
- addr.emplace(dim_name, get_ident(ctx, false));
+ addr.emplace(dim_name, get_label(ctx));
} else {
size_t idx = get_size_t(ctx);
if (idx < dim.size) {
@@ -704,7 +722,7 @@ void parse_tensor_create_convenient(ParseContext &ctx, const ValueType &type,
}
}
if (addr.back().is_mapped()) {
- addr.back().name = get_ident(ctx, false);
+ addr.back().name = get_label(ctx);
ctx.skip_spaces();
ctx.eat(':');
}
@@ -797,28 +815,16 @@ void parse_tensor_peek(ParseContext &ctx) {
auto dim_name = get_ident(ctx, false);
ctx.skip_spaces();
ctx.eat(':');
- if (!ctx.failed()) {
- ParseContext::InputMark before_label = ctx.get_input_mark();
- auto label = get_ident(ctx, false);
- bool verbatim = !ctx.failed() && ctx.find_expression_end();
- if (verbatim) {
- peek_spec.emplace(dim_name, label);
+ ctx.skip_spaces();
+ if (ctx.get() == '(') {
+ auto expr = get_expression(ctx);
+ if (auto num = nodes::as<nodes::Number>(*expr)) {
+ peek_spec.emplace(dim_name, make_string("%ld", int64_t(round(num->value()))));
} else {
- ctx.restore_input_mark(before_label);
- auto expr = get_expression(ctx);
- if (auto num = nodes::as<nodes::Number>(*expr)) {
- size_t index(num->value());
- peek_spec.emplace(dim_name, make_string("%zu", index));
- } else if (auto str = nodes::as<nodes::String>(*expr)) {
- if (is_ident(str->value())) {
- peek_spec.emplace(dim_name, str->value());
- } else {
- ctx.fail(make_string("invalid identifier: '%s'", str->value().c_str()));
- }
- } else {
- peek_spec.emplace(dim_name, std::move(expr));
- }
+ peek_spec.emplace(dim_name, std::move(expr));
}
+ } else {
+ peek_spec.emplace(dim_name, get_label(ctx));
}
}
ctx.eat('}');
diff --git a/eval/src/vespa/eval/eval/string_stuff.cpp b/eval/src/vespa/eval/eval/string_stuff.cpp
index 1fd8649bb57..ce8e95b6d05 100644
--- a/eval/src/vespa/eval/eval/string_stuff.cpp
+++ b/eval/src/vespa/eval/eval/string_stuff.cpp
@@ -5,6 +5,46 @@
namespace vespalib::eval {
+vespalib::string as_quoted_string(const vespalib::string &str) {
+ vespalib::string res;
+ res.push_back('"');
+ for (char c: str) {
+ switch (c) {
+ case '\\':
+ res.append("\\\\");
+ break;
+ case '"':
+ res.append("\\\"");
+ break;
+ case '\t':
+ res.append("\\t");
+ break;
+ case '\n':
+ res.append("\\n");
+ break;
+ case '\r':
+ res.append("\\r");
+ break;
+ case '\f':
+ res.append("\\f");
+ break;
+ default:
+ if (static_cast<unsigned char>(c) >= 32 &&
+ static_cast<unsigned char>(c) <= 126)
+ {
+ res.push_back(c);
+ } else {
+ const char *lookup = "0123456789abcdef";
+ res.append("\\x");
+ res.push_back(lookup[(c >> 4) & 0xf]);
+ res.push_back(lookup[c & 0xf]);
+ }
+ }
+ }
+ res.push_back('"');
+ return res;
+}
+
bool is_number(const vespalib::string &str) {
for (char c: str) {
if (!isdigit(c)) {
@@ -24,7 +64,7 @@ vespalib::string as_string(const TensorSpec::Address &address) {
for (const auto &label: address) {
label_list.maybe_add_comma(str);
if (label.second.is_mapped()) {
- str += make_string("%s:%s", label.first.c_str(), label.second.name.c_str());
+ str += make_string("%s:%s", label.first.c_str(), as_quoted_string(label.second.name).c_str());
} else {
str += make_string("%s:%zu", label.first.c_str(), label.second.index);
}
diff --git a/eval/src/vespa/eval/eval/string_stuff.h b/eval/src/vespa/eval/eval/string_stuff.h
index 68b45b0aa38..692b7119691 100644
--- a/eval/src/vespa/eval/eval/string_stuff.h
+++ b/eval/src/vespa/eval/eval/string_stuff.h
@@ -38,6 +38,11 @@ struct CommaTracker {
};
/**
+ * Convert the given string to a quoted string with escaped special characters.
+ **/
+vespalib::string as_quoted_string(const vespalib::string &str);
+
+/**
* Is this string a positive integer (dimension index)
**/
bool is_number(const vespalib::string &str);
diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp
index 1f87c0390af..e98a56a0148 100644
--- a/eval/src/vespa/eval/eval/tensor_function.cpp
+++ b/eval/src/vespa/eval/eval/tensor_function.cpp
@@ -168,14 +168,14 @@ void op_tensor_peek(State &state, uint64_t param) {
addr.emplace(pos->first, std::get<TensorSpec::Label>(pos->second));
} else {
assert(std::holds_alternative<TensorFunction::Child>(pos->second));
- size_t index(state.peek(child_cnt++).as_double());
+ double index = round(state.peek(child_cnt++).as_double());
size_t dim_idx = self.param_type().dimension_index(pos->first);
assert(dim_idx != ValueType::Dimension::npos);
const auto &param_dim = self.param_type().dimensions()[dim_idx];
if (param_dim.is_mapped()) {
- addr.emplace(pos->first, vespalib::make_string("%zu", index));
+ addr.emplace(pos->first, vespalib::make_string("%ld", int64_t(index)));
} else {
- addr.emplace(pos->first, index);
+ addr.emplace(pos->first, size_t(index));
}
}
}
diff --git a/eval/src/vespa/eval/eval/tensor_nodes.h b/eval/src/vespa/eval/eval/tensor_nodes.h
index ca3369ad9ba..a307faf9b36 100644
--- a/eval/src/vespa/eval/eval/tensor_nodes.h
+++ b/eval/src/vespa/eval/eval/tensor_nodes.h
@@ -285,7 +285,7 @@ public:
str += ")";
}
} else {
- str += dim.second.label;
+ str += as_quoted_string(dim.second.label);
}
}
str += "}";
diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp
index 4eb13a1964c..3af7ea0a1f3 100644
--- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp
+++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp
@@ -817,10 +817,10 @@ struct TestContext {
void test_tensor_peek() {
auto param_double = spec({x({"0", "1"}),y(2)}, Seq({1.0, 2.0, 3.0, 4.0}));
auto param_float = spec(float_cells({x({"0", "1"}),y(2)}), Seq({1.0, 2.0, 3.0, 4.0}));
- TEST_DO(test_tensor_peek("tensor(x[2]):[a{x:1,y:1},a{x:b-1,y:b-1}]", param_double, spec(x(2), Seq({4.0, 1.0}))));
- TEST_DO(test_tensor_peek("tensor(x[2]):[a{x:1,y:1},a{x:b-1,y:b-1}]", param_float, spec(x(2), Seq({4.0, 1.0}))));
- TEST_DO(test_tensor_peek("tensor<float>(x[2]):[a{x:1,y:1},a{x:b-1,y:b-1}]", param_double, spec(float_cells({x(2)}), Seq({4.0, 1.0}))));
- TEST_DO(test_tensor_peek("tensor<float>(x[2]):[a{x:1,y:1},a{x:b-1,y:b-1}]", param_float, spec(float_cells({x(2)}), Seq({4.0, 1.0}))));
+ TEST_DO(test_tensor_peek("tensor(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_double, spec(x(2), Seq({4.0, 1.0}))));
+ TEST_DO(test_tensor_peek("tensor(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_float, spec(x(2), Seq({4.0, 1.0}))));
+ TEST_DO(test_tensor_peek("tensor<float>(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_double, spec(float_cells({x(2)}), Seq({4.0, 1.0}))));
+ TEST_DO(test_tensor_peek("tensor<float>(x[2]):[a{x:1,y:1},a{x:(b-1),y:(b-1)}]", param_float, spec(float_cells({x(2)}), Seq({4.0, 1.0}))));
TEST_DO(test_tensor_peek("a{x:(b)}", param_double, spec(y(2), Seq({3.0, 4.0}))));
TEST_DO(test_tensor_peek("a{x:(b)}", param_float, spec(float_cells({y(2)}), Seq({3.0, 4.0}))));
TEST_DO(test_tensor_peek("a{y:(b)}", param_double, spec(x({"0", "1"}), Seq({2.0, 4.0}))));