diff options
author | Håvard Pettersen <havardpe@oath.com> | 2019-12-17 14:59:22 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2019-12-17 14:59:22 +0000 |
commit | 664c951713cb1d603269367f1f15eceb594c626a (patch) | |
tree | 1c68d9925fd0e962d80a021f63573653eca023ba /eval | |
parent | a6f89a895b0d825b93ce08b1a194175d7d293eef (diff) |
allow single quotes for strings (includes quoted tensor labels)
Diffstat (limited to 'eval')
-rw-r--r-- | eval/src/tests/eval/function/function_test.cpp | 12 | ||||
-rw-r--r-- | eval/src/vespa/eval/eval/function.cpp | 21 |
2 files changed, 25 insertions, 8 deletions
diff --git a/eval/src/tests/eval/function/function_test.cpp b/eval/src/tests/eval/function/function_test.cpp index fef0ed35668..5316217d549 100644 --- a/eval/src/tests/eval/function/function_test.cpp +++ b/eval/src/tests/eval/function/function_test.cpp @@ -168,6 +168,11 @@ TEST("require that strings are parsed and dumped correctly") { } } +TEST("require that strings with single quotes can be parsed") { + EXPECT_EQUAL(Function::parse("'foo'")->dump(), "\"foo\""); + EXPECT_EQUAL(Function::parse("'fo\\'o'")->dump(), "\"fo'o\""); +} + TEST("require that free arrays cannot be parsed") { verify_error("[1,2,3]", "[]...[missing value]...[[1,2,3]]"); } @@ -830,11 +835,13 @@ TEST("require that verbose tensor create can be parsed") { auto dense = Function::parse("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}"); auto sparse1 = Function::parse("tensor(x{}):{{x:a}:1,{x:b}:2,{x:c}:3}"); auto sparse2 = Function::parse("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}"); + auto sparse3 = Function::parse("tensor(x{}):{{x:'a'}:1,{x:'b'}:2,{x:'c'}:3}"); auto mixed1 = Function::parse("tensor(x{},y[2]):{{x:a,y:0}:1,{x:a,y:1}:2}"); auto mixed2 = Function::parse("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}"); EXPECT_EQUAL("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}", dense->dump()); EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse1->dump()); EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse2->dump()); + EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse3->dump()); EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed1->dump()); EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed2->dump()); } @@ -880,6 +887,8 @@ TEST("require that verbose tensor create detects non-numeric indexes for indexed TEST("require that verbose tensor create indexes cannot be quoted") { TEST_DO(verify_error("tensor(x[1]):{{x:\"1\"}:1}", "[tensor(x[1]):{{x:]...[expected number]...[\"1\"}:1}]")); + TEST_DO(verify_error("tensor(x[1]):{{x:'1'}:1}", + "[tensor(x[1]):{{x:]...[expected number]...['1'}:1}]")); } //----------------------------------------------------------------------------- @@ -888,11 +897,13 @@ TEST("require that convenient tensor create can be parsed") { auto dense = Function::parse("tensor(x[3]):[1,2,3]"); auto sparse1 = Function::parse("tensor(x{}):{a:1,b:2,c:3}"); auto sparse2 = Function::parse("tensor(x{}):{\"a\":1,\"b\":2,\"c\":3}"); + auto sparse3 = Function::parse("tensor(x{}):{'a':1,'b':2,'c':3}"); auto mixed1 = Function::parse("tensor(x{},y[2]):{a:[1,2]}"); auto mixed2 = Function::parse("tensor(x{},y[2]):{\"a\":[1,2]}"); EXPECT_EQUAL("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}", dense->dump()); EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse1->dump()); EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse2->dump()); + EXPECT_EQUAL("tensor(x{}):{{x:\"a\"}:1,{x:\"b\"}:2,{x:\"c\"}:3}", sparse3->dump()); EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed1->dump()); EXPECT_EQUAL("tensor(x{},y[2]):{{x:\"a\",y:0}:1,{x:\"a\",y:1}:2}", mixed2->dump()); } @@ -955,6 +966,7 @@ TEST("require that convenient tensor create detects under-specified cells") { TEST("require that tensor peek can be parsed") { TEST_DO(verify_parse("t{x:\"1\",y:\"foo\"}", "f(t)(t{x:\"1\",y:\"foo\"})")); + TEST_DO(verify_parse("t{x:'1',y:'foo'}", "f(t)(t{x:\"1\",y:\"foo\"})")); TEST_DO(verify_parse("t{x:1,y:foo}", "f(t)(t{x:\"1\",y:\"foo\"})")); } diff --git a/eval/src/vespa/eval/eval/function.cpp b/eval/src/vespa/eval/eval/function.cpp index 108380f52b7..3d6e66acf04 100644 --- a/eval/src/vespa/eval/eval/function.cpp +++ b/eval/src/vespa/eval/eval/function.cpp @@ -363,9 +363,9 @@ int unhex(char c) { return -1; } -void extract_quoted_string(ParseContext &ctx, vespalib::string &str) { - ctx.eat('"'); - while (!ctx.eos() && ctx.get() != '"') { +void extract_quoted_string(ParseContext &ctx, vespalib::string &str, char quote) { + ctx.eat(quote); + while (!ctx.eos() && (ctx.get() != quote)) { if (ctx.get() == '\\') { ctx.next(); if (ctx.get() == 'x') { @@ -380,6 +380,7 @@ void extract_quoted_string(ParseContext &ctx, vespalib::string &str) { } else { switch(ctx.get()) { case '"': str.push_back('"'); break; + case '\'': str.push_back('\''); break; case '\\': str.push_back('\\'); break; case 'f': str.push_back('\f'); break; case 'n': str.push_back('\n'); break; @@ -393,12 +394,12 @@ void extract_quoted_string(ParseContext &ctx, vespalib::string &str) { } ctx.next(); } - ctx.eat('"'); + ctx.eat(quote); } -void parse_string(ParseContext &ctx) { +void parse_string(ParseContext &ctx, char quote) { vespalib::string &str = ctx.scratch(); - extract_quoted_string(ctx, str); + extract_quoted_string(ctx, str, quote); ctx.push_expression(Node_UP(new nodes::String(str))); } @@ -485,7 +486,9 @@ vespalib::string get_label(ParseContext &ctx) { ctx.skip_spaces(); vespalib::string label; if (ctx.get() == '"') { - extract_quoted_string(ctx, label); + extract_quoted_string(ctx, label, '"'); + } else if (ctx.get() == '\'') { + extract_quoted_string(ctx, label, '\''); } else { while (!is_label_end(ctx.get())) { label.push_back(ctx.get()); @@ -945,7 +948,9 @@ void parse_value(ParseContext &ctx) { parse_expression(ctx); ctx.eat(')'); } else if (ctx.get() == '"') { - parse_string(ctx); + parse_string(ctx, '"'); + } else if (ctx.get() == '\'') { + parse_string(ctx, '\''); } else if (isdigit(ctx.get())) { parse_number(ctx); } else { |