diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-04-08 15:24:48 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-04-08 15:24:48 +0200 |
commit | 6935cdc74fe60a94b26091413f46218c9833f0be (patch) | |
tree | 25a37dfafbd7d61f83ad48111a36905e5cadc9dd /eval | |
parent | ceee063299f3a10b37975152a0859afd2d5072a0 (diff) | |
parent | b84078e4bcb7bad171f42d40b7ee5337ded3ed35 (diff) |
Merge pull request #9029 from vespa-engine/havardpe/parse-and-ignore-tensor-cell-type
parse (and ignore for now) tensor cell type in tensor type spec
Diffstat (limited to 'eval')
-rw-r--r-- | eval/src/tests/eval/value_type/value_type_test.cpp | 5 | ||||
-rw-r--r-- | eval/src/vespa/eval/eval/value_type_spec.cpp | 39 |
2 files changed, 42 insertions, 2 deletions
diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp index ffdc601932e..f7db7816fad 100644 --- a/eval/src/tests/eval/value_type/value_type_test.cpp +++ b/eval/src/tests/eval/value_type/value_type_test.cpp @@ -268,6 +268,8 @@ TEST("require that value type spec can be parsed") { EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor(y[10])")); EXPECT_EQUAL(ValueType::tensor_type({{"z", 0}}), ValueType::from_spec("tensor(z[])")); EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y", 10}, {"z", 0}}), ValueType::from_spec("tensor(x{},y[10],z[])")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor<double>(y[10])")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor<float>(y[10])")); } TEST("require that value type spec can be parsed with extra whitespace") { @@ -280,6 +282,8 @@ TEST("require that value type spec can be parsed with extra whitespace") { EXPECT_EQUAL(ValueType::tensor_type({{"z", 0}}), ValueType::from_spec(" tensor ( z [ ] ) ")); EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y", 10}, {"z", 0}}), ValueType::from_spec(" tensor ( x { } , y [ 10 ] , z [ ] ) ")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec(" tensor < double > ( y [ 10 ] ) ")); + EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec(" tensor < float > ( y [ 10 ] ) ")); } TEST("require that malformed value type spec is parsed as error") { @@ -300,6 +304,7 @@ TEST("require that malformed value type spec is parsed as error") { EXPECT_TRUE(ValueType::from_spec("tensor(x{},x{})").is_error()); EXPECT_TRUE(ValueType::from_spec("tensor(x{},x[10])").is_error()); EXPECT_TRUE(ValueType::from_spec("tensor(x{},x[])").is_error()); + EXPECT_TRUE(ValueType::from_spec("tensor<float16>(x[10])").is_error()); } struct ParseResult { diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp index 229a9201f08..6076988ad50 100644 --- a/eval/src/vespa/eval/eval/value_type_spec.cpp +++ b/eval/src/vespa/eval/eval/value_type_spec.cpp @@ -12,6 +12,14 @@ namespace { class ParseContext { +public: + struct Mark { + const char *pos; + char curr; + bool failed; + Mark(const char *pos_in, char curr_in, bool failed_in) + : pos(pos_in), curr(curr_in), failed(failed_in) {} + }; private: const char *_pos; const char *_end; @@ -34,6 +42,14 @@ public: _pos_after = nullptr; } } + Mark mark() const { + return Mark(_pos, _curr, _failed); + } + void revert(Mark mark) { + _pos = mark.pos; + _curr = mark.curr; + _failed = mark.failed; + } void fail() { _failed = true; _curr = 0; @@ -77,6 +93,7 @@ vespalib::string parse_ident(ParseContext &ctx) { } size_t parse_int(ParseContext &ctx) { + ctx.skip_spaces(); vespalib::string num; for (; isdigit(ctx.get()); ctx.next()) { num.push_back(ctx.get()); @@ -91,11 +108,11 @@ ValueType::Dimension parse_dimension(ParseContext &ctx) { ValueType::Dimension dimension(parse_ident(ctx)); ctx.skip_spaces(); if (ctx.get() == '{') { - ctx.next(); // '{' + ctx.eat('{'); ctx.skip_spaces(); ctx.eat('}'); } else if (ctx.get() == '[') { - ctx.next(); // '[' + ctx.eat('['); ctx.skip_spaces(); if (ctx.get() == ']') { dimension.size = 0; @@ -129,6 +146,20 @@ std::vector<ValueType::Dimension> parse_dimension_list(ParseContext &ctx) { return list; } +vespalib::string parse_cell_type(ParseContext &ctx) { + auto mark = ctx.mark(); + ctx.skip_spaces(); + ctx.eat('<'); + auto cell_type = parse_ident(ctx); + ctx.skip_spaces(); + ctx.eat('>'); + if (ctx.failed()) { + ctx.revert(mark); + cell_type = "double"; + } + return cell_type; +} + } // namespace vespalib::eval::value_type::<anonymous> ValueType @@ -143,6 +174,10 @@ parse_spec(const char *pos_in, const char *end_in, const char *&pos_out) } else if (type_name == "double") { return ValueType::double_type(); } else if (type_name == "tensor") { + vespalib::string cell_type = parse_cell_type(ctx); + if ((cell_type != "double") && (cell_type != "float")) { + ctx.fail(); + } std::vector<ValueType::Dimension> list = parse_dimension_list(ctx); if (!ctx.failed()) { return ValueType::tensor_type(std::move(list)); |