From 26ea003c227e65b929e80b9ac412ca7d7c2a5404 Mon Sep 17 00:00:00 2001
From: Håvard Pettersen <havardpe@oath.com>
Date: Fri, 5 Apr 2019 10:15:09 +0000
Subject: parse (and ignore for now) tensor cell type in tensor type spec

example: tensor<float>(x[10])
---
 eval/src/tests/eval/value_type/value_type_test.cpp |  5 +++++
 eval/src/vespa/eval/eval/value_type_spec.cpp       | 21 +++++++++++++++++++--
 2 files changed, 24 insertions(+), 2 deletions(-)

(limited to 'eval/src')

diff --git a/eval/src/tests/eval/value_type/value_type_test.cpp b/eval/src/tests/eval/value_type/value_type_test.cpp
index ffdc601932e..f7db7816fad 100644
--- a/eval/src/tests/eval/value_type/value_type_test.cpp
+++ b/eval/src/tests/eval/value_type/value_type_test.cpp
@@ -268,6 +268,8 @@ TEST("require that value type spec can be parsed") {
     EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor(y[10])"));
     EXPECT_EQUAL(ValueType::tensor_type({{"z", 0}}), ValueType::from_spec("tensor(z[])"));
     EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y", 10}, {"z", 0}}), ValueType::from_spec("tensor(x{},y[10],z[])"));
+    EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor<double>(y[10])"));
+    EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec("tensor<float>(y[10])"));
 }
 
 TEST("require that value type spec can be parsed with extra whitespace") {
@@ -280,6 +282,8 @@ TEST("require that value type spec can be parsed with extra whitespace") {
     EXPECT_EQUAL(ValueType::tensor_type({{"z", 0}}), ValueType::from_spec(" tensor ( z [ ] ) "));
     EXPECT_EQUAL(ValueType::tensor_type({{"x"}, {"y", 10}, {"z", 0}}),
                  ValueType::from_spec(" tensor ( x { } , y [ 10 ] , z [ ] ) "));
+    EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec(" tensor < double > ( y [ 10 ] ) "));
+    EXPECT_EQUAL(ValueType::tensor_type({{"y", 10}}), ValueType::from_spec(" tensor < float > ( y [ 10 ] ) "));
 }
 
 TEST("require that malformed value type spec is parsed as error") {
@@ -300,6 +304,7 @@ TEST("require that malformed value type spec is parsed as error") {
     EXPECT_TRUE(ValueType::from_spec("tensor(x{},x{})").is_error());
     EXPECT_TRUE(ValueType::from_spec("tensor(x{},x[10])").is_error());
     EXPECT_TRUE(ValueType::from_spec("tensor(x{},x[])").is_error());
+    EXPECT_TRUE(ValueType::from_spec("tensor<float16>(x[10])").is_error());
 }
 
 struct ParseResult {
diff --git a/eval/src/vespa/eval/eval/value_type_spec.cpp b/eval/src/vespa/eval/eval/value_type_spec.cpp
index 229a9201f08..9bd0e54882f 100644
--- a/eval/src/vespa/eval/eval/value_type_spec.cpp
+++ b/eval/src/vespa/eval/eval/value_type_spec.cpp
@@ -77,6 +77,7 @@ vespalib::string parse_ident(ParseContext &ctx) {
 }
 
 size_t parse_int(ParseContext &ctx) {
+    ctx.skip_spaces();
     vespalib::string num;
     for (; isdigit(ctx.get()); ctx.next()) {
         num.push_back(ctx.get());
@@ -91,11 +92,11 @@ ValueType::Dimension parse_dimension(ParseContext &ctx) {
     ValueType::Dimension dimension(parse_ident(ctx));
     ctx.skip_spaces();
     if (ctx.get() == '{') {
-        ctx.next(); // '{'
+        ctx.eat('{');
         ctx.skip_spaces();
         ctx.eat('}');
     } else if (ctx.get() == '[') {
-        ctx.next(); // '['
+        ctx.eat('[');
         ctx.skip_spaces();
         if (ctx.get() == ']') {
             dimension.size = 0;
@@ -129,6 +130,18 @@ std::vector<ValueType::Dimension> parse_dimension_list(ParseContext &ctx) {
     return list;
 }
 
+vespalib::string parse_cell_type(ParseContext &ctx) {
+    vespalib::string cell_type = "double";
+    ctx.skip_spaces();
+    if (ctx.get() == '<') {
+        ctx.eat('<');
+        cell_type = parse_ident(ctx);
+        ctx.skip_spaces();
+        ctx.eat('>');
+    }
+    return cell_type;
+}
+
 } // namespace vespalib::eval::value_type::<anonymous>
 
 ValueType
@@ -143,6 +156,10 @@ parse_spec(const char *pos_in, const char *end_in, const char *&pos_out)
     } else if (type_name == "double") {
         return ValueType::double_type();
     } else if (type_name == "tensor") {
+        vespalib::string cell_type = parse_cell_type(ctx);
+        if ((cell_type != "double") && (cell_type != "float")) {
+            ctx.fail();
+        }
         std::vector<ValueType::Dimension> list = parse_dimension_list(ctx);
         if (!ctx.failed()) {
             return ValueType::tensor_type(std::move(list));
-- 
cgit v1.2.3