remove deprecated tensor model

fix and extend tests to also verify unstable cell types

3 files changed, 88 insertions, 93 deletions

diff --git a/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp b/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp
index 8d9ce8d5511..c67ac55e9f4 100644
--- a/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp
+++ b/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp
@@ -1,7 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
-#include <vespa/eval/eval/test/tensor_model.h>
+#include <vespa/eval/eval/test/gen_spec.h>
 #include <vespa/eval/eval/value_codec.h>
 #include <vespa/document/update/tensor_partial_update.h>
 #include <vespa/vespalib/util/stringfmt.h>
@@ -15,14 +15,10 @@ using namespace vespalib::eval::test;
 
 using vespalib::make_string_short::fmt;
 
-std::vector<Layout> add_layouts = {
-    {x({"a"})},                           {x({"b"})},
-    {x({"a","b"})},                       {x({"a","c"})},
-    float_cells({x({"a","b"})}),          {x({"a","c"})},
-    {x({"a","b"})},                       float_cells({x({"a","c"})}),
-    float_cells({x({"a","b"})}),          float_cells({x({"a","c"})}),
-    {x({"a","b","c"}),y({"d","e"})},      {x({"b","f"}),y({"d","g"})},             
-    {x(3),y({"a","b"})},                  {x(3),y({"b","c"})}
+std::vector<std::pair<vespalib::string,vespalib::string>> add_layouts = {
+    {     "x4_1",     "x4_2" },
+    { "x4_2y4_1", "x4_1y4_2" },
+    {   "x3y4_1",   "x3y4_2" }
 };
 
 TensorSpec reference_add(const TensorSpec &a, const TensorSpec &b) {
@@ -37,7 +33,7 @@ TensorSpec reference_add(const TensorSpec &a, const TensorSpec &b) {
             result.add(cell.first, cell.second);
         }
     }
-    return result;
+    return result.normalize();
 }
 
 Value::UP try_partial_add(const TensorSpec &a, const TensorSpec &b) {
@@ -54,32 +50,34 @@ TensorSpec perform_partial_add(const TensorSpec &a, const TensorSpec &b) {
 }
 
 TEST(PartialAddTest, partial_add_works_for_simple_values) {
-    ASSERT_TRUE((add_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < add_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(add_layouts[i], N());
-        TensorSpec rhs = spec(add_layouts[i + 1], Div16(N()));
-        SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
-        auto expect = reference_add(lhs, rhs);
-        auto actual = perform_partial_add(lhs, rhs);
-        EXPECT_EQ(actual, expect);
+    for (const auto &layouts: add_layouts) {
+        for (auto lhs_ct: CellTypeUtils::list_types()) {
+            for (auto rhs_ct: CellTypeUtils::list_types()) {
+                TensorSpec lhs = GenSpec::from_desc(layouts.first).cells(lhs_ct).seq(N());
+                TensorSpec rhs = GenSpec::from_desc(layouts.second).cells(rhs_ct).seq(Div16(N()));
+                SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
+                auto expect = reference_add(lhs, rhs);
+                auto actual = perform_partial_add(lhs, rhs);
+                EXPECT_EQ(actual, expect);
+            }
+        }
     }
 }
 
-std::vector<Layout> bad_layouts = {
-    {x(3)},                               {x(3),y(1)},
-    {x(3),y(1)},                          {x(3)},
-    {x(3),y(3)},                          {x(3),y({"a"})},
-    {x(3),y({"a"})},                      {x(3),y(3)},
-    {x({"a"})},                           {x({"a"}),y({"b"})},
-    {x({"a"}),y({"b"})},                  {x({"a"})},
-    {x({"a"})},                           {x({"a"}),y(1)}
+std::vector<std::pair<vespalib::string,vespalib::string>> bad_layouts = {
+    {       "x3",     "x3y1" },
+    {     "x3y1",       "x3" },
+    {     "x3y3",   "x3y3_1" },
+    {   "x3y3_1",     "x3y3" },
+    {     "x3_1", "x3_1y3_1" },
+    { "x3_1y3_1",     "x3_1" },
+    {     "x3_1",   "x3_1y1" }
 };
 
 TEST(PartialAddTest, partial_add_returns_nullptr_on_invalid_inputs) {
-    ASSERT_TRUE((bad_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < bad_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(bad_layouts[i], N());
-        TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N()));
+    for (const auto &layouts: bad_layouts) {
+        TensorSpec lhs = GenSpec::from_desc(layouts.first).seq(N());
+        TensorSpec rhs = GenSpec::from_desc(layouts.second).seq(Div16(N()));
         SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
         auto actual = try_partial_add(lhs, rhs);
         auto expect = Value::UP();
diff --git a/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp b/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp
index 71f35102d17..26f7a10468a 100644
--- a/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp
+++ b/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp
@@ -1,7 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
-#include <vespa/eval/eval/test/tensor_model.h>
+#include <vespa/eval/eval/test/gen_spec.h>
 #include <vespa/eval/eval/value_codec.h>
 #include <vespa/document/update/tensor_partial_update.h>
 #include <vespa/vespalib/util/stringfmt.h>
@@ -15,15 +15,13 @@ using namespace vespalib::eval::test;
 
 using vespalib::make_string_short::fmt;
 
-std::vector<Layout> modify_layouts = {
-    {x({"a"})},                           {x({"a"})},
-    {x({"a",""})},                        {x({"b","c","d","e"})},
-    {x(5)},                               {x({"1","2","foo","17"})},
-    {x({"a","b","c"}),y({"d","e"})},      {x({"b"}),y({"d"})},             
-    {x({"a","b","c"})},                   {x({"b","c","d"})},
-    {x(4),y({"a","b","c","d"}),z(5)},     {x({"1","2"}),y({"b","d"}),z({"1","3"})},
-    {x(3),y(2)},                          {x({"0","1"}),y({"0","1"})},
-    {x({"a","","b"})},                    {x({""})}
+std::vector<std::pair<vespalib::string,vespalib::string>> modify_layouts = {
+    {       "x4_1",         "x4_1" },
+    {       "x4_1",         "x4_2" },
+    {         "x4",         "x4_2" },
+    {   "x4_1y4_2",     "x4_2y4_1" },
+    {   "x4y4_1z4", "x4_2y4_2z4_2" },
+    {       "x3y2",     "x2_1y2_1" }
 };
 
 TensorSpec::Address sparsify(const TensorSpec::Address &input) {
@@ -52,7 +50,7 @@ TensorSpec reference_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t
             result.add(cell.first, fun(v, iter->second));
         }
     }
-    return result;
+    return result.normalize();
 }
 
 Value::UP try_partial_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) {
@@ -69,36 +67,38 @@ TensorSpec perform_partial_modify(const TensorSpec &a, const TensorSpec &b, join
 }
 
 TEST(PartialModifyTest, partial_modify_works_for_simple_values) {
-    ASSERT_TRUE((modify_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < modify_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(modify_layouts[i], N());
-        TensorSpec rhs = spec(modify_layouts[i + 1], Div16(N()));
-        SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
-        for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f}) {
-            auto expect = reference_modify(lhs, rhs, fun);
-            auto actual = perform_partial_modify(lhs, rhs, fun);
-            EXPECT_EQ(actual, expect);
+    for (const auto &layouts: modify_layouts) {
+        for (auto lhs_ct: CellTypeUtils::list_types()) {
+            for (auto rhs_ct: CellTypeUtils::list_types()) {
+                TensorSpec lhs = GenSpec::from_desc(layouts.first).cells(lhs_ct).seq(N());
+                TensorSpec rhs = GenSpec::from_desc(layouts.second).cells(rhs_ct).seq(Div16(N()));
+                SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
+                for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f}) {
+                    auto expect = reference_modify(lhs, rhs, fun);
+                    auto actual = perform_partial_modify(lhs, rhs, fun);
+                    EXPECT_EQ(actual, expect);
+                }
+                auto fun = [](double, double keep) { return keep; };
+                auto expect = reference_modify(lhs, rhs, fun);
+                auto actual = perform_partial_modify(lhs, rhs, fun);
+                EXPECT_EQ(actual, expect);
+            }
         }
-        auto fun = [](double, double keep) { return keep; };
-        auto expect = reference_modify(lhs, rhs, fun);
-        auto actual = perform_partial_modify(lhs, rhs, fun);
-        EXPECT_EQ(actual, expect);
     }
 }
 
-std::vector<Layout> bad_layouts = {
-    {x(3)},                               {x(3)},
-    {x(3),y({"a"})},                      {x(3),y({"a"})},
-    {x({"a"})},                           {x({"a"}),y({"b"})},
-    {x({"a"}),y({"b"})},                  {x({"a"})},
-    {x({"a"})},                           {x({"a"}),y(1)}
+std::vector<std::pair<vespalib::string,vespalib::string>> bad_layouts = {
+    {       "x3",       "x3" },
+    {   "x3y4_1",   "x3y4_1" },
+    {     "x4_1", "x4_1y4_1" },
+    { "x4_1y4_1",     "x4_1" },
+    {     "x4_1",   "x4_1y1" }
 };
 
 TEST(PartialModifyTest, partial_modify_returns_nullptr_on_invalid_inputs) {
-    ASSERT_TRUE((bad_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < bad_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(bad_layouts[i], N());
-        TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N()));
+    for (const auto &layouts: bad_layouts) {
+        TensorSpec lhs = GenSpec::from_desc(layouts.first).seq(N());
+        TensorSpec rhs = GenSpec::from_desc(layouts.second).seq(Div16(N()));
         SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
         for (auto fun: {operation::Add::f}) {
             auto actual = try_partial_modify(lhs, rhs, fun);
diff --git a/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp b/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp
index 58a06273139..9107f590125 100644
--- a/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp
+++ b/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp
@@ -1,7 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
-#include <vespa/eval/eval/test/tensor_model.h>
+#include <vespa/eval/eval/test/gen_spec.h>
 #include <vespa/eval/eval/value_codec.h>
 #include <vespa/document/update/tensor_partial_update.h>
 #include <vespa/vespalib/util/stringfmt.h>
@@ -15,15 +15,10 @@ using namespace vespalib::eval::test;
 
 using vespalib::make_string_short::fmt;
 
-std::vector<Layout> remove_layouts = {
-    {x({"a"})},                           {x({"b"})},
-    {x({"a","b"})},                       {x({"a","c"})},
-    {x({"a","b"})},                       {x({"a","b"})},
-    float_cells({x({"a","b"})}),          {x({"a","c"})},
-    {x({"a","b"})},                       float_cells({x({"a","c"})}),
-    float_cells({x({"a","b"})}),          float_cells({x({"a","c"})}),
-    {x({"a","b","c"}),y({"d","e"})},      {x({"b","f"}),y({"d","g"})},             
-    {x(3),y({"a","b"})},                  {y({"b","c"})}
+std::vector<std::pair<vespalib::string,vespalib::string>> remove_layouts = {
+    {     "x4_1",     "x4_2" },
+    { "x4_2y4_1", "x4_1y4_2" },
+    {   "x3y4_1",     "y4_2" }
 };
 
 TensorSpec::Address only_sparse(const TensorSpec::Address &input) {
@@ -45,7 +40,7 @@ TensorSpec reference_remove(const TensorSpec &a, const TensorSpec &b) {
             result.add(cell.first, cell.second);
         }
     }
-    return result;
+    return result.normalize();
 }
 
 Value::UP try_partial_remove(const TensorSpec &a, const TensorSpec &b) {
@@ -62,30 +57,32 @@ TensorSpec perform_partial_remove(const TensorSpec &a, const TensorSpec &b) {
 }
 
 TEST(PartialRemoveTest, partial_remove_works_for_simple_values) {
-    ASSERT_TRUE((remove_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < remove_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(remove_layouts[i], N());
-        TensorSpec rhs = spec(remove_layouts[i + 1], Div16(N()));
-        SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
-        auto expect = reference_remove(lhs, rhs);
-        auto actual = perform_partial_remove(lhs, rhs);
-        EXPECT_EQ(actual, expect);
+    for (const auto &layouts: remove_layouts) {
+        for (auto lhs_ct: CellTypeUtils::list_types()) {
+            for (auto rhs_ct: CellTypeUtils::list_types()) {
+                TensorSpec lhs = GenSpec::from_desc(layouts.first).cells(lhs_ct).seq(N());
+                TensorSpec rhs = GenSpec::from_desc(layouts.second).cells(rhs_ct).seq(Div16(N()));
+                SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
+                auto expect = reference_remove(lhs, rhs);
+                auto actual = perform_partial_remove(lhs, rhs);
+                EXPECT_EQ(actual, expect);
+            }
+        }
     }
 }
 
-std::vector<Layout> bad_layouts = {
-    {x(3)},                               {x(3)},
-    {x(3),y({"a"})},                      {x(3)},
-    {x(3),y({"a"})},                      {x(3),y({"a"})},
-    {x({"a"})},                           {y({"a"})},
-    {x({"a"})},                           {x({"a"}),y({"b"})}
+std::vector<std::pair<vespalib::string,vespalib::string>> bad_layouts = {
+    {      "x3",       "x3" },
+    {  "x3y4_1",       "x3" },
+    {  "x3y4_1",   "x3y4_2" },
+    {    "x4_1",     "y4_1" },
+    {    "x4_1", "x4_2y4_1" }
 };
 
 TEST(PartialRemoveTest, partial_remove_returns_nullptr_on_invalid_inputs) {
-    ASSERT_TRUE((bad_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < bad_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(bad_layouts[i], N());
-        TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N()));
+    for (const auto &layouts: bad_layouts) {
+        TensorSpec lhs = GenSpec::from_desc(layouts.first).seq(N());
+        TensorSpec rhs = GenSpec::from_desc(layouts.second).seq(Div16(N()));
         SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
         auto actual = try_partial_remove(lhs, rhs);
         auto expect = Value::UP();