remove deprecated tensor model

fix and extend tests to also verify unstable cell types

7 files changed, 88 insertions, 218 deletions

diff --git a/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp b/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp
index 8d9ce8d5511..c67ac55e9f4 100644
--- a/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp
+++ b/document/src/tests/tensor_fieldvalue/partial_add/partial_add_test.cpp
@@ -1,7 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
-#include <vespa/eval/eval/test/tensor_model.h>
+#include <vespa/eval/eval/test/gen_spec.h>
 #include <vespa/eval/eval/value_codec.h>
 #include <vespa/document/update/tensor_partial_update.h>
 #include <vespa/vespalib/util/stringfmt.h>
@@ -15,14 +15,10 @@ using namespace vespalib::eval::test;
 
 using vespalib::make_string_short::fmt;
 
-std::vector<Layout> add_layouts = {
-    {x({"a"})},                           {x({"b"})},
-    {x({"a","b"})},                       {x({"a","c"})},
-    float_cells({x({"a","b"})}),          {x({"a","c"})},
-    {x({"a","b"})},                       float_cells({x({"a","c"})}),
-    float_cells({x({"a","b"})}),          float_cells({x({"a","c"})}),
-    {x({"a","b","c"}),y({"d","e"})},      {x({"b","f"}),y({"d","g"})},             
-    {x(3),y({"a","b"})},                  {x(3),y({"b","c"})}
+std::vector<std::pair<vespalib::string,vespalib::string>> add_layouts = {
+    {     "x4_1",     "x4_2" },
+    { "x4_2y4_1", "x4_1y4_2" },
+    {   "x3y4_1",   "x3y4_2" }
 };
 
 TensorSpec reference_add(const TensorSpec &a, const TensorSpec &b) {
@@ -37,7 +33,7 @@ TensorSpec reference_add(const TensorSpec &a, const TensorSpec &b) {
             result.add(cell.first, cell.second);
         }
     }
-    return result;
+    return result.normalize();
 }
 
 Value::UP try_partial_add(const TensorSpec &a, const TensorSpec &b) {
@@ -54,32 +50,34 @@ TensorSpec perform_partial_add(const TensorSpec &a, const TensorSpec &b) {
 }
 
 TEST(PartialAddTest, partial_add_works_for_simple_values) {
-    ASSERT_TRUE((add_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < add_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(add_layouts[i], N());
-        TensorSpec rhs = spec(add_layouts[i + 1], Div16(N()));
-        SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
-        auto expect = reference_add(lhs, rhs);
-        auto actual = perform_partial_add(lhs, rhs);
-        EXPECT_EQ(actual, expect);
+    for (const auto &layouts: add_layouts) {
+        for (auto lhs_ct: CellTypeUtils::list_types()) {
+            for (auto rhs_ct: CellTypeUtils::list_types()) {
+                TensorSpec lhs = GenSpec::from_desc(layouts.first).cells(lhs_ct).seq(N());
+                TensorSpec rhs = GenSpec::from_desc(layouts.second).cells(rhs_ct).seq(Div16(N()));
+                SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
+                auto expect = reference_add(lhs, rhs);
+                auto actual = perform_partial_add(lhs, rhs);
+                EXPECT_EQ(actual, expect);
+            }
+        }
     }
 }
 
-std::vector<Layout> bad_layouts = {
-    {x(3)},                               {x(3),y(1)},
-    {x(3),y(1)},                          {x(3)},
-    {x(3),y(3)},                          {x(3),y({"a"})},
-    {x(3),y({"a"})},                      {x(3),y(3)},
-    {x({"a"})},                           {x({"a"}),y({"b"})},
-    {x({"a"}),y({"b"})},                  {x({"a"})},
-    {x({"a"})},                           {x({"a"}),y(1)}
+std::vector<std::pair<vespalib::string,vespalib::string>> bad_layouts = {
+    {       "x3",     "x3y1" },
+    {     "x3y1",       "x3" },
+    {     "x3y3",   "x3y3_1" },
+    {   "x3y3_1",     "x3y3" },
+    {     "x3_1", "x3_1y3_1" },
+    { "x3_1y3_1",     "x3_1" },
+    {     "x3_1",   "x3_1y1" }
 };
 
 TEST(PartialAddTest, partial_add_returns_nullptr_on_invalid_inputs) {
-    ASSERT_TRUE((bad_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < bad_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(bad_layouts[i], N());
-        TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N()));
+    for (const auto &layouts: bad_layouts) {
+        TensorSpec lhs = GenSpec::from_desc(layouts.first).seq(N());
+        TensorSpec rhs = GenSpec::from_desc(layouts.second).seq(Div16(N()));
         SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
         auto actual = try_partial_add(lhs, rhs);
         auto expect = Value::UP();
diff --git a/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp b/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp
index 71f35102d17..26f7a10468a 100644
--- a/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp
+++ b/document/src/tests/tensor_fieldvalue/partial_modify/partial_modify_test.cpp
@@ -1,7 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
-#include <vespa/eval/eval/test/tensor_model.h>
+#include <vespa/eval/eval/test/gen_spec.h>
 #include <vespa/eval/eval/value_codec.h>
 #include <vespa/document/update/tensor_partial_update.h>
 #include <vespa/vespalib/util/stringfmt.h>
@@ -15,15 +15,13 @@ using namespace vespalib::eval::test;
 
 using vespalib::make_string_short::fmt;
 
-std::vector<Layout> modify_layouts = {
-    {x({"a"})},                           {x({"a"})},
-    {x({"a",""})},                        {x({"b","c","d","e"})},
-    {x(5)},                               {x({"1","2","foo","17"})},
-    {x({"a","b","c"}),y({"d","e"})},      {x({"b"}),y({"d"})},             
-    {x({"a","b","c"})},                   {x({"b","c","d"})},
-    {x(4),y({"a","b","c","d"}),z(5)},     {x({"1","2"}),y({"b","d"}),z({"1","3"})},
-    {x(3),y(2)},                          {x({"0","1"}),y({"0","1"})},
-    {x({"a","","b"})},                    {x({""})}
+std::vector<std::pair<vespalib::string,vespalib::string>> modify_layouts = {
+    {       "x4_1",         "x4_1" },
+    {       "x4_1",         "x4_2" },
+    {         "x4",         "x4_2" },
+    {   "x4_1y4_2",     "x4_2y4_1" },
+    {   "x4y4_1z4", "x4_2y4_2z4_2" },
+    {       "x3y2",     "x2_1y2_1" }
 };
 
 TensorSpec::Address sparsify(const TensorSpec::Address &input) {
@@ -52,7 +50,7 @@ TensorSpec reference_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t
             result.add(cell.first, fun(v, iter->second));
         }
     }
-    return result;
+    return result.normalize();
 }
 
 Value::UP try_partial_modify(const TensorSpec &a, const TensorSpec &b, join_fun_t fun) {
@@ -69,36 +67,38 @@ TensorSpec perform_partial_modify(const TensorSpec &a, const TensorSpec &b, join
 }
 
 TEST(PartialModifyTest, partial_modify_works_for_simple_values) {
-    ASSERT_TRUE((modify_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < modify_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(modify_layouts[i], N());
-        TensorSpec rhs = spec(modify_layouts[i + 1], Div16(N()));
-        SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
-        for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f}) {
-            auto expect = reference_modify(lhs, rhs, fun);
-            auto actual = perform_partial_modify(lhs, rhs, fun);
-            EXPECT_EQ(actual, expect);
+    for (const auto &layouts: modify_layouts) {
+        for (auto lhs_ct: CellTypeUtils::list_types()) {
+            for (auto rhs_ct: CellTypeUtils::list_types()) {
+                TensorSpec lhs = GenSpec::from_desc(layouts.first).cells(lhs_ct).seq(N());
+                TensorSpec rhs = GenSpec::from_desc(layouts.second).cells(rhs_ct).seq(Div16(N()));
+                SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
+                for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f}) {
+                    auto expect = reference_modify(lhs, rhs, fun);
+                    auto actual = perform_partial_modify(lhs, rhs, fun);
+                    EXPECT_EQ(actual, expect);
+                }
+                auto fun = [](double, double keep) { return keep; };
+                auto expect = reference_modify(lhs, rhs, fun);
+                auto actual = perform_partial_modify(lhs, rhs, fun);
+                EXPECT_EQ(actual, expect);
+            }
         }
-        auto fun = [](double, double keep) { return keep; };
-        auto expect = reference_modify(lhs, rhs, fun);
-        auto actual = perform_partial_modify(lhs, rhs, fun);
-        EXPECT_EQ(actual, expect);
     }
 }
 
-std::vector<Layout> bad_layouts = {
-    {x(3)},                               {x(3)},
-    {x(3),y({"a"})},                      {x(3),y({"a"})},
-    {x({"a"})},                           {x({"a"}),y({"b"})},
-    {x({"a"}),y({"b"})},                  {x({"a"})},
-    {x({"a"})},                           {x({"a"}),y(1)}
+std::vector<std::pair<vespalib::string,vespalib::string>> bad_layouts = {
+    {       "x3",       "x3" },
+    {   "x3y4_1",   "x3y4_1" },
+    {     "x4_1", "x4_1y4_1" },
+    { "x4_1y4_1",     "x4_1" },
+    {     "x4_1",   "x4_1y1" }
 };
 
 TEST(PartialModifyTest, partial_modify_returns_nullptr_on_invalid_inputs) {
-    ASSERT_TRUE((bad_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < bad_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(bad_layouts[i], N());
-        TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N()));
+    for (const auto &layouts: bad_layouts) {
+        TensorSpec lhs = GenSpec::from_desc(layouts.first).seq(N());
+        TensorSpec rhs = GenSpec::from_desc(layouts.second).seq(Div16(N()));
         SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
         for (auto fun: {operation::Add::f}) {
             auto actual = try_partial_modify(lhs, rhs, fun);
diff --git a/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp b/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp
index 58a06273139..9107f590125 100644
--- a/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp
+++ b/document/src/tests/tensor_fieldvalue/partial_remove/partial_remove_test.cpp
@@ -1,7 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
-#include <vespa/eval/eval/test/tensor_model.h>
+#include <vespa/eval/eval/test/gen_spec.h>
 #include <vespa/eval/eval/value_codec.h>
 #include <vespa/document/update/tensor_partial_update.h>
 #include <vespa/vespalib/util/stringfmt.h>
@@ -15,15 +15,10 @@ using namespace vespalib::eval::test;
 
 using vespalib::make_string_short::fmt;
 
-std::vector<Layout> remove_layouts = {
-    {x({"a"})},                           {x({"b"})},
-    {x({"a","b"})},                       {x({"a","c"})},
-    {x({"a","b"})},                       {x({"a","b"})},
-    float_cells({x({"a","b"})}),          {x({"a","c"})},
-    {x({"a","b"})},                       float_cells({x({"a","c"})}),
-    float_cells({x({"a","b"})}),          float_cells({x({"a","c"})}),
-    {x({"a","b","c"}),y({"d","e"})},      {x({"b","f"}),y({"d","g"})},             
-    {x(3),y({"a","b"})},                  {y({"b","c"})}
+std::vector<std::pair<vespalib::string,vespalib::string>> remove_layouts = {
+    {     "x4_1",     "x4_2" },
+    { "x4_2y4_1", "x4_1y4_2" },
+    {   "x3y4_1",     "y4_2" }
 };
 
 TensorSpec::Address only_sparse(const TensorSpec::Address &input) {
@@ -45,7 +40,7 @@ TensorSpec reference_remove(const TensorSpec &a, const TensorSpec &b) {
             result.add(cell.first, cell.second);
         }
     }
-    return result;
+    return result.normalize();
 }
 
 Value::UP try_partial_remove(const TensorSpec &a, const TensorSpec &b) {
@@ -62,30 +57,32 @@ TensorSpec perform_partial_remove(const TensorSpec &a, const TensorSpec &b) {
 }
 
 TEST(PartialRemoveTest, partial_remove_works_for_simple_values) {
-    ASSERT_TRUE((remove_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < remove_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(remove_layouts[i], N());
-        TensorSpec rhs = spec(remove_layouts[i + 1], Div16(N()));
-        SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
-        auto expect = reference_remove(lhs, rhs);
-        auto actual = perform_partial_remove(lhs, rhs);
-        EXPECT_EQ(actual, expect);
+    for (const auto &layouts: remove_layouts) {
+        for (auto lhs_ct: CellTypeUtils::list_types()) {
+            for (auto rhs_ct: CellTypeUtils::list_types()) {
+                TensorSpec lhs = GenSpec::from_desc(layouts.first).cells(lhs_ct).seq(N());
+                TensorSpec rhs = GenSpec::from_desc(layouts.second).cells(rhs_ct).seq(Div16(N()));
+                SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
+                auto expect = reference_remove(lhs, rhs);
+                auto actual = perform_partial_remove(lhs, rhs);
+                EXPECT_EQ(actual, expect);
+            }
+        }
     }
 }
 
-std::vector<Layout> bad_layouts = {
-    {x(3)},                               {x(3)},
-    {x(3),y({"a"})},                      {x(3)},
-    {x(3),y({"a"})},                      {x(3),y({"a"})},
-    {x({"a"})},                           {y({"a"})},
-    {x({"a"})},                           {x({"a"}),y({"b"})}
+std::vector<std::pair<vespalib::string,vespalib::string>> bad_layouts = {
+    {      "x3",       "x3" },
+    {  "x3y4_1",       "x3" },
+    {  "x3y4_1",   "x3y4_2" },
+    {    "x4_1",     "y4_1" },
+    {    "x4_1", "x4_2y4_1" }
 };
 
 TEST(PartialRemoveTest, partial_remove_returns_nullptr_on_invalid_inputs) {
-    ASSERT_TRUE((bad_layouts.size() % 2) == 0);
-    for (size_t i = 0; i < bad_layouts.size(); i += 2) {
-        TensorSpec lhs = spec(bad_layouts[i], N());
-        TensorSpec rhs = spec(bad_layouts[i + 1], Div16(N()));
+    for (const auto &layouts: bad_layouts) {
+        TensorSpec lhs = GenSpec::from_desc(layouts.first).seq(N());
+        TensorSpec rhs = GenSpec::from_desc(layouts.second).seq(Div16(N()));
         SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
         auto actual = try_partial_remove(lhs, rhs);
         auto expect = Value::UP();
diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp
index 7c577ef0086..11e602dfeb3 100644
--- a/eval/src/apps/tensor_conformance/generate.cpp
+++ b/eval/src/apps/tensor_conformance/generate.cpp
@@ -3,7 +3,6 @@
 #include "generate.h"
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/eval/eval/test/gen_spec.h>
-#include <vespa/eval/eval/test/tensor_model.h>
 #include <vespa/eval/eval/value_type_spec.h>
 #include <vespa/eval/eval/aggr.h>
 #include <vespa/vespalib/util/stringfmt.h>
diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt
index de628244523..aeccc00a01c 100644
--- a/eval/src/vespa/eval/eval/test/CMakeLists.txt
+++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt
@@ -7,7 +7,6 @@ vespa_add_library(eval_eval_test OBJECT
     gen_spec.cpp
     reference_evaluation.cpp
     reference_operations.cpp
-    tensor_model.cpp
     test_io.cpp
     value_compare.cpp
 )
diff --git a/eval/src/vespa/eval/eval/test/tensor_model.cpp b/eval/src/vespa/eval/eval/test/tensor_model.cpp
deleted file mode 100644
index b7ad4d0071f..00000000000
--- a/eval/src/vespa/eval/eval/test/tensor_model.cpp
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "tensor_model.h"
-#include <vespa/eval/eval/value_type.h>
-
-namespace vespalib::eval::test {
-
-Layout float_cells(const Layout &layout) {
-    return Layout(CellType::FLOAT, layout.domains);
-}
-
-Domain x() { return Domain("x", {}); }
-Domain x(size_t size) { return Domain("x", size); }
-Domain x(const std::vector<vespalib::string> &keys) { return Domain("x", keys); }
-
-Domain y() { return Domain("y", {}); }
-Domain y(size_t size) { return Domain("y", size); }
-Domain y(const std::vector<vespalib::string> &keys) { return Domain("y", keys); }
-
-Domain z() { return Domain("z", {}); }
-Domain z(size_t size) { return Domain("z", size); }
-Domain z(const std::vector<vespalib::string> &keys) { return Domain("z", keys); }
-
-vespalib::string infer_type(const Layout &layout) {
-    return GenSpec(layout.domains).cells(layout.cell_type).type().to_spec();
-}
-
-TensorSpec spec(const Layout &layout, const Sequence &seq) {
-    return GenSpec(layout.domains).cells(layout.cell_type).seq(seq);
-}
-TensorSpec spec(const Domain &domain, const Sequence &seq) {
-    return spec(Layout({domain}), seq);
-}
-TensorSpec spec(double value) {
-    return GenSpec(value);
-}
-
-TensorSpec spec(const vespalib::string &type,
-                const std::vector<std::pair<TensorSpec::Address, TensorSpec::Value>> &cells) {
-    TensorSpec spec("tensor(" + type + ")");
-
-    for (const auto &cell : cells) {
-        spec.add(cell.first, cell.second);
-    }
-    return spec;
-}
-
-TensorSpec spec(const vespalib::string &value_expr) {
-    return TensorSpec::from_expr(value_expr);
-}
-
-}
diff --git a/eval/src/vespa/eval/eval/test/tensor_model.h b/eval/src/vespa/eval/eval/test/tensor_model.h
deleted file mode 100644
index 85f6e5ccfa4..00000000000
--- a/eval/src/vespa/eval/eval/test/tensor_model.h
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "gen_spec.h"
-
-namespace vespalib::eval::test {
-
-// custom op1
-struct MyOp {
-    static double f(double a) {
-        return ((a + 1) * 2);
-    }
-};
-
-// 'a in [1,5,7,13,42]'
-struct MyIn {
-    static double f(double a) {
-        if ((a ==  1) ||
-            (a ==  5) ||
-            (a ==  7) ||
-            (a == 13) ||
-            (a == 42))
-        {
-            return 1.0;
-        } else {
-            return 0.0;
-        }
-    }
-};
-
-using Domain = DimSpec;
-
-struct Layout {
-    CellType cell_type;
-    std::vector<Domain> domains;
-    Layout(std::initializer_list<Domain> domains_in)
-        : cell_type(CellType::DOUBLE), domains(domains_in) {}
-    Layout(CellType cell_type_in, std::vector<Domain> domains_in)
-        : cell_type(cell_type_in), domains(std::move(domains_in)) {}
-    auto begin() const { return domains.begin(); }
-    auto end() const { return domains.end(); }
-    auto size() const { return domains.size(); }
-    auto operator[](size_t idx) const { return domains[idx]; }
-};
-
-Layout float_cells(const Layout &layout);
-
-Domain x();
-Domain x(size_t size);
-Domain x(const std::vector<vespalib::string> &keys);
-
-Domain y();
-Domain y(size_t size);
-Domain y(const std::vector<vespalib::string> &keys);
-
-Domain z();
-Domain z(size_t size);
-Domain z(const std::vector<vespalib::string> &keys);
-
-// Infer the tensor type implied by the given layout
-vespalib::string infer_type(const Layout &layout);
-
-TensorSpec spec(const Layout &layout, const Sequence &seq);
-TensorSpec spec(const Domain &domain, const Sequence &seq);
-TensorSpec spec(double value);
-TensorSpec spec(const vespalib::string &type,
-                const std::vector<std::pair<TensorSpec::Address, TensorSpec::Value>> &cells);
-TensorSpec spec(const vespalib::string &value_expr);
-
-}