add reference implementation of concat, with test

5 files changed, 141 insertions, 0 deletions

diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt
index 485d68b4ae3..18c3676c366 100644
--- a/eval/CMakeLists.txt
+++ b/eval/CMakeLists.txt
@@ -34,6 +34,7 @@ vespa_define_module(
     src/tests/eval/value_codec
     src/tests/eval/value_type
     src/tests/gp/ponder_nov2017
+    src/tests/instruction/generic_concat
     src/tests/instruction/generic_join
     src/tests/instruction/generic_merge
     src/tests/instruction/generic_reduce
diff --git a/eval/src/tests/instruction/generic_concat/CMakeLists.txt b/eval/src/tests/instruction/generic_concat/CMakeLists.txt
new file mode 100644
index 00000000000..ddc6d4f5944
--- /dev/null
+++ b/eval/src/tests/instruction/generic_concat/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(eval_generic_concat_test_app TEST
+    SOURCES
+    generic_concat_test.cpp
+    DEPENDS
+    vespaeval
+    GTest::GTest
+)
+vespa_add_test(NAME eval_generic_concat_test_app COMMAND eval_generic_concat_test_app)
diff --git a/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp
new file mode 100644
index 00000000000..b4822e2115e
--- /dev/null
+++ b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp
@@ -0,0 +1,123 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/eval/eval/simple_value.h>
+#include <vespa/eval/eval/value_codec.h>
+#include <vespa/eval/eval/simple_tensor.h>
+#include <vespa/eval/eval/simple_tensor_engine.h>
+#include <vespa/eval/eval/value_codec.h>
+#include <vespa/eval/instruction/generic_join.h>
+#include <vespa/eval/eval/interpreted_function.h>
+#include <vespa/eval/eval/test/tensor_model.hpp>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using namespace vespalib;
+using namespace vespalib::eval;
+using namespace vespalib::eval::instruction;
+using namespace vespalib::eval::test;
+
+using vespalib::make_string_short::fmt;
+
+std::vector<Layout> concat_layouts = {
+    {},                                                 {},
+    {},                                                 {x(5)},
+    {x(5)},                                             {},
+    {x(2)},                                             {x(3)},
+    {x(2)},                                             {y(3)},
+    {y(2)},                                             {z(3)},
+    {x(5)},                                             {x(2),y(5)},
+    {y(3)},                                             {x(2),z(3)},
+    {x(3),y(5)},                                        {y(5),z(7)},
+    float_cells({x(3),y(5)}),                           {y(5),z(7)},
+    {x(3),y(5)},                                        float_cells({y(5),z(7)}),
+    float_cells({x(3),y(5)}),                           float_cells({y(5),z(7)}),
+    {y({"a","b","c"})},                                 {y({"a","b","c"})},
+    {y({"a","b","c"})},                                 {y({"a","b"})},
+    {y({"a","b","c"})},                                 {y({"b","c","d"})},
+    float_cells({y({"a","b","c"})}),                    {y({"b","c","d"})},
+    {y({"a","b","c"})},                                 float_cells({y({"b","c","d"})}),
+    float_cells({y({"a","b","c"})}),                    float_cells({z({"foo","bar","baz"})}),
+    {y({"a","b","c"})},                                 {y({"a","b","c"}),z({"foo","bar","baz"})},
+    {y({"a","b"}),z({"foo","bar","baz"})},              {y({"a","b","c"}),z({"foo","bar"})},
+    {x(2),y({"a","b","c"})},                            {x(3),y({"b","c","d"})},
+    {x(2),y({"a","b"})},                                {x(3),z({"c","d"})}
+};
+
+TensorSpec perform_simpletensor_concat(const TensorSpec &a, const TensorSpec &b, const std::string &dimension) {
+    auto lhs = SimpleTensor::create(a);
+    auto rhs = SimpleTensor::create(b);
+    auto out = SimpleTensor::concat(*lhs, *rhs, dimension);
+    return SimpleTensorEngine::ref().to_spec(*out);
+}
+
+bool concat_address(const TensorSpec::Address &me, const TensorSpec::Address &other,
+                    const std::string &concat_dim, size_t my_offset,
+                    TensorSpec::Address &my_out, TensorSpec::Address &other_out)
+{
+    my_out.insert_or_assign(concat_dim, my_offset);
+    for (const auto &my_dim: me) {
+        const auto & name = my_dim.first;
+        const auto & label = my_dim.second;
+        if (name == concat_dim) {
+            my_out.insert_or_assign(name, label.index + my_offset);
+        } else {
+            auto pos = other.find(name);
+            if ((pos == other.end()) || (pos->second == label)) {
+                my_out.insert_or_assign(name, label);
+                other_out.insert_or_assign(name, label);
+            } else {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+bool concat_addresses(const TensorSpec::Address &a, const TensorSpec::Address &b,
+                      const std::string &concat_dim, size_t b_offset,
+                      TensorSpec::Address &a_out, TensorSpec::Address &b_out)
+{
+    return concat_address(a, b, concat_dim,        0, a_out, b_out) &&
+           concat_address(b, a, concat_dim, b_offset, b_out, a_out);
+}
+
+TensorSpec reference_concat(const TensorSpec &a, const TensorSpec &b, const std::string &concat_dim) {
+    ValueType a_type = ValueType::from_spec(a.type());
+    ValueType b_type = ValueType::from_spec(b.type());
+    size_t cc_dim_a_size = 1;
+    for (const auto & dim : a_type.dimensions()) {
+        if (dim.name == concat_dim) {
+            EXPECT_TRUE(dim.is_indexed());
+            cc_dim_a_size = dim.size;
+        }
+    }
+    ValueType res_type = ValueType::concat(a_type, b_type, concat_dim);
+    EXPECT_FALSE(res_type.is_error());
+    TensorSpec result(res_type.to_spec());
+    for (const auto &cell_a: a.cells()) {
+        for (const auto &cell_b: b.cells()) {
+            TensorSpec::Address addr_a;
+            TensorSpec::Address addr_b;
+            if (concat_addresses(cell_a.first, cell_b.first, concat_dim, cc_dim_a_size, addr_a, addr_b)) {
+                result.set(addr_a, cell_a.second);
+                result.set(addr_b, cell_b.second);
+            }
+        }
+    }
+    return result;
+}
+
+TEST(GenericConcatTest, generic_reference_concat_works) {
+    ASSERT_TRUE((concat_layouts.size() % 2) == 0);
+    for (size_t i = 0; i < concat_layouts.size(); i += 2) {
+        const TensorSpec lhs = spec(concat_layouts[i], N());
+        const TensorSpec rhs = spec(concat_layouts[i + 1], Div16(N()));
+        SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str()));
+        auto expect = reference_concat(lhs, rhs, "x");
+        auto actual = perform_simpletensor_concat(lhs, rhs, "x");
+        // auto actual = perform_generic_concat(lhs, rhs, "x");
+        EXPECT_EQ(actual, expect);
+    }
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/eval/src/vespa/eval/eval/tensor_spec.cpp b/eval/src/vespa/eval/eval/tensor_spec.cpp
index 712ee282c71..d3aafc7632a 100644
--- a/eval/src/vespa/eval/eval/tensor_spec.cpp
+++ b/eval/src/vespa/eval/eval/tensor_spec.cpp
@@ -45,6 +45,13 @@ TensorSpec & TensorSpec::operator = (const TensorSpec &) = default;
 
 TensorSpec::~TensorSpec() { }
 
+TensorSpec &
+TensorSpec::set(Address address, double value) {
+    auto res = _cells.emplace(std::move(address), value);
+    if (!res.second) { assert(res.first->second.value == value); }
+    return *this;
+}
+
 vespalib::string
 TensorSpec::to_string() const
 {
diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h
index 974ad4a1f4c..8a4343b3faa 100644
--- a/eval/src/vespa/eval/eval/tensor_spec.h
+++ b/eval/src/vespa/eval/eval/tensor_spec.h
@@ -68,6 +68,7 @@ public:
     TensorSpec(const TensorSpec &);
     TensorSpec & operator = (const TensorSpec &);
     ~TensorSpec();
+    TensorSpec &set(Address address, double value);
     TensorSpec &add(Address address, double value) {
         auto res = _cells.emplace(std::move(address), value);
         if (!res.second) {