ignore some conformance tests for Java

peek out-of-bounds
nan vs non-existing cells in merge/join

4 files changed, 40 insertions, 14 deletions

diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp
index 9ae33c1234f..3c2a9b12272 100644
--- a/eval/src/apps/tensor_conformance/generate.cpp
+++ b/eval/src/apps/tensor_conformance/generate.cpp
@@ -436,9 +436,9 @@ void generate_converting_lambda(TestBuilder &dst) {
     auto sparse = GenSpec::from_desc("y5_2");
     auto mixed = GenSpec::from_desc("x3y5_2");
     // change cell type and dimension types
-    dst.add("tensor<bfloat16>(x[5])(a{x:(x)})", {{"a", dense}});
-    dst.add("tensor<bfloat16>(y[10])(a{y:(y)})", {{"a", sparse}});
-    dst.add("tensor<bfloat16>(x[5],y[10])(a{x:(x),y:(y)})", {{"a", mixed}});
+    dst.add_ignore_java("tensor<bfloat16>(x[5])(a{x:(x)})", {{"a", dense}});
+    dst.add_ignore_java("tensor<bfloat16>(y[10])(a{y:(y)})", {{"a", sparse}});
+    dst.add_ignore_java("tensor<bfloat16>(x[5],y[10])(a{x:(x),y:(y)})", {{"a", mixed}});
 }
 
 //-----------------------------------------------------------------------------
@@ -492,12 +492,12 @@ void generate_nan_existence(TestBuilder &dst) {
     const vespalib::string inner_expr = "f(x,y)(if(isNan(x),11,x)+if(isNan(y),22,y))";
     vespalib::string merge_expr = fmt("merge(a,b,%s)", inner_expr.c_str());
     vespalib::string join_expr = fmt("join(a,b,%s)", inner_expr.c_str());
-    dst.add(merge_expr, {{"a", sparse1}, {"b", sparse2}});
-    dst.add(merge_expr, {{"a",  mixed1}, {"b",  mixed2}});
-    dst.add(join_expr, {{"a", sparse1}, {"b", sparse2}});
-    dst.add(join_expr, {{"a",  mixed1}, {"b",  mixed2}});
-    dst.add(join_expr, {{"a", sparse1}, {"b",  mixed2}});
-    dst.add(join_expr, {{"a",  mixed1}, {"b", sparse2}});
+    dst.add_ignore_java(merge_expr, {{"a", sparse1}, {"b", sparse2}});
+    dst.add_ignore_java(merge_expr, {{"a",  mixed1}, {"b",  mixed2}});
+    dst.add_ignore_java(join_expr, {{"a", sparse1}, {"b", sparse2}});
+    dst.add_ignore_java(join_expr, {{"a",  mixed1}, {"b",  mixed2}});
+    dst.add_ignore_java(join_expr, {{"a", sparse1}, {"b",  mixed2}});
+    dst.add_ignore_java(join_expr, {{"a",  mixed1}, {"b", sparse2}});
 }
 
 //-----------------------------------------------------------------------------
diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h
index a71531f7cf3..e9482b9015c 100644
--- a/eval/src/apps/tensor_conformance/generate.h
+++ b/eval/src/apps/tensor_conformance/generate.h
@@ -4,13 +4,25 @@
 
 #include <vespa/eval/eval/tensor_spec.h>
 #include <map>
+#include <set>
 
 struct TestBuilder {
     bool full;
     TestBuilder(bool full_in) : full(full_in) {}
     using TensorSpec = vespalib::eval::TensorSpec;
     virtual void add(const vespalib::string &expression,
-                     const std::map<vespalib::string,TensorSpec> &inputs) = 0;
+                     const std::map<vespalib::string,TensorSpec> &inputs,
+                     const std::set<vespalib::string> &ignore) = 0;
+    void add(const vespalib::string &expression,
+             const std::map<vespalib::string,TensorSpec> &inputs)
+    {
+        add(expression, inputs, {});
+    }
+    void add_ignore_java(const vespalib::string &expression,
+                         const std::map<vespalib::string,TensorSpec> &inputs)
+    {
+        add(expression, inputs, {"vespajlib"});
+    }
     virtual ~TestBuilder() {}
 };
 
diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
index 47089e1a8ff..e6bbb1f8a41 100644
--- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp
+++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp
@@ -37,6 +37,7 @@ using namespace std::placeholders;
 //-----------------------------------------------------------------------------
 
 size_t fail_cnt = 0;
+size_t ignore_cnt = 0;
 
 //-----------------------------------------------------------------------------
 
@@ -176,7 +177,8 @@ private:
 public:
     MyTestBuilder(bool full_in, Output &out) : TestBuilder(full_in), _writer(out) {}
     void add(const vespalib::string &expression,
-             const std::map<vespalib::string,TensorSpec> &inputs_in) override
+             const std::map<vespalib::string,TensorSpec> &inputs_in,
+             const std::set<vespalib::string> &ignore_in) override
     {
         Cursor &test = _writer.create();
         test.setString("expression", expression);
@@ -185,6 +187,12 @@ public:
             insert_value(inputs, name, spec);
         }
         test.setObject("result");
+        if (!ignore_in.empty()) {
+            Cursor &ignore = test.setObject("ignore");
+            for (const auto &impl: ignore_in) {
+                ignore.setBool(impl, true);
+            }
+        }
     }
     void add_failing_test(bool ignore_fail) {
         Cursor &test = _writer.create();
@@ -192,7 +200,7 @@ public:
         insert_value(test.setObject("inputs"), "a", GenSpec(1).idx("x", 3));
         insert_value(test.setObject("result"), "dummy", GenSpec(2).idx("x", 3));
         if (ignore_fail) {
-            test.setBool("ignore_fail", true);
+            test.setObject("ignore").setBool("dummy", true);
         }
     }
 };
@@ -232,8 +240,10 @@ void verify(Input &in, Output &out) {
             ++result_map[result];
             auto actual_result = extract_value(slime["result"][result]);
             if (!require_impl::eq(actual_result, reference_result)) {
-                bool ignore_fail = slime["ignore_fail"].asBool();
-                if (!ignore_fail) {
+                bool ignore_fail = slime["ignore"][result].asBool();
+                if (ignore_fail) {
+                    ++ignore_cnt;
+                } else {
                     ++fail_cnt;
                 }
                 fprintf(stderr, "%sexpression failed('%s'): '%s'\n", ignore_fail ? "IGNORED: " : "",
@@ -249,7 +259,9 @@ void verify(Input &in, Output &out) {
             stats.setLong(entry.first, entry.second);
         }
         REQUIRE(!slime["fail_cnt"].valid());
+        REQUIRE(!slime["ignore_cnt"].valid());
         slime.get().setLong("fail_cnt", fail_cnt);
+        slime.get().setLong("ignore_cnt", ignore_cnt);
         JsonFormat::encode(slime, out, false);
     };
     for_each_test(in, handle_test, handle_summary);
diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
index 3f582779202..8271e8b57ee 100644
--- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
+++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp
@@ -26,6 +26,8 @@ TEST("require that (some) cross-language tensor conformance tests pass with C++
     int streamed_tests = result["stats"]["cpp_streamed_value"].asLong();
     EXPECT_TRUE(result["fail_cnt"].valid());
     EXPECT_EQUAL(result["fail_cnt"].asLong(), 0);
+    EXPECT_TRUE(result["ignore_cnt"].valid());
+    EXPECT_EQUAL(result["ignore_cnt"].asLong(), 0);
     EXPECT_GREATER(num_tests, 1000);
     EXPECT_EQUAL(prod_tests, num_tests);
     EXPECT_EQUAL(simple_tests, num_tests);