avoid crash on run-time onnx errors

- warn about onnx model dry-run being disabled
- catch and report onnx errors during ranking
- zero-fill failed results to avoid re-using previous results
- use explicit output size in fragile model
  (output became float[2] instead of float[batch] anyways)

3 files changed, 55 insertions, 1 deletions

diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp
index 6b45172ef80..2a336ecf099 100644
--- a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp
+++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp
@@ -178,6 +178,9 @@ TEST(OnnxTest, simple_onnx_model_can_be_evaluated)
     ctx.eval();
     EXPECT_EQ(output.cells().typify<float>()[0], 80.0);
     //-------------------------------------------------------------------------
+    ctx.clear_results();
+    EXPECT_EQ(output.cells().typify<float>()[0], 0.0);
+    //-------------------------------------------------------------------------
 }
 
 TEST(OnnxTest, dynamic_onnx_model_can_be_evaluated)
@@ -224,6 +227,9 @@ TEST(OnnxTest, dynamic_onnx_model_can_be_evaluated)
     ctx.eval();
     EXPECT_EQ(output.cells().typify<float>()[0], 81.0);
     //-------------------------------------------------------------------------
+    ctx.clear_results();
+    EXPECT_EQ(output.cells().typify<float>()[0], 0.0);
+    //-------------------------------------------------------------------------
 }
 
 TEST(OnnxTest, int_types_onnx_model_can_be_evaluated)
@@ -270,6 +276,9 @@ TEST(OnnxTest, int_types_onnx_model_can_be_evaluated)
     ctx.eval();
     EXPECT_EQ(output.cells().typify<double>()[0], 80.0);
     //-------------------------------------------------------------------------
+    ctx.clear_results();
+    EXPECT_EQ(output.cells().typify<double>()[0], 0.0);
+    //-------------------------------------------------------------------------
 }
 
 TEST(OnnxTest, we_guess_batch_dimension_size_when_inference_fails) {    
@@ -311,6 +320,15 @@ TEST(OnnxTest, we_guess_batch_dimension_size_when_inference_fails) {
     EXPECT_EQ(out_3, expect_3);
     EXPECT_EQ(out_4, expect_4);
     //-------------------------------------------------------------------------
+    auto zero_3 = TensorSpec::from_expr("tensor<float>(d0[3]):[0,0,0]");
+    auto zero_4 = TensorSpec::from_expr("tensor<float>(d0[4]):[0,0,0,0]");
+    ctx_3.clear_results();
+    EXPECT_EQ(TensorSpec::from_value(ctx_3.get_result(0)), zero_3);
+    EXPECT_EQ(TensorSpec::from_value(ctx_4.get_result(0)), expect_4);
+    ctx_4.clear_results();
+    EXPECT_EQ(TensorSpec::from_value(ctx_3.get_result(0)), zero_3);
+    EXPECT_EQ(TensorSpec::from_value(ctx_4.get_result(0)), zero_4);
+    //-------------------------------------------------------------------------
 }
 
 TEST(OnnxTest, zero_copy_unstable_types) {
@@ -356,6 +374,14 @@ TEST(OnnxTest, zero_copy_unstable_types) {
     EXPECT_EQ(cells16.typify<BFloat16>()[1], 2.0);
     EXPECT_EQ(cells16.typify<BFloat16>()[2], 3.0);
     //-------------------------------------------------------------------------
+    ctx.clear_results();
+    EXPECT_EQ(cells8.typify<Int8Float>()[0], 0.0);
+    EXPECT_EQ(cells8.typify<Int8Float>()[1], 0.0);
+    EXPECT_EQ(cells8.typify<Int8Float>()[2], 0.0);
+    EXPECT_EQ(cells16.typify<BFloat16>()[0], 0.0);
+    EXPECT_EQ(cells16.typify<BFloat16>()[1], 0.0);
+    EXPECT_EQ(cells16.typify<BFloat16>()[2], 0.0);
+    //-------------------------------------------------------------------------
 }
 
 TEST(OnnxTest, converted_unstable_types) {
@@ -401,6 +427,14 @@ TEST(OnnxTest, converted_unstable_types) {
     EXPECT_EQ(cells16.typify<BFloat16>()[1], 2.0);
     EXPECT_EQ(cells16.typify<BFloat16>()[2], 3.0);
     //-------------------------------------------------------------------------
+    ctx.clear_results();
+    EXPECT_EQ(cells8.typify<Int8Float>()[0], 0.0);
+    EXPECT_EQ(cells8.typify<Int8Float>()[1], 0.0);
+    EXPECT_EQ(cells8.typify<Int8Float>()[2], 0.0);
+    EXPECT_EQ(cells16.typify<BFloat16>()[0], 0.0);
+    EXPECT_EQ(cells16.typify<BFloat16>()[1], 0.0);
+    EXPECT_EQ(cells16.typify<BFloat16>()[2], 0.0);
+    //-------------------------------------------------------------------------
 }
 
 TEST(OnnxTest, inspect_float_to_int8_conversion) {
diff --git a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp
index e2528fcb1c3..6e857c51b2b 100644
--- a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp
+++ b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp
@@ -97,6 +97,17 @@ struct CreateVespaTensor {
     }
 };
 
+struct ClearVespaTensor {
+    template <typename CT> static void invoke(const Value &value) {
+        auto cells = unconstify(value.cells().typify<CT>());
+        std::fill(cells.begin(), cells.end(), CT{});
+    }
+    void operator()(const Value &value) {
+        return typify_invoke<1,TypifyCellType,ClearVespaTensor>(value.type().cell_type(), value);
+    }
+};
+ClearVespaTensor clear_vespa_tensor;
+
 //-----------------------------------------------------------------------------
 
 template <typename E> vespalib::string type_name(E enum_value) {
@@ -202,7 +213,7 @@ std::vector<int64_t> extract_sizes(const ValueType &type) {
     return sizes;
 }
 
-}
+} // <unnamed>
 
 vespalib::string
 Onnx::DimSize::as_string() const
@@ -488,6 +499,14 @@ Onnx::EvalContext::eval()
     }
 }
 
+void
+Onnx::EvalContext::clear_results()
+{
+    for (const Value::UP &result: _results) {
+        clear_vespa_tensor(*result);
+    }
+}
+
 const Value &
 Onnx::EvalContext::get_result(size_t i) const
 {
diff --git a/eval/src/vespa/eval/onnx/onnx_wrapper.h b/eval/src/vespa/eval/onnx/onnx_wrapper.h
index 9392536eae7..7084efab207 100644
--- a/eval/src/vespa/eval/onnx/onnx_wrapper.h
+++ b/eval/src/vespa/eval/onnx/onnx_wrapper.h
@@ -134,6 +134,7 @@ public:
         size_t num_results() const { return _result_values.size(); }
         void bind_param(size_t i, const Value &param);
         void eval();
+        void clear_results();
         const Value &get_result(size_t i) const;
     };