diff options
Diffstat (limited to 'model-integration/src/test')
7 files changed, 141 insertions, 6 deletions
diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java index db2e9db1277..75da4d163bb 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java @@ -9,9 +9,16 @@ import org.junit.Test; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.logging.Handler; +import java.util.logging.LogRecord; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @@ -83,12 +90,14 @@ public class OnnxEvaluatorTest { var runtime = new OnnxRuntime(); assertEvaluate(runtime, "add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); assertEvaluate(runtime, "add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); + assertEvaluate(runtime, "add_float16.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); + //Add is not a supported operation for bfloat16 types in onnx operators. + assertEvaluate(runtime, "sign_bfloat16.onnx", "tensor<bfloat16>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); + assertEvaluate(runtime, "add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); assertEvaluate(runtime, "cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]"); assertEvaluate(runtime, "cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]"); - - // ONNX Runtime 1.8.0 does not support much of bfloat16 yet - // assertEvaluate("cast_bfloat16_float.onnx", "tensor<float>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); + assertEvaluate(runtime,"cast_bfloat16_float.onnx", "tensor<float>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); } @Test @@ -170,6 +179,29 @@ public class OnnxEvaluatorTest { evaluator.close(); } + @Test + public void testLoggingMessages() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + Logger logger = Logger.getLogger(OnnxEvaluator.class.getName()); + CustomLogHandler logHandler = new CustomLogHandler(); + logger.addHandler(logHandler); + var runtime = new OnnxRuntime(); + var model = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx")); + OnnxEvaluatorOptions options = new OnnxEvaluatorOptions(); + options.setGpuDevice(0); + var evaluator = runtime.evaluatorOf(model,options); + evaluator.close(); + List<LogRecord> records = logHandler.getLogRecords(); + assertEquals(1,records.size()); + assertEquals(Level.INFO,records.get(0).getLevel()); + String message = records.get(0).getMessage(); + assertEquals("Failed to create session with CUDA using GPU device 0. " + + "Falling back to CPU. Reason: Error code - ORT_EP_FAIL - message:" + + " Failed to find CUDA shared provider", message); + logger.removeHandler(logHandler); + + } + private void assertEvaluate(OnnxRuntime runtime, String model, String output, String... input) { OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/" + model); Map<String, Tensor> inputs = new HashMap<>(); @@ -182,4 +214,25 @@ public class OnnxEvaluatorTest { assertEquals(expected.type().valueType(), result.type().valueType()); } + static class CustomLogHandler extends Handler { + private List<LogRecord> records = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + records.add(record); + } + + @Override + public void flush() { + } + + @Override + public void close() throws SecurityException { + } + + public List<LogRecord> getLogRecords() { + return records; + } + } + } diff --git a/model-integration/src/test/models/onnx/add_float16.onnx b/model-integration/src/test/models/onnx/add_float16.onnx new file mode 100644 index 00000000000..df0f7fdcdba --- /dev/null +++ b/model-integration/src/test/models/onnx/add_float16.onnx @@ -0,0 +1,19 @@ +add_float16.py:f + +input1 +input2output"AddaddZ +input1 + + + +Z +input2 + + + +b +output + + + +B
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/add_float16.py b/model-integration/src/test/models/onnx/add_float16.py new file mode 100755 index 00000000000..a637cf8b0dd --- /dev/null +++ b/model-integration/src/test/models/onnx/add_float16.py @@ -0,0 +1,27 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import onnx +from onnx import helper, TensorProto + +INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT16, [1]) +INPUT_2 = helper.make_tensor_value_info('input2', TensorProto.FLOAT16, [1]) +OUTPUT = helper.make_tensor_value_info('output', TensorProto.FLOAT16, [1]) + +nodes = [ + helper.make_node( + 'Add', + ['input1', 'input2'], + ['output'], + ), +] +graph_def = helper.make_graph( + nodes, + 'add', + [ + INPUT_1, + INPUT_2 + ], + [OUTPUT], +) +model_def = helper.make_model(graph_def, producer_name='add_float16.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +onnx.save(model_def, 'add_float16.onnx') diff --git a/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx b/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx index cb19592abf4..9fcbd7f1b3c 100644 --- a/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx +++ b/model-integration/src/test/models/onnx/cast_bfloat16_float.onnx @@ -1,4 +1,4 @@ -cast_bfloat16_float.py:U +cast_bfloat16_float.py:U ! input1output"Cast* to castZ @@ -9,4 +9,4 @@ output -B
\ No newline at end of file +B
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/cast_bfloat16_float.py b/model-integration/src/test/models/onnx/cast_bfloat16_float.py index 51d04747958..952e4c469c1 100755 --- a/model-integration/src/test/models/onnx/cast_bfloat16_float.py +++ b/model-integration/src/test/models/onnx/cast_bfloat16_float.py @@ -20,5 +20,5 @@ graph_def = helper.make_graph( [INPUT_1], [OUTPUT], ) -model_def = helper.make_model(graph_def, producer_name='cast_bfloat16_float.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +model_def = helper.make_model(graph_def, producer_name='cast_bfloat16_float.py', opset_imports=[onnx.OperatorSetIdProto(version=19)]) onnx.save(model_def, 'cast_bfloat16_float.onnx') diff --git a/model-integration/src/test/models/onnx/sign_bfloat16.onnx b/model-integration/src/test/models/onnx/sign_bfloat16.onnx new file mode 100644 index 00000000000..176451108ba --- /dev/null +++ b/model-integration/src/test/models/onnx/sign_bfloat16.onnx @@ -0,0 +1,11 @@ +sign_bfloat16.py:J + +input1output"SignsignZ +input1 + + +b +output + + +B
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/sign_bfloat16.py b/model-integration/src/test/models/onnx/sign_bfloat16.py new file mode 100755 index 00000000000..b74e48e91c8 --- /dev/null +++ b/model-integration/src/test/models/onnx/sign_bfloat16.py @@ -0,0 +1,25 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import onnx +from onnx import helper, TensorProto + +INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.BFLOAT16, [1]) +OUTPUT = helper.make_tensor_value_info('output', TensorProto.BFLOAT16, [1]) + +nodes = [ + helper.make_node( + 'Sign', + ['input1'], + ['output'], + ), +] +graph_def = helper.make_graph( + nodes, + 'sign', + [ + INPUT_1 + ], + [OUTPUT], +) +model_def = helper.make_model(graph_def, producer_name='sign_bfloat16.py', opset_imports=[onnx.OperatorSetIdProto(version=19)]) +onnx.save(model_def, 'sign_bfloat16.onnx') |