Evaluate ONNX models in model-evaluation with ONNX RT

7 files changed, 125 insertions, 0 deletions

diff --git a/model-evaluation/src/test/resources/config/onnx/models/add_mul.onnx b/model-evaluation/src/test/resources/config/onnx/models/add_mul.onnx
new file mode 100644
index 00000000000..ab054d112e9
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/onnx/models/add_mul.onnx
@@ -0,0 +1,24 @@
+
+add_mul.py:£

+
+input1
+input2output1"Mul
+
+input1
+input2output2"Addadd_mulZ
+input1
+
+

+
Z
+input2
+
+

+
b
+output1
+
+

+
b
+output2
+
+

+
B
\ No newline at end of file
diff --git a/model-evaluation/src/test/resources/config/onnx/models/add_mul.py b/model-evaluation/src/test/resources/config/onnx/models/add_mul.py
new file mode 100755
index 00000000000..3a4522042e8
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/onnx/models/add_mul.py
@@ -0,0 +1,30 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import onnx
+from onnx import helper, TensorProto
+
+INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT, [1])
+INPUT_2 = helper.make_tensor_value_info('input2', TensorProto.FLOAT, [1])
+OUTPUT_1 = helper.make_tensor_value_info('output1', TensorProto.FLOAT, [1])
+OUTPUT_2 = helper.make_tensor_value_info('output2', TensorProto.FLOAT, [1])
+
+nodes = [
+    helper.make_node(
+        'Mul',
+        ['input1', 'input2'],
+        ['output1'],
+    ),
+    helper.make_node(
+        'Add',
+        ['input1', 'input2'],
+        ['output2'],
+    ),
+]
+graph_def = helper.make_graph(
+    nodes,
+    'add_mul',
+    [INPUT_1, INPUT_2],
+    [OUTPUT_1, OUTPUT_2],
+)
+model_def = helper.make_model(graph_def, producer_name='add_mul.py', opset_imports=[onnx.OperatorSetIdProto(version=12)])
+onnx.save(model_def, 'add_mul.onnx')
diff --git a/model-evaluation/src/test/resources/config/onnx/models/one_layer.onnx b/model-evaluation/src/test/resources/config/onnx/models/one_layer.onnx
new file mode 100644
index 00000000000..dc9f664b943
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/onnx/models/one_layer.onnx
diff --git a/model-evaluation/src/test/resources/config/onnx/models/pytorch_one_layer.py b/model-evaluation/src/test/resources/config/onnx/models/pytorch_one_layer.py
new file mode 100755
index 00000000000..1296d84e180
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/onnx/models/pytorch_one_layer.py
@@ -0,0 +1,38 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import torch
+import torch.onnx
+
+
+class MyModel(torch.nn.Module):
+    def __init__(self):
+        super(MyModel, self).__init__()
+        self.linear = torch.nn.Linear(in_features=3, out_features=1)
+        self.logistic = torch.nn.Sigmoid()
+
+    def forward(self, vec):
+        return self.logistic(self.linear(vec))
+
+
+def main():
+    model = MyModel()
+
+    # Omit training - just export randomly initialized network
+
+    data = torch.FloatTensor([[0.1, 0.2, 0.3],[0.4, 0.5, 0.6]])
+    torch.onnx.export(model,
+                      data,
+                      "one_layer.onnx",
+                      input_names = ["input"],
+                      output_names = ["output"],
+                      dynamic_axes = {
+                          "input": {0: "batch"},
+                          "output": {0: "batch"},
+                      },
+                      opset_version=12)
+
+
+if __name__ == "__main__":
+    main()
+
+
diff --git a/model-evaluation/src/test/resources/config/onnx/onnx-models.cfg b/model-evaluation/src/test/resources/config/onnx/onnx-models.cfg
new file mode 100644
index 00000000000..9ad9c7f6a07
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/onnx/onnx-models.cfg
@@ -0,0 +1,16 @@
+model[0].name "add_mul"
+model[0].fileref "models/add_mul.onnx"
+model[0].input[0].name "input1"
+model[0].input[0].source "input1"
+model[0].input[1].name "input2"
+model[0].input[1].source "input2"
+model[0].output[0].name "output1"
+model[0].output[0].as "output1"
+model[0].output[1].name "output2"
+model[0].output[1].as "output2"
+model[1].name "one_layer"
+model[1].fileref "models/one_layer.onnx"
+model[1].input[0].name "input"
+model[1].input[0].source "input"
+model[1].output[0].name "output"
+model[1].output[0].as "output"
diff --git a/model-evaluation/src/test/resources/config/onnx/rank-profiles.cfg b/model-evaluation/src/test/resources/config/onnx/rank-profiles.cfg
new file mode 100644
index 00000000000..047b7c3c77b
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/onnx/rank-profiles.cfg
@@ -0,0 +1,17 @@
+rankprofile[0].name "add_mul"
+rankprofile[0].fef.property[0].name "rankingExpression(output1).rankingScript"
+rankprofile[0].fef.property[0].value "onnxModel(add_mul).output1"
+rankprofile[0].fef.property[1].name "rankingExpression(output1).type"
+rankprofile[0].fef.property[1].value "tensor<float>(d0[1])"
+rankprofile[0].fef.property[2].name "rankingExpression(output2).rankingScript"
+rankprofile[0].fef.property[2].value "onnxModel(add_mul).output2"
+rankprofile[0].fef.property[3].name "rankingExpression(output2).type"
+rankprofile[0].fef.property[3].value "tensor<float>(d0[1])"
+rankprofile[1].name "one_layer"
+rankprofile[1].fef.property[0].name "rankingExpression(output).rankingScript"
+rankprofile[1].fef.property[0].value "onnxModel(one_layer)"
+rankprofile[1].fef.property[1].name "rankingExpression(output).type"
+rankprofile[1].fef.property[1].value "tensor<float>(d0[],d1[1])"
+rankprofile[2].name "no_model"
+rankprofile[2].fef.property[0].name "rankingExpression(output).rankingScript"
+rankprofile[2].fef.property[0].value "onnxModel(no_model)"
diff --git a/model-evaluation/src/test/resources/config/onnx/ranking-constants.cfg b/model-evaluation/src/test/resources/config/onnx/ranking-constants.cfg
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/model-evaluation/src/test/resources/config/onnx/ranking-constants.cfg