diff options
Diffstat (limited to 'eval/src')
| -rw-r--r-- | eval/src/apps/analyze_onnx_model/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp | 44 | ||||
| -rw-r--r-- | eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt | 9 | ||||
| -rw-r--r-- | eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp | 137 | ||||
| -rw-r--r-- | eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp | 22 | ||||
| -rw-r--r-- | eval/src/tests/tensor/onnx_wrapper/probe_model.onnx | 30 | ||||
| -rwxr-xr-x | eval/src/tests/tensor/onnx_wrapper/probe_model.py | 35 | ||||
| -rw-r--r-- | eval/src/vespa/eval/eval/test/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | eval/src/vespa/eval/eval/test/eval_onnx.cpp | 54 | ||||
| -rw-r--r-- | eval/src/vespa/eval/eval/test/eval_onnx.h | 13 |
10 files changed, 346 insertions, 2 deletions
diff --git a/eval/src/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/apps/analyze_onnx_model/CMakeLists.txt index e2ed64cd8cc..dc89213f9eb 100644 --- a/eval/src/apps/analyze_onnx_model/CMakeLists.txt +++ b/eval/src/apps/analyze_onnx_model/CMakeLists.txt @@ -1,7 +1,8 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespa-analyze-onnx-model +vespa_add_executable(eval_analyze_onnx_model_app SOURCES analyze_onnx_model.cpp + OUTPUT_NAME vespa-analyze-onnx-model INSTALL bin DEPENDS vespaeval diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index 2f22f903f2e..506073ae8b3 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -4,6 +4,7 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/test/test_io.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/guard.h> @@ -11,8 +12,13 @@ using vespalib::make_string_short::fmt; +using vespalib::Slime; +using vespalib::slime::JsonFormat; +using vespalib::slime::Inspector; +using vespalib::slime::Cursor; using vespalib::FilePointer; using namespace vespalib::eval; +using namespace vespalib::eval::test; bool read_line(FilePointer &file, vespalib::string &line) { char line_buffer[1024]; @@ -169,14 +175,50 @@ int usage(const char *self) { fprintf(stderr, " load onnx model and report memory usage\n"); fprintf(stderr, " options are used to specify unknown values, like dimension sizes\n"); fprintf(stderr, " options are accepted in the order in which they are needed\n"); - fprintf(stderr, " tip: run without options first, to see which you need\n"); + fprintf(stderr, " tip: run without options first, to see which you need\n\n"); + fprintf(stderr, "usage: %s --probe-types\n", self); + fprintf(stderr, " use onnx model to infer/probe output types based on input types\n"); + fprintf(stderr, " parameters are read from stdin and results are written to stdout\n"); + fprintf(stderr, " input format (json): {model:<filename>, inputs:{<name>:vespa-type-string}}\n"); + fprintf(stderr, " output format (json): {outputs:{<name>:vespa-type-string}}\n"); return 1; } +int probe_types() { + StdIn std_in; + StdOut std_out; + Slime params; + if (!JsonFormat::decode(std_in, params)) { + return 3; + } + Slime result; + auto &root = result.setObject(); + auto &types = root.setObject("outputs"); + Onnx model(params["model"].asString().make_string(), Onnx::Optimize::DISABLE); + Onnx::WirePlanner planner; + for (size_t i = 0; i < model.inputs().size(); ++i) { + auto spec = params["inputs"][model.inputs()[i].name].asString().make_string(); + auto input_type = ValueType::from_spec(spec); + REQUIRE(!input_type.is_error()); + REQUIRE(planner.bind_input_type(input_type, model.inputs()[i])); + } + planner.prepare_output_types(model); + for (const auto &output: model.outputs()) { + auto output_type = planner.make_output_type(output); + REQUIRE(!output_type.is_error()); + types.setString(output.name, output_type.to_spec()); + } + write_compact(result, std_out); + return 0; +} + int my_main(int argc, char **argv) { if (argc < 2) { return usage(argv[0]); } + if ((argc == 2) && (vespalib::string(argv[1]) == "--probe-types")) { + return probe_types(); + } Options opts; for (int i = 2; i < argc; ++i) { opts.add_option(argv[i]); diff --git a/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt new file mode 100644 index 00000000000..7b70360a622 --- /dev/null +++ b/eval/src/tests/apps/analyze_onnx_model/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_analyze_onnx_model_test_app TEST + SOURCES + analyze_onnx_model_test.cpp + DEPENDS + vespaeval +) +vespa_add_test(NAME eval_analyze_onnx_model_test_app COMMAND eval_analyze_onnx_model_test_app + DEPENDS eval_analyze_onnx_model_test_app eval_analyze_onnx_model_app) diff --git a/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp new file mode 100644 index 00000000000..2c1b2b21b9e --- /dev/null +++ b/eval/src/tests/apps/analyze_onnx_model/analyze_onnx_model_test.cpp @@ -0,0 +1,137 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/util/child_process.h> +#include <vespa/vespalib/data/input.h> +#include <vespa/vespalib/data/output.h> +#include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/eval/eval/test/test_io.h> + +using namespace vespalib; +using namespace vespalib::eval::test; +using vespalib::make_string_short::fmt; +using vespalib::slime::JsonFormat; +using vespalib::slime::Inspector; + +vespalib::string module_build_path("../../../../"); +vespalib::string binary = module_build_path + "src/apps/analyze_onnx_model/vespa-analyze-onnx-model"; +vespalib::string probe_cmd = binary + " --probe-types"; + +std::string get_source_dir() { + const char *dir = getenv("SOURCE_DIRECTORY"); + return (dir ? dir : "."); +} +std::string source_dir = get_source_dir(); +std::string guess_batch_model = source_dir + "/../../tensor/onnx_wrapper/guess_batch.onnx"; + +//----------------------------------------------------------------------------- + +void read_until_eof(Input &input) { + for (auto mem = input.obtain(); mem.size > 0; mem = input.obtain()) { + input.evict(mem.size); + } +} + +// Output adapter used to write to stdin of a child process +class ChildIn : public Output { + ChildProcess &_child; + SimpleBuffer _output; +public: + ChildIn(ChildProcess &child) : _child(child) {} + WritableMemory reserve(size_t bytes) override { + return _output.reserve(bytes); + } + Output &commit(size_t bytes) override { + _output.commit(bytes); + Memory buf = _output.obtain(); + ASSERT_TRUE(_child.write(buf.data, buf.size)); + _output.evict(buf.size); + return *this; + } +}; + +// Input adapter used to read from stdout of a child process +class ChildOut : public Input { + ChildProcess &_child; + SimpleBuffer _input; +public: + ChildOut(ChildProcess &child) + : _child(child) + { + EXPECT_TRUE(_child.running()); + EXPECT_TRUE(!_child.failed()); + } + Memory obtain() override { + if ((_input.get().size == 0) && !_child.eof()) { + WritableMemory buf = _input.reserve(4_Ki); + uint32_t res = _child.read(buf.data, buf.size); + ASSERT_TRUE((res > 0) || _child.eof()); + _input.commit(res); + } + return _input.obtain(); + } + Input &evict(size_t bytes) override { + _input.evict(bytes); + return *this; + } +}; + +//----------------------------------------------------------------------------- + +void dump_message(const char *prefix, const Slime &slime) { + SimpleBuffer buf; + slime::JsonFormat::encode(slime, buf, true); + auto str = buf.get().make_string(); + fprintf(stderr, "%s%s\n", prefix, str.c_str()); +} + +class Server { +private: + TimeBomb _bomb; + ChildProcess _child; + ChildIn _child_stdin; + ChildOut _child_stdout; +public: + Server(vespalib::string cmd) + : _bomb(60), + _child(cmd.c_str()), + _child_stdin(_child), + _child_stdout(_child) {} + ~Server(); + Slime invoke(const Slime &req) { + dump_message("request --> ", req); + write_compact(req, _child_stdin); + Slime reply; + ASSERT_TRUE(JsonFormat::decode(_child_stdout, reply)); + dump_message(" reply <-- ", reply); + return reply; + } +}; +Server::~Server() { + _child.close(); + read_until_eof(_child_stdout); + ASSERT_TRUE(_child.wait()); + ASSERT_TRUE(!_child.running()); + ASSERT_TRUE(!_child.failed()); +} + +//----------------------------------------------------------------------------- + +TEST_F("require that output types can be probed", Server(probe_cmd)) { + Slime params; + params.setObject(); + params.get().setString("model", guess_batch_model); + params.get().setObject("inputs"); + params["inputs"].setString("in1", "tensor<float>(x[3])"); + params["inputs"].setString("in2", "tensor<float>(x[3])"); + Slime result = f1.invoke(params); + EXPECT_EQUAL(result["outputs"]["out"].asString().make_string(), vespalib::string("tensor<float>(d0[3])")); +} + +//----------------------------------------------------------------------------- + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp index da957673f95..e50c41e2e09 100644 --- a/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp +++ b/eval/src/tests/tensor/onnx_wrapper/onnx_wrapper_test.cpp @@ -2,6 +2,7 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/int8float.h> +#include <vespa/eval/eval/test/eval_onnx.h> #include <vespa/eval/onnx/onnx_wrapper.h> #include <vespa/eval/onnx/onnx_model_cache.h> #include <vespa/vespalib/util/bfloat16.h> @@ -28,6 +29,7 @@ std::string int_types_model = source_dir + "/int_types.onnx"; std::string guess_batch_model = source_dir + "/guess_batch.onnx"; std::string unstable_types_model = source_dir + "/unstable_types.onnx"; std::string float_to_int8_model = source_dir + "/float_to_int8.onnx"; +std::string probe_model = source_dir + "/probe_model.onnx"; void dump_info(const char *ctx, const std::vector<TensorInfo> &info) { fprintf(stderr, "%s:\n", ctx); @@ -504,4 +506,24 @@ TEST(OnnxModelCacheTest, share_and_evict_onnx_models) { EXPECT_EQ(OnnxModelCache::count_refs(), 0); } +TensorSpec val(const vespalib::string &expr) { + auto result = TensorSpec::from_expr(expr); + EXPECT_FALSE(ValueType::from_spec(result.type()).is_error()); + return result; +} + +TEST(OnnxTest, eval_onnx_with_probe_model) { + Onnx model(probe_model, Onnx::Optimize::ENABLE); + auto in1 = val("tensor<float>( x[2], y[3]):[[ 1, 2, 3],[ 4, 5, 6]]"); + auto in2 = val("tensor<float>( x[2], y[3]):[[ 7, 8, 9],[ 4, 5, 6]]"); + auto out1 = val("tensor<float>(d0[2],d1[3]):[[ 8,10,12],[ 8,10,12]]"); + auto out2 = val("tensor<float>(d0[2],d1[3]):[[-6,-6,-6],[ 0, 0, 0]]"); + auto out3 = val("tensor<float>(d0[2],d1[3]):[[ 7,16,27],[16,25,36]]"); + auto result = test::eval_onnx(model, {in1, in2}); + ASSERT_EQ(result.size(), 3); + EXPECT_EQ(result[0], out1); + EXPECT_EQ(result[1], out2); + EXPECT_EQ(result[2], out3); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx b/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx new file mode 100644 index 00000000000..89dab2e7c4c --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/probe_model.onnx @@ -0,0 +1,30 @@ +probe_model.py:’ + +in1 +in2out1"Add + +in1 +in2out2"Sub + +in1 +in2out3"Mul probe_modelZ# +in1 + + ÿÿÿÿÿÿÿÿÿ +innerZ# +in2 + +outer + ÿÿÿÿÿÿÿÿÿb$ +out1 + + ÿÿÿÿÿÿÿÿÿ +innerb$ +out2 + +outer + ÿÿÿÿÿÿÿÿÿb( +out3 + + ÿÿÿÿÿÿÿÿÿ + ÿÿÿÿÿÿÿÿÿB
\ No newline at end of file diff --git a/eval/src/tests/tensor/onnx_wrapper/probe_model.py b/eval/src/tests/tensor/onnx_wrapper/probe_model.py new file mode 100755 index 00000000000..529fa23b2b1 --- /dev/null +++ b/eval/src/tests/tensor/onnx_wrapper/probe_model.py @@ -0,0 +1,35 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +import onnx +from onnx import helper, TensorProto + +IN1 = helper.make_tensor_value_info('in1', TensorProto.FLOAT, [-1, 'inner']) +IN2 = helper.make_tensor_value_info('in2', TensorProto.FLOAT, ['outer', -1]) +OUT1 = helper.make_tensor_value_info('out1', TensorProto.FLOAT, [-1, 'inner']) +OUT2 = helper.make_tensor_value_info('out2', TensorProto.FLOAT, ['outer', -1]) +OUT3 = helper.make_tensor_value_info('out3', TensorProto.FLOAT, [-1, -1]) + +nodes = [ + helper.make_node( + 'Add', + ['in1', 'in2'], + ['out1'], + ), + helper.make_node( + 'Sub', + ['in1', 'in2'], + ['out2'], + ), + helper.make_node( + 'Mul', + ['in1', 'in2'], + ['out3'], + ), +] +graph_def = helper.make_graph( + nodes, + 'probe_model', + [IN1, IN2], + [OUT1, OUT2, OUT3], +) +model_def = helper.make_model(graph_def, producer_name='probe_model.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) +onnx.save(model_def, 'probe_model.onnx') diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt index e8a291adf2a..ff1505a4010 100644 --- a/eval/src/vespa/eval/eval/test/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(eval_eval_test OBJECT SOURCES cell_type_space.cpp eval_fixture.cpp + eval_onnx.cpp eval_spec.cpp gen_spec.cpp reference_evaluation.cpp diff --git a/eval/src/vespa/eval/eval/test/eval_onnx.cpp b/eval/src/vespa/eval/eval/test/eval_onnx.cpp new file mode 100644 index 00000000000..74a83b130c2 --- /dev/null +++ b/eval/src/vespa/eval/eval/test/eval_onnx.cpp @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "eval_onnx.h" +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> + +#include <vespa/log/log.h> +LOG_SETUP(".eval.eval.test.eval_onnx"); + +namespace vespalib::eval::test { + +std::vector<TensorSpec> eval_onnx(const Onnx &model, const std::vector<TensorSpec> ¶ms) { + if (params.size() != model.inputs().size()) { + LOG(error, "model with %zu inputs run with %zu parameters", model.inputs().size(), params.size()); + return {}; // wrong number of parameters + } + Onnx::WirePlanner planner; + for (size_t i = 0; i < model.inputs().size(); ++i) { + if (!planner.bind_input_type(ValueType::from_spec(params[i].type()), model.inputs()[i])) { + LOG(error, "unable to bind input type: %s -> %s", params[i].type().c_str(), model.inputs()[i].type_as_string().c_str()); + return {}; // inconsistent input types + } + } + planner.prepare_output_types(model); + for (size_t i = 0; i < model.outputs().size(); ++i) { + if (planner.make_output_type(model.outputs()[i]).is_error()) { + LOG(error, "unable to make output type: %s -> error", model.outputs()[i].type_as_string().c_str()); + return {}; // unable to infer/probe output type + } + } + planner.prepare_output_types(model); + auto wire_info = planner.get_wire_info(model); + try { + Onnx::EvalContext context(model, wire_info); + std::vector<Value::UP> inputs; + for (const auto ¶m: params) { + inputs.push_back(value_from_spec(param, FastValueBuilderFactory::get())); + } + for (size_t i = 0; i < model.inputs().size(); ++i) { + context.bind_param(i, *inputs[i]); + } + context.eval(); + std::vector<TensorSpec> results; + for (size_t i = 0; i < model.outputs().size(); ++i) { + results.push_back(spec_from_value(context.get_result(i))); + } + return results; + } catch (const Ort::Exception &ex) { + LOG(error, "model run failed: %s", ex.what()); + return {}; // evaluation failed + } +} + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/eval_onnx.h b/eval/src/vespa/eval/eval/test/eval_onnx.h new file mode 100644 index 00000000000..bb346b7f21e --- /dev/null +++ b/eval/src/vespa/eval/eval/test/eval_onnx.h @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/onnx/onnx_wrapper.h> +#include <vector> + +namespace vespalib::eval::test { + +std::vector<TensorSpec> eval_onnx(const Onnx &model, const std::vector<TensorSpec> ¶ms); + +} // namespace |