diff options
author | Håvard Pettersen <havardpe@oath.com> | 2020-11-30 13:42:14 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@oath.com> | 2020-12-01 14:20:12 +0000 |
commit | 31fb5db2fe872581ee9b7d81514d5309c136b6b1 (patch) | |
tree | 0e28f9c97e9549bdb4472b1d80305227ec236294 /eval | |
parent | 8d9680d2101c76229190598547362ab2760d3d2f (diff) |
clean up tensor conformance testing
Diffstat (limited to 'eval')
5 files changed, 170 insertions, 400 deletions
diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 1f8069db06d..4e0b5d62b9a 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -7,14 +7,15 @@ #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/tensor.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/tensor_engine.h> -#include <vespa/eval/eval/simple_tensor_engine.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/streamed/streamed_value_builder_factory.h> +#include <vespa/eval/eval/test/reference_evaluation.h> #include <vespa/eval/eval/test/test_io.h> #include <unistd.h> #include <functional> @@ -28,11 +29,16 @@ using namespace vespalib::slime::convenience; using vespalib::slime::inject; using vespalib::slime::SlimeInserter; using slime::JsonFormat; -using tensor::DefaultTensorEngine; using namespace std::placeholders; //----------------------------------------------------------------------------- +const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); +const ValueBuilderFactory &simple_factory = SimpleValueBuilderFactory::get(); +const ValueBuilderFactory &streamed_factory = StreamedValueBuilderFactory::get(); + +//----------------------------------------------------------------------------- + uint8_t unhex(char c) { if (c >= '0' && c <= '9') { return (c - '0'); @@ -67,15 +73,25 @@ nbostream extract_data(const Inspector &value) { void insert_value(Cursor &cursor, const vespalib::string &name, const TensorSpec &spec) { nbostream data; - Value::UP value = SimpleTensorEngine::ref().from_spec(spec); - SimpleTensorEngine::ref().encode(*value, data); + Value::UP value = value_from_spec(spec, simple_factory); + encode_value(*value, data); cursor.setData(name, Memory(data.peek(), data.size())); } TensorSpec extract_value(const Inspector &inspector) { nbostream data = extract_data(inspector); - const auto &engine = SimpleTensorEngine::ref(); - return engine.to_spec(*engine.decode(data)); + return spec_from_value(*decode_value(data, simple_factory)); +} + +//----------------------------------------------------------------------------- + +TensorSpec ref_eval(const Inspector &test) { + auto fun = Function::parse(test["expression"].asString().make_string()); + std::vector<TensorSpec> params; + for (size_t i = 0; i < fun->num_params(); ++i) { + params.push_back(extract_value(test["inputs"][fun->param_name(i)])); + } + return ReferenceEvaluation::eval(*fun, params); } //----------------------------------------------------------------------------- @@ -88,23 +104,21 @@ std::vector<ValueType> get_types(const std::vector<Value::UP> ¶m_values) { return param_types; } -TensorSpec eval_expr(const Inspector &test, EngineOrFactory engine, bool typed) { +TensorSpec eval_expr(const Inspector &test, const ValueBuilderFactory &factory) { auto fun = Function::parse(test["expression"].asString().make_string()); std::vector<Value::UP> param_values; std::vector<Value::CREF> param_refs; for (size_t i = 0; i < fun->num_params(); ++i) { - param_values.emplace_back(engine.from_spec(extract_value(test["inputs"][fun->param_name(i)]))); + param_values.emplace_back(value_from_spec(extract_value(test["inputs"][fun->param_name(i)]), factory)); param_refs.emplace_back(*param_values.back()); } - NodeTypes types = typed ? NodeTypes(*fun, get_types(param_values)) : NodeTypes(); - InterpretedFunction ifun(engine, *fun, types); + NodeTypes types = NodeTypes(*fun, get_types(param_values)); + InterpretedFunction ifun(factory, *fun, types); InterpretedFunction::Context ctx(ifun); SimpleObjectParams params(param_refs); const Value &result = ifun.eval(ctx, params); - if (typed) { - ASSERT_EQUAL(result.type(), types.get_type(fun->root())); - } - return engine.to_spec(result); + ASSERT_EQUAL(result.type(), types.get_type(fun->root())); + return spec_from_value(result); } //----------------------------------------------------------------------------- @@ -138,8 +152,7 @@ private: if (expect != nullptr) { insert_value(test.setObject("result"), "expect", *expect); } else { - insert_value(test.setObject("result"), "expect", - eval_expr(test, SimpleTensorEngine::ref(), false)); + insert_value(test.setObject("result"), "expect", ref_eval(test)); } } public: @@ -168,11 +181,11 @@ void evaluate(Input &in, Output &out) { auto handle_test = [&out](Slime &slime) { insert_value(slime["result"], "cpp_prod", - eval_expr(slime.get(), DefaultTensorEngine::ref(), true)); - insert_value(slime["result"], "cpp_prod_untyped", - eval_expr(slime.get(), DefaultTensorEngine::ref(), false)); - insert_value(slime["result"], "cpp_ref_typed", - eval_expr(slime.get(), SimpleTensorEngine::ref(), true)); + eval_expr(slime.get(), prod_factory)); + insert_value(slime["result"], "cpp_simple_value", + eval_expr(slime.get(), simple_factory)); + insert_value(slime["result"], "cpp_streamed_value", + eval_expr(slime.get(), streamed_factory)); write_compact(slime, out); }; auto handle_summary = [&out](Slime &slime) @@ -196,7 +209,7 @@ void verify(Input &in, Output &out) { std::map<vespalib::string,size_t> result_map; auto handle_test = [&result_map](Slime &slime) { - TensorSpec reference_result = eval_expr(slime.get(), SimpleTensorEngine::ref(), false); + TensorSpec reference_result = ref_eval(slime.get()); for (const auto &result: extract_fields(slime["result"])) { ++result_map[result]; TEST_STATE(make_string("verifying result: '%s'", result.c_str()).c_str()); diff --git a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp index 6468f50a00e..765905c67fc 100644 --- a/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp +++ b/eval/src/tests/tensor/tensor_conformance/tensor_conformance_test.cpp @@ -1,32 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/test/tensor_conformance.h> -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/streamed/streamed_value_builder_factory.h> #include <vespa/eval/eval/fast_value.h> -#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/vespalib/util/stringfmt.h> using vespalib::eval::SimpleValueBuilderFactory; using vespalib::eval::StreamedValueBuilderFactory; using vespalib::eval::FastValueBuilderFactory; -using vespalib::eval::SimpleTensorEngine; using vespalib::eval::test::TensorConformance; -using vespalib::tensor::DefaultTensorEngine; using vespalib::make_string; vespalib::string module_src_path(TEST_PATH("../../../../")); vespalib::string module_build_path("../../../../"); -TEST("require that reference tensor implementation passes all conformance tests") { - TEST_DO(TensorConformance::run_tests(module_src_path, SimpleTensorEngine::ref())); -} - -TEST("require that production tensor implementation passes all conformance tests") { - TEST_DO(TensorConformance::run_tests(module_src_path, DefaultTensorEngine::ref())); -} - TEST("require that SimpleValue implementation passes all conformance tests") { TEST_DO(TensorConformance::run_tests(module_src_path, SimpleValueBuilderFactory::get())); } @@ -46,13 +34,6 @@ TEST("require that tensor serialization test spec can be generated") { EXPECT_EQUAL(system(make_string("diff -u %s binary_test_spec.json", spec.c_str()).c_str()), 0); } -TEST("require that cross-language tensor conformance test spec can be generated") { - vespalib::string spec = module_src_path + "src/apps/tensor_conformance/test_spec.json"; - vespalib::string binary = module_build_path + "src/apps/tensor_conformance/vespa-tensor-conformance"; - EXPECT_EQUAL(system(make_string("%s generate > conformance_test_spec.json", binary.c_str()).c_str()), 0); - EXPECT_EQUAL(system(make_string("%s compare %s conformance_test_spec.json", binary.c_str(), spec.c_str()).c_str()), 0); -} - TEST("require that cross-language tensor conformance tests pass with production C++ expression evaluation") { vespalib::string spec = module_src_path + "src/apps/tensor_conformance/test_spec.json"; vespalib::string binary = module_build_path + "src/apps/tensor_conformance/vespa-tensor-conformance"; diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 701595920ac..d95ce9715ae 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensor_conformance.h" -#include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/function.h> -#include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/aggr.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/eval/eval/simple_value.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/objects/nbostream.h> @@ -14,247 +14,60 @@ #include <vespa/vespalib/io/mapped_file_input.h> #include "tensor_model.hpp" #include "test_io.h" +#include "reference_evaluation.h" + +using vespalib::make_string_short::fmt; + +namespace vespalib::eval::test { -namespace vespalib { -namespace eval { -namespace test { namespace { using slime::Cursor; using slime::Inspector; using slime::JsonFormat; -double as_double(const TensorSpec &spec) { - return spec.cells().empty() ? 0.0 : spec.cells().begin()->second.value; -} - -// abstract evaluation wrapper -struct Eval { - // typed result wrapper - class Result { - private: - enum class Type { ERROR, NUMBER, TENSOR }; - Type _type; - double _number; - TensorSpec _tensor; - public: - Result() : _type(Type::ERROR), _number(0.0), _tensor("error") {} - Result(EngineOrFactory engine, const Value &value) : _type(Type::ERROR), _number(0.0), _tensor("error") { - if (value.is_double()) { - _type = Type::NUMBER; - } - if (value.is_tensor()) { - EXPECT_TRUE(_type == Type::ERROR); - _type = Type::TENSOR; - } - _number = value.as_double(); - _tensor = engine.to_spec(value); - } - bool is_error() const { return (_type == Type::ERROR); } - bool is_number() const { return (_type == Type::NUMBER); } - bool is_tensor() const { return (_type == Type::TENSOR); } - double number() const { - EXPECT_TRUE(is_number()); - return _number; - } - const TensorSpec &tensor() const { - EXPECT_TRUE(is_tensor()); - return _tensor; - } - }; - virtual Result eval(EngineOrFactory) const { - TEST_ERROR("wrong signature"); - return Result(); - } - virtual Result eval(EngineOrFactory, const TensorSpec &) const { - TEST_ERROR("wrong signature"); - return Result(); - } - virtual Result eval(EngineOrFactory, const TensorSpec &, const TensorSpec &) const { - TEST_ERROR("wrong signature"); - return Result(); - } - virtual ~Eval() {} -}; - -// catches exceptions trying to keep the test itself safe from eval side-effects -struct SafeEval : Eval { - const Eval &unsafe; - SafeEval(const Eval &unsafe_in) : unsafe(unsafe_in) {} - Result eval(EngineOrFactory engine) const override { - try { - return unsafe.eval(engine); - } catch (std::exception &e) { - TEST_ERROR(e.what()); - return Result(); - } - } - Result eval(EngineOrFactory engine, const TensorSpec &a) const override { - try { - return unsafe.eval(engine, a); - } catch (std::exception &e) { - TEST_ERROR(e.what()); - return Result(); - } - - } - Result eval(EngineOrFactory engine, const TensorSpec &a, const TensorSpec &b) const override { - try { - return unsafe.eval(engine, a, b); - } catch (std::exception &e) { - TEST_ERROR(e.what()); - return Result(); - } - } -}; -SafeEval safe(const Eval &eval) { return SafeEval(eval); } +//----------------------------------------------------------------------------- -const Value &check_type(const Value &value, const ValueType &expect_type) { - EXPECT_EQUAL(value.type(), expect_type); - return value; +TensorSpec ref_eval(const vespalib::string &expr, const std::vector<TensorSpec> ¶ms) { + TensorSpec result = ReferenceEvaluation::eval(*Function::parse(expr), params); + EXPECT_FALSE(ValueType::from_spec(result.type()).is_error()); + return result; } -// expression(void) -struct Expr_V : Eval { - const vespalib::string &expr; - Expr_V(const vespalib::string &expr_in) : expr(expr_in) {} - Result eval(EngineOrFactory engine) const override { - auto fun = Function::parse(expr); - NodeTypes types(*fun, {}); - InterpretedFunction ifun(engine, *fun, types); - InterpretedFunction::Context ctx(ifun); - SimpleObjectParams params({}); - return Result(engine, check_type(ifun.eval(ctx, params), types.get_type(fun->root()))); - } -}; - -// expression(tensor) -struct Expr_T : Eval { - const vespalib::string &expr; - Expr_T(const vespalib::string &expr_in) : expr(expr_in) {} - Result eval(EngineOrFactory engine, const TensorSpec &a) const override { - auto fun = Function::parse(expr); - auto a_type = ValueType::from_spec(a.type()); - NodeTypes types(*fun, {a_type}); - InterpretedFunction ifun(engine, *fun, types); - InterpretedFunction::Context ctx(ifun); - Value::UP va = engine.from_spec(a); - SimpleObjectParams params({*va}); - return Result(engine, check_type(ifun.eval(ctx, params), types.get_type(fun->root()))); - } -}; - -// expression(tensor,tensor) -struct Expr_TT : Eval { - vespalib::string expr; - Expr_TT(const vespalib::string &expr_in) : expr(expr_in) {} - Result eval(EngineOrFactory engine, const TensorSpec &a, const TensorSpec &b) const override { - auto fun = Function::parse(expr); - auto a_type = ValueType::from_spec(a.type()); - auto b_type = ValueType::from_spec(b.type()); - NodeTypes types(*fun, {a_type, b_type}); - InterpretedFunction ifun(engine, *fun, types); - InterpretedFunction::Context ctx(ifun); - Value::UP va = engine.from_spec(a); - Value::UP vb = engine.from_spec(b); - SimpleObjectParams params({*va,*vb}); - return Result(engine, check_type(ifun.eval(ctx, params), types.get_type(fun->root()))); - } -}; - -const Value &make_value(EngineOrFactory engine, const TensorSpec &spec, Stash &stash) { - return *stash.create<Value::UP>(engine.from_spec(spec)); +TensorSpec eval(const ValueBuilderFactory &factory, const vespalib::string &expr, const std::vector<TensorSpec> ¶ms) { + auto fun = Function::parse(expr); + std::vector<ValueType> param_types; + std::vector<Value::UP> param_values; + std::vector<Value::CREF> param_refs; + for (const auto ¶m: params) { + param_types.push_back(ValueType::from_spec(param.type())); + param_values.push_back(value_from_spec(param, factory)); + param_refs.emplace_back(*param_values.back()); + } + NodeTypes types(*fun, param_types); + const auto &expect_type = types.get_type(fun->root()); + ASSERT_FALSE(expect_type.is_error()); + InterpretedFunction ifun(factory, *fun, types); + InterpretedFunction::Context ctx(ifun); + const Value &result = ifun.eval(ctx, SimpleObjectParams{param_refs}); + EXPECT_EQUAL(result.type(), expect_type); + return spec_from_value(result); } -//----------------------------------------------------------------------------- - -// evaluate tensor reduce operation using tensor engine immediate api -struct ImmediateReduce : Eval { - Aggr aggr; - std::vector<vespalib::string> dimensions; - ImmediateReduce(Aggr aggr_in) : aggr(aggr_in), dimensions() {} - ImmediateReduce(Aggr aggr_in, const vespalib::string &dimension) - : aggr(aggr_in), dimensions({dimension}) {} - Result eval(EngineOrFactory engine, const TensorSpec &a) const override { - Stash stash; - const auto &lhs = make_value(engine, a, stash); - return Result(engine, engine.reduce(lhs, aggr, dimensions, stash)); - } -}; - -// evaluate tensor map operation using tensor engine immediate api -struct ImmediateMap : Eval { - using fun_t = double (*)(double); - fun_t function; - ImmediateMap(fun_t function_in) : function(function_in) {} - Result eval(EngineOrFactory engine, const TensorSpec &a) const override { - Stash stash; - const auto &lhs = make_value(engine, a, stash); - return Result(engine, engine.map(lhs, function, stash)); - } -}; - -// evaluate tensor join operation using tensor engine immediate api -struct ImmediateJoin : Eval { - using fun_t = double (*)(double, double); - fun_t function; - ImmediateJoin(fun_t function_in) : function(function_in) {} - Result eval(EngineOrFactory engine, const TensorSpec &a, const TensorSpec &b) const override { - Stash stash; - const auto &lhs = make_value(engine, a, stash); - const auto &rhs = make_value(engine, b, stash); - return Result(engine, engine.join(lhs, rhs, function, stash)); - } -}; - -// evaluate tensor concat operation using tensor engine immediate api -struct ImmediateConcat : Eval { - vespalib::string dimension; - ImmediateConcat(const vespalib::string &dimension_in) : dimension(dimension_in) {} - Result eval(EngineOrFactory engine, const TensorSpec &a, const TensorSpec &b) const override { - Stash stash; - const auto &lhs = make_value(engine, a, stash); - const auto &rhs = make_value(engine, b, stash); - return Result(engine, engine.concat(lhs, rhs, dimension, stash)); - } -}; +void verify_result(const ValueBuilderFactory &factory, const vespalib::string &expr, const std::vector<TensorSpec> ¶ms, const TensorSpec &expect) { + auto actual = eval(factory, expr, params); + EXPECT_EQUAL(actual, expect); +} -// evaluate tensor rename operation using tensor engine immediate api -struct ImmediateRename : Eval { - std::vector<vespalib::string> from; - std::vector<vespalib::string> to; - ImmediateRename(const std::vector<vespalib::string> &from_in, const std::vector<vespalib::string> &to_in) - : from(from_in), to(to_in) {} - Result eval(EngineOrFactory engine, const TensorSpec &a) const override { - Stash stash; - const auto &lhs = make_value(engine, a, stash); - return Result(engine, engine.rename(lhs, from, to, stash)); - } -}; +void verify_result(const ValueBuilderFactory &factory, const vespalib::string &expr, const std::vector<TensorSpec> ¶ms) { + TEST_DO(verify_result(factory, expr, params, ref_eval(expr, params))); +} //----------------------------------------------------------------------------- // NaN value const double my_nan = std::numeric_limits<double>::quiet_NaN(); -void verify_result(const Eval::Result &result, const Eval::Result &expect) { - if (expect.is_number()) { - EXPECT_EQUAL(result.number(), expect.number()); - } else if (expect.is_tensor()) { - EXPECT_EQUAL(result.tensor(), expect.tensor()); - } else { - TEST_FATAL("expected result should be valid"); - } -} - -void verify_result(const Eval::Result &result, const TensorSpec &expect) { - if (expect.type() == "double") { - EXPECT_EQUAL(result.number(), as_double(expect)); - } else { - EXPECT_EQUAL(result.tensor(), expect); - } -} - uint8_t unhex(char c) { if (c >= '0' && c <= '9') { return (c - '0'); @@ -284,16 +97,15 @@ bool is_same(const nbostream &a, const nbostream &b) { struct TestContext { vespalib::string module_path; - EngineOrFactory ref_engine; - EngineOrFactory engine; + const ValueBuilderFactory &factory; - TestContext(const vespalib::string &module_path_in, EngineOrFactory engine_in) - : module_path(module_path_in), ref_engine(SimpleTensorEngine::ref()), engine(engine_in) {} + TestContext(const vespalib::string &module_path_in, const ValueBuilderFactory &factory_in) + : module_path(module_path_in), factory(factory_in) {} //------------------------------------------------------------------------- void verify_create_type(const vespalib::string &type_spec) { - Value::UP value = engine.from_spec(TensorSpec(type_spec)); + Value::UP value = value_from_spec(TensorSpec(type_spec), factory); EXPECT_EQUAL(type_spec, value->type().to_spec()); } @@ -312,10 +124,6 @@ struct TestContext { //------------------------------------------------------------------------- - void verify_reduce_result(const Eval &eval, const TensorSpec &a, const Eval::Result &expect) { - TEST_DO(verify_result(eval.eval(engine, a), expect)); - } - void test_reduce_op(Aggr aggr, const Sequence &seq) { std::vector<Layout> layouts = { {x(3)}, @@ -333,21 +141,17 @@ struct TestContext { for (const Layout &layout: layouts) { TensorSpec input = spec(layout, seq); for (const Domain &domain: layout) { - Eval::Result expect = ImmediateReduce(aggr, domain.dimension).eval(ref_engine, input); - TEST_STATE(make_string("shape: %s, reduce dimension: %s", - infer_type(layout).c_str(), domain.dimension.c_str()).c_str()); - vespalib::string expr = make_string("reduce(a,%s,%s)", - AggrNames::name_of(aggr)->c_str(), domain.dimension.c_str()); - TEST_DO(verify_reduce_result(Expr_T(expr), input, expect)); - TEST_DO(verify_reduce_result(ImmediateReduce(aggr, domain.dimension), input, expect)); + TEST_STATE(fmt("shape: %s, reduce dimension: %s", + infer_type(layout).c_str(), domain.dimension.c_str()).c_str()); + vespalib::string expr = fmt("reduce(a,%s,%s)", + AggrNames::name_of(aggr)->c_str(), domain.dimension.c_str()); + TEST_DO(verify_result(factory, expr, {input})); } { - Eval::Result expect = ImmediateReduce(aggr).eval(ref_engine, input); - TEST_STATE(make_string("shape: %s, reduce all dimensions", - infer_type(layout).c_str()).c_str()); - vespalib::string expr = make_string("reduce(a,%s)", AggrNames::name_of(aggr)->c_str()); - TEST_DO(verify_reduce_result(Expr_T(expr), input, expect)); - TEST_DO(verify_reduce_result(ImmediateReduce(aggr), input, expect)); + TEST_STATE(fmt("shape: %s, reduce all dimensions", + infer_type(layout).c_str()).c_str()); + vespalib::string expr = fmt("reduce(a,%s)", AggrNames::name_of(aggr)->c_str()); + TEST_DO(verify_result(factory, expr, {input})); } } } @@ -364,7 +168,7 @@ struct TestContext { //------------------------------------------------------------------------- - void test_map_op(const Eval &eval, map_fun_t ref_op, const Sequence &seq) { + void test_map_op_inner(const vespalib::string &expr, map_fun_t ref_op, const Sequence &seq) { std::vector<Layout> layouts = { {}, {x(3)}, @@ -380,14 +184,13 @@ struct TestContext { float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) }; for (const Layout &layout: layouts) { - TEST_DO(verify_result(eval.eval(engine, spec(layout, seq)), spec(layout, OpSeq(seq, ref_op)))); + TEST_DO(verify_result(factory, expr, {spec(layout, seq)}, spec(layout, OpSeq(seq, ref_op)))); } } void test_map_op(const vespalib::string &expr, map_fun_t op, const Sequence &seq) { - TEST_DO(test_map_op(ImmediateMap(op), op, seq)); - TEST_DO(test_map_op(Expr_T(expr), op, seq)); - TEST_DO(test_map_op(Expr_T(make_string("map(x,f(a)(%s))", expr.c_str())), op, seq)); + TEST_DO(test_map_op_inner(expr, op, seq)); + TEST_DO(test_map_op_inner(fmt("map(x,f(a)(%s))", expr.c_str()), op, seq)); } void test_tensor_map() { @@ -420,29 +223,29 @@ struct TestContext { //------------------------------------------------------------------------- - void test_apply_op(const Eval &eval, + void test_apply_op(const vespalib::string &expr, const TensorSpec &expect, const TensorSpec &lhs, const TensorSpec &rhs) { - TEST_DO(verify_result(safe(eval).eval(engine, lhs, rhs), expect)); + TEST_DO(verify_result(factory, expr, {lhs, rhs}, expect)); } - void test_fixed_sparse_cases_apply_op(const Eval &eval, + void test_fixed_sparse_cases_apply_op(const vespalib::string &expr, join_fun_t op) { - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{}", {}), spec("x{}", { { {{"x","1"}}, 3 } }), spec("x{}", { { {{"x","2"}}, 5 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{}", { { {{"x","1"}}, op(3,5) } }), spec("x{}", { { {{"x","1"}}, 3 } }), spec("x{}", { { {{"x","1"}}, 5 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{}", { { {{"x","1"}}, op(3,-5) } }), spec("x{}", { { {{"x","1"}}, 3 } }), spec("x{}", { { {{"x","1"}}, -5 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{},z{}", { { {{"x","-"},{"y","2"},{"z","-"}}, op(5,7) }, @@ -454,7 +257,7 @@ struct TestContext { spec("y{},z{}", { { {{"y","-"},{"z","3"}}, 11 }, { {{"y","2"},{"z","-"}}, 7 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{},z{}", { { {{"x","-"},{"y","2"},{"z","-"}}, op(7,5) }, @@ -466,7 +269,7 @@ struct TestContext { spec("x{},y{}", { { {{"x","-"},{"y","2"}}, 5 }, { {{"x","1"},{"y","-"}}, 3 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("y{},z{}", { { {{"y","2"},{"z","-"}}, op(5,7) } }), @@ -474,7 +277,7 @@ struct TestContext { spec("y{},z{}", { { {{"y","-"},{"z","3"}}, 11 }, { {{"y","2"},{"z","-"}}, 7 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("y{},z{}", { { {{"y","2"},{"z","-"}}, op(7,5) } }), @@ -482,7 +285,7 @@ struct TestContext { { { {{"y","-"},{"z","3"}}, 11 }, { {{"y","2"},{"z","-"}}, 7 } }), spec("y{}", { { {{"y","2"}}, 5 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{}", { { {{"x","-"},{"y","2"}}, op(5,7) } }), @@ -490,7 +293,7 @@ struct TestContext { { { {{"x","-"},{"y","2"}}, 5 }, { {{"x","1"},{"y","-"}}, 3 } }), spec("y{}", { { {{"y","2"}}, 7 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{}", { { {{"x","-"},{"y","2"}}, op(7,5) } }), @@ -498,19 +301,19 @@ struct TestContext { spec("x{},y{}", { { {{"x","-"},{"y","2"}}, 5 }, { {{"x","1"},{"y","-"}}, 3 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},z{}", { { {{"x","1"},{"z","3"}}, op(3,11) } }), spec("x{}", { { {{"x","1"}}, 3 } }), spec("z{}", { { {{"z","3"}}, 11 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},z{}", { { {{"x","1"},{"z","3"}}, op(11,3) } }), spec("z{}",{ { {{"z","3"}}, 11 } }), spec("x{}",{ { {{"x","1"}}, 3 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{}", { { {{"x","1"},{"y","1"}}, op(3,5) }, @@ -521,7 +324,7 @@ struct TestContext { { {{"x","2"}}, 7 } }), spec("y{}", { { {{"y","1"}}, 5 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{},z{}", { { {{"x","1"},{"y","1"},{"z","1"}}, op(1,7) }, @@ -541,7 +344,7 @@ struct TestContext { { { {{"y","1"},{"z","1"}}, 7 }, { {{"y","1"},{"z","2"}}, 13 }, { {{"y","2"},{"z","1"}}, 11 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{},z{}", { { {{"x","1"},{"y","1"},{"z","1"}}, op(1,7) } }), @@ -550,7 +353,7 @@ struct TestContext { { {{"x","1"},{"y","1"}}, 1 } }), spec("y{},z{}", { { {{"y","1"},{"z","1"}}, 7 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{},z{}", { { {{"x","1"},{"y","-"},{"z","1"}}, op(5,11) }, @@ -562,7 +365,7 @@ struct TestContext { spec("y{},z{}", { { {{"y","-"},{"z","1"}}, 11 }, { {{"y","1"},{"z","1"}}, 7 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{},z{}", { { {{"x","1"},{"y","1"},{"z","1"}}, op(1,7) } }), @@ -571,7 +374,7 @@ struct TestContext { { {{"x","1"},{"y","1"}}, 1 } }), spec("y{},z{}", { { {{"y","1"},{"z","1"}}, 7 } }))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec("x{},y{},z{}", { { {{"x","-"},{"y","-"},{"z", "-"}}, op(5,11) }, @@ -585,28 +388,28 @@ struct TestContext { { {{"y","1"},{"z","1"}}, 7 } }))); } - void test_fixed_dense_cases_apply_op(const Eval &eval, + void test_fixed_dense_cases_apply_op(const vespalib::string &expr, join_fun_t op) { - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec(op(0.1,0.2)), spec(0.1), spec(0.2))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec(x(1), Seq({ op(3,5) })), spec(x(1), Seq({ 3 })), spec(x(1), Seq({ 5 })))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec(x(1), Seq({ op(3,-5) })), spec(x(1), Seq({ 3 })), spec(x(1), Seq({ -5 })))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec(x(2), Seq({ op(3,7), op(5,11) })), spec(x(2), Seq({ 3, 5 })), spec(x(2), Seq({ 7, 11 })))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec({x(1),y(1)}, Seq({ op(3,5) })), spec({x(1),y(1)}, Seq({ 3 })), spec({x(1),y(1)}, Seq({ 5 })))); - TEST_DO(test_apply_op(eval, + TEST_DO(test_apply_op(expr, spec({x(2),y(2),z(2)}, Seq({ op(1, 7), op(1, 11), op(2, 13), op(2, 17), @@ -621,7 +424,7 @@ struct TestContext { 13, 17 })))); } - void test_apply_op(const Eval &eval, join_fun_t op, const Sequence &seq) { + void test_apply_op_inner(const vespalib::string &expr, join_fun_t op, const Sequence &seq) { std::vector<Layout> layouts = { {}, {}, {x(5)}, {x(5)}, @@ -651,20 +454,18 @@ struct TestContext { for (size_t i = 0; i < layouts.size(); i += 2) { TensorSpec lhs_input = spec(layouts[i], seq); TensorSpec rhs_input = spec(layouts[i + 1], seq); - TEST_STATE(make_string("lhs shape: %s, rhs shape: %s", - lhs_input.type().c_str(), - rhs_input.type().c_str()).c_str()); - Eval::Result expect = ImmediateJoin(op).eval(ref_engine, lhs_input, rhs_input); - TEST_DO(verify_result(safe(eval).eval(engine, lhs_input, rhs_input), expect)); + TEST_STATE(fmt("lhs shape: %s, rhs shape: %s", + lhs_input.type().c_str(), + rhs_input.type().c_str()).c_str()); + TEST_DO(verify_result(factory, expr, {lhs_input, rhs_input})); } - TEST_DO(test_fixed_sparse_cases_apply_op(eval, op)); - TEST_DO(test_fixed_dense_cases_apply_op(eval, op)); + TEST_DO(test_fixed_sparse_cases_apply_op(expr, op)); + TEST_DO(test_fixed_dense_cases_apply_op(expr, op)); } void test_apply_op(const vespalib::string &expr, join_fun_t op, const Sequence &seq) { - TEST_DO(test_apply_op(ImmediateJoin(op), op, seq)); - TEST_DO(test_apply_op(Expr_TT(expr), op, seq)); - TEST_DO(test_apply_op(Expr_TT(make_string("join(x,y,f(a,b)(%s))", expr.c_str())), op, seq)); + TEST_DO(test_apply_op_inner(expr, op, seq)); + TEST_DO(test_apply_op_inner(fmt("join(x,y,f(a,b)(%s))", expr.c_str()), op, seq)); } void test_tensor_apply() { @@ -697,8 +498,8 @@ struct TestContext { const TensorSpec &lhs, const TensorSpec &rhs) { - Expr_TT eval("reduce(a*b,sum)"); - TEST_DO(verify_result(safe(eval).eval(engine, lhs, rhs), spec(expect))); + vespalib::string expr("reduce(a*b,sum)"); + TEST_DO(verify_result(factory, expr, {lhs, rhs}, spec(expect))); } void test_dot_product(double expect, @@ -724,10 +525,8 @@ struct TestContext { const vespalib::string &dimension, const TensorSpec &expect) { - vespalib::string expr = make_string("concat(a,b,%s)", dimension.c_str()); - ImmediateConcat eval(dimension); - TEST_DO(verify_result(eval.eval(engine, a, b), expect)); - TEST_DO(verify_result(Expr_TT(expr).eval(engine, a, b), expect)); + vespalib::string expr = fmt("concat(a,b,%s)", dimension.c_str()); + TEST_DO(verify_result(factory, expr, {a, b}, expect)); } void test_concat() { @@ -760,29 +559,25 @@ struct TestContext { void test_rename(const vespalib::string &expr, const TensorSpec &input, - const std::vector<vespalib::string> &from, - const std::vector<vespalib::string> &to, const TensorSpec &expect) { - ImmediateRename eval(from, to); - TEST_DO(verify_result(eval.eval(engine, input), expect)); - TEST_DO(verify_result(Expr_T(expr).eval(engine, input), expect)); + TEST_DO(verify_result(factory, expr, {input}, expect)); } void test_rename() { - TEST_DO(test_rename("rename(a,x,y)", spec(x(5), N()), {"x"}, {"y"}, spec(y(5), N()))); - TEST_DO(test_rename("rename(a,y,x)", spec({y(5),z(5)}, N()), {"y"}, {"x"}, spec({x(5),z(5)}, N()))); - TEST_DO(test_rename("rename(a,y,x)", spec(float_cells({y(5),z(5)}), N()), {"y"}, {"x"}, spec(float_cells({x(5),z(5)}), N()))); - TEST_DO(test_rename("rename(a,z,x)", spec({y(5),z(5)}, N()), {"z"}, {"x"}, spec({y(5),x(5)}, N()))); - TEST_DO(test_rename("rename(a,x,z)", spec({x(5),y(5)}, N()), {"x"}, {"z"}, spec({z(5),y(5)}, N()))); - TEST_DO(test_rename("rename(a,y,z)", spec({x(5),y(5)}, N()), {"y"}, {"z"}, spec({x(5),z(5)}, N()))); - TEST_DO(test_rename("rename(a,(x,y),(y,x))", spec({x(5),y(5)}, N()), {"x","y"}, {"y","x"}, spec({y(5),x(5)}, N()))); + TEST_DO(test_rename("rename(a,x,y)", spec(x(5), N()), spec(y(5), N()))); + TEST_DO(test_rename("rename(a,y,x)", spec({y(5),z(5)}, N()), spec({x(5),z(5)}, N()))); + TEST_DO(test_rename("rename(a,y,x)", spec(float_cells({y(5),z(5)}), N()), spec(float_cells({x(5),z(5)}), N()))); + TEST_DO(test_rename("rename(a,z,x)", spec({y(5),z(5)}, N()), spec({y(5),x(5)}, N()))); + TEST_DO(test_rename("rename(a,x,z)", spec({x(5),y(5)}, N()), spec({z(5),y(5)}, N()))); + TEST_DO(test_rename("rename(a,y,z)", spec({x(5),y(5)}, N()), spec({x(5),z(5)}, N()))); + TEST_DO(test_rename("rename(a,(x,y),(y,x))", spec({x(5),y(5)}, N()), spec({y(5),x(5)}, N()))); } //------------------------------------------------------------------------- void test_tensor_lambda(const vespalib::string &expr, const TensorSpec &expect) { - TEST_DO(verify_result(Expr_V(expr).eval(engine), expect)); + TEST_DO(verify_result(factory, expr, {}, expect)); } void test_tensor_lambda() { @@ -800,7 +595,7 @@ struct TestContext { //------------------------------------------------------------------------- void test_tensor_create(const vespalib::string &expr, double a, double b, const TensorSpec &expect) { - TEST_DO(verify_result(Expr_TT(expr).eval(engine, spec(a), spec(b)), expect)); + TEST_DO(verify_result(factory, expr, {spec(a), spec(b)}, expect)); } void test_tensor_create() { @@ -813,7 +608,7 @@ struct TestContext { //------------------------------------------------------------------------- void test_tensor_peek(const vespalib::string &expr, const TensorSpec ¶m, const TensorSpec &expect) { - TEST_DO(verify_result(Expr_TT(expr).eval(engine, param, spec(1.0)), expect)); + TEST_DO(verify_result(factory, expr, {param, spec(1.0)}, expect)); } void test_tensor_peek() { @@ -838,13 +633,13 @@ struct TestContext { for (bool a_float: {false, true}) { for (bool b_float: {false, true}) { bool both_float = a_float && b_float; - vespalib::string a_expr = make_string("tensor%s(%s):%s", a_float ? "<float>" : "", type_base.c_str(), a_str.c_str()); - vespalib::string b_expr = make_string("tensor%s(%s):%s", b_float ? "<float>" : "", type_base.c_str(), b_str.c_str()); - vespalib::string expect_expr = make_string("tensor%s(%s):%s", both_float ? "<float>" : "", type_base.c_str(), expect_str.c_str()); + vespalib::string a_expr = fmt("tensor%s(%s):%s", a_float ? "<float>" : "", type_base.c_str(), a_str.c_str()); + vespalib::string b_expr = fmt("tensor%s(%s):%s", b_float ? "<float>" : "", type_base.c_str(), b_str.c_str()); + vespalib::string expect_expr = fmt("tensor%s(%s):%s", both_float ? "<float>" : "", type_base.c_str(), expect_str.c_str()); TensorSpec a = spec(a_expr); TensorSpec b = spec(b_expr); TensorSpec expect = spec(expect_expr); - TEST_DO(verify_result(Expr_TT(expr).eval(engine, a, b), expect)); + TEST_DO(verify_result(factory, expr, {a, b}, expect)); } } } @@ -858,19 +653,22 @@ struct TestContext { //------------------------------------------------------------------------- void verify_encode_decode(const TensorSpec &spec, - EngineOrFactory encode_engine, - EngineOrFactory decode_engine) + const ValueBuilderFactory &encode_factory, + const ValueBuilderFactory &decode_factory) { - Stash stash; nbostream data; - encode_engine.encode(make_value(encode_engine, spec, stash), data); - TEST_DO(verify_result(Eval::Result(decode_engine, *decode_engine.decode(data)), spec)); + auto value = value_from_spec(spec, encode_factory); + encode_value(*value, data); + auto value2 = decode_value(data, decode_factory); + TensorSpec spec2 = spec_from_value(*value2); + EXPECT_EQUAL(spec2, spec); } void verify_encode_decode(const TensorSpec &spec) { - TEST_DO(verify_encode_decode(spec, engine, ref_engine)); - if (&engine != &ref_engine) { - TEST_DO(verify_encode_decode(spec, ref_engine, engine)); + const ValueBuilderFactory &simple = SimpleValueBuilderFactory::get(); + TEST_DO(verify_encode_decode(spec, factory, simple)); + if (&factory != &simple) { + TEST_DO(verify_encode_decode(spec, simple, factory)); } } @@ -880,13 +678,13 @@ struct TestContext { const Inspector &binary = test["binary"]; EXPECT_GREATER(binary.entries(), 0u); nbostream encoded; - engine.encode(make_value(engine, spec, stash), encoded); + encode_value(*value_from_spec(spec, factory), encoded); test.setData("encoded", Memory(encoded.peek(), encoded.size())); bool matched_encode = false; for (size_t i = 0; i < binary.entries(); ++i) { nbostream data = extract_data(binary[i].asString()); matched_encode = (matched_encode || is_same(encoded, data)); - TEST_DO(verify_result(Eval::Result(engine, *engine.decode(data)), spec)); + EXPECT_EQUAL(spec_from_value(*decode_value(data, factory)), spec); EXPECT_EQUAL(data.size(), 0u); } EXPECT_TRUE(matched_encode); @@ -946,16 +744,14 @@ struct TestContext { } }; -} // namespace vespalib::eval::test::<unnamed> +} // <unnamed> void -TensorConformance::run_tests(const vespalib::string &module_path, EngineOrFactory engine) +TensorConformance::run_tests(const vespalib::string &module_path, const ValueBuilderFactory &factory) { - TestContext ctx(module_path, engine); + TestContext ctx(module_path, factory); fprintf(stderr, "module path: '%s'\n", ctx.module_path.c_str()); ctx.run_tests(); } -} // namespace vespalib::eval::test -} // namespace vespalib::eval -} // namespace vespalib +} // namespace diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.h b/eval/src/vespa/eval/eval/test/tensor_conformance.h index 9aed3840191..ad30b60e24c 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.h +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.h @@ -2,21 +2,17 @@ #pragma once -#include <vespa/eval/eval/engine_or_factory.h> +#include <vespa/eval/eval/value.h> #include <vespa/vespalib/stllike/string.h> -namespace vespalib { -namespace eval { -namespace test { +namespace vespalib::eval::test { /** * A collection of tensor-related tests that can be run for various - * implementations of the TensorEngine interface. + * implementations. **/ struct TensorConformance { - static void run_tests(const vespalib::string &module_path, EngineOrFactory engine); + static void run_tests(const vespalib::string &module_path, const ValueBuilderFactory &factory); }; -} // namespace vespalib::eval::test -} // namespace vespalib::eval -} // namespace vespalib +} // namespace diff --git a/eval/src/vespa/eval/eval/test/tensor_model.hpp b/eval/src/vespa/eval/eval/test/tensor_model.hpp index 78d6798ac4c..2f8c0d68946 100644 --- a/eval/src/vespa/eval/eval/test/tensor_model.hpp +++ b/eval/src/vespa/eval/eval/test/tensor_model.hpp @@ -6,11 +6,9 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/operation.h> -#include <vespa/eval/eval/tensor_engine.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/node_types.h> #include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/simple_tensor_engine.h> namespace vespalib { namespace eval { @@ -298,21 +296,7 @@ TensorSpec spec(const vespalib::string &type, } TensorSpec spec(const vespalib::string &value_expr) { - if (value_expr == "error") { - return TensorSpec("error"); - } - const auto &engine = SimpleTensorEngine::ref(); - auto fun = Function::parse(value_expr); - ASSERT_TRUE(!fun->has_error()); - ASSERT_EQUAL(fun->num_params(), 0u); - NodeTypes types(*fun, {}); - ASSERT_TRUE(!types.get_type(fun->root()).is_error()); - InterpretedFunction ifun(engine, *fun, types); - InterpretedFunction::Context ctx(ifun); - SimpleObjectParams params({}); - auto result = engine.to_spec(ifun.eval(ctx, params)); - ASSERT_TRUE(!result.cells().empty()); - return result; + return TensorSpec::from_expr(value_expr); } } // namespace vespalib::eval::test |