diff options
25 files changed, 889 insertions, 83 deletions
diff --git a/README.md b/README.md index 97c4bf69bc3..89bd7d95e59 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ You can also setup CentOS 7 natively and install the following build dependencie ### Build Java modules - export MAVEN_OPTS="-Xms128m -Xmx512m" + export MAVEN_OPTS="-Xms128m -Xmx1024m" source /opt/rh/rh-maven33/enable bash bootstrap.sh java mvn -T <num-threads> install diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 0526c2f6581..81fb1b29162 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -205,6 +205,13 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-search</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- OPTIMIZATION: very large (44 MB) and only used for query sorting --> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 1226b3bbbbe..8df28095e51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -9,7 +9,7 @@ import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.vespa.athenz.utils.AthenzIdentityVerifier; +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzSslContextProvider; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneList; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index 72fe61d7107..33c303f9574 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -98,6 +98,21 @@ TensorSpec eval_expr(const Inspector &test, const TensorEngine &engine, bool typ return engine.to_spec(ifun.eval(ctx, params)); } +TensorSpec eval_expr_tf(const Inspector &test, const TensorEngine &engine) { + Stash stash; + Function fun = Function::parse(test["expression"].asString().make_string()); + std::vector<Value::UP> param_values; + std::vector<Value::CREF> param_refs; + for (size_t i = 0; i < fun.num_params(); ++i) { + param_values.emplace_back(engine.from_spec(extract_value(test["inputs"][fun.param_name(i)]))); + param_refs.emplace_back(*param_values.back()); + } + SimpleObjectParams params(param_refs); + NodeTypes types = NodeTypes(fun, get_types(param_values)); + const auto &tfun = make_tensor_function(engine, fun.root(), types, stash); + return engine.to_spec(tfun.eval(engine, params, stash)); +} + //----------------------------------------------------------------------------- std::vector<vespalib::string> extract_fields(const Inspector &object) { @@ -164,6 +179,8 @@ void evaluate(Input &in, Output &out) { eval_expr(slime.get(), DefaultTensorEngine::ref(), false)); insert_value(slime["result"], "cpp_ref_typed", eval_expr(slime.get(), SimpleTensorEngine::ref(), true)); + insert_value(slime["result"], "cpp_tensor_function", + eval_expr_tf(slime.get(), DefaultTensorEngine::ref())); write_compact(slime, out); }; auto handle_summary = [&out](Slime &slime) diff --git a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp index 91eb741f334..802f9555360 100644 --- a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp +++ b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp @@ -52,13 +52,27 @@ struct MyEvalTest : test::EvalSpec::EvalTest { if (is_supported && !has_issues) { vespalib::string desc = as_string(param_names, param_values, expression); SimpleParams params(param_values); - verify_result(SimpleTensorEngine::ref(), function, false, "[untyped simple] "+desc, params, expected_result); - verify_result(DefaultTensorEngine::ref(), function, false, "[untyped prod] "+desc, params, expected_result); - verify_result(DefaultTensorEngine::ref(), function, true, "[typed prod] "+desc, params, expected_result); + verify_result(SimpleTensorEngine::ref(), function, false, "[untyped simple] "+desc, params, expected_result); + verify_result(DefaultTensorEngine::ref(), function, false, "[untyped prod] "+desc, params, expected_result); + verify_result(DefaultTensorEngine::ref(), function, true, "[typed prod] "+desc, params, expected_result); + verify_tensor_function(DefaultTensorEngine::ref(), function, "[tensor function]"+desc, params, expected_result); } } - void verify_result(const TensorEngine& engine, + void report_result(bool is_double, double result, double expect, const vespalib::string &desc) + { + if (is_double && is_same(expect, result)) { + print_pass && fprintf(stderr, "verifying: %s -> %g ... PASS\n", + desc.c_str(), expect); + ++pass_cnt; + } else { + print_fail && fprintf(stderr, "verifying: %s -> %g ... FAIL: got %g\n", + desc.c_str(), expect, result); + ++fail_cnt; + } + } + + void verify_result(const TensorEngine &engine, const Function &function, bool typed, const vespalib::string &description, @@ -72,18 +86,20 @@ struct MyEvalTest : test::EvalSpec::EvalTest { ASSERT_EQUAL(ifun.num_params(), params.params.size()); InterpretedFunction::Context ictx(ifun); const Value &result_value = ifun.eval(ictx, params); - double result = result_value.as_double(); - if (result_value.is_double() && is_same(expected_result, result)) { - print_pass && fprintf(stderr, "verifying: %s -> %g ... PASS\n", - description.c_str(), - expected_result); - ++pass_cnt; - } else { - print_fail && fprintf(stderr, "verifying: %s -> %g ... FAIL: got %g\n", - description.c_str(), - expected_result, result); - ++fail_cnt; - } + report_result(result_value.is_double(), result_value.as_double(), expected_result, description); + } + + void verify_tensor_function(const TensorEngine &engine, + const Function &function, + const vespalib::string &description, + const SimpleParams ¶ms, + double expected_result) + { + Stash stash; + NodeTypes node_types = NodeTypes(function, std::vector<ValueType>(params.params.size(), ValueType::double_type())); + const auto &tfun = make_tensor_function(engine, function.root(), node_types, stash); + const Value &result_value = tfun.eval(engine, params, stash); + report_result(result_value.is_double(), result_value.as_double(), expected_result, description); } }; diff --git a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp index a5dbf390562..4e52fd8b47b 100644 --- a/eval/src/tests/eval/tensor_function/tensor_function_test.cpp +++ b/eval/src/tests/eval/tensor_function/tensor_function_test.cpp @@ -27,13 +27,35 @@ struct EvalCtx { tensors.push_back(std::move(tensor)); return id; } + void replace_tensor(size_t idx, Value::UP tensor) { + params[idx] = *tensor; + tensors[idx] = std::move(tensor); + } const Value &eval(const TensorFunction &fun) { - return fun.eval(SimpleObjectParams(params), stash); + return fun.eval(engine, SimpleObjectParams(params), stash); } const TensorFunction &compile(const tensor_function::Node &expr) { return engine.compile(expr, stash); } - Value::UP make_tensor_inject() { + Value::UP make_true() { + return engine.from_spec(TensorSpec("double").add({}, 1.0)); + } + Value::UP make_false() { + return engine.from_spec(TensorSpec("double").add({}, 0.0)); + } + Value::UP make_tensor_matrix_first_half() { + return engine.from_spec( + TensorSpec("tensor(x[2])") + .add({{"x", 0}}, 1.0) + .add({{"x", 1}}, 3.0)); + } + Value::UP make_tensor_matrix_second_half() { + return engine.from_spec( + TensorSpec("tensor(x[2])") + .add({{"x", 0}}, 2.0) + .add({{"x", 1}}, 4.0)); + } + Value::UP make_tensor_matrix() { return engine.from_spec( TensorSpec("tensor(x[2],y[2])") .add({{"x", 0}, {"y", 0}}, 1.0) @@ -41,6 +63,14 @@ struct EvalCtx { .add({{"x", 1}, {"y", 0}}, 3.0) .add({{"x", 1}, {"y", 1}}, 4.0)); } + Value::UP make_tensor_matrix_renamed() { + return engine.from_spec( + TensorSpec("tensor(y[2],z[2])") + .add({{"z", 0}, {"y", 0}}, 1.0) + .add({{"z", 0}, {"y", 1}}, 2.0) + .add({{"z", 1}, {"y", 0}}, 3.0) + .add({{"z", 1}, {"y", 1}}, 4.0)); + } Value::UP make_tensor_reduce_input() { return engine.from_spec( TensorSpec("tensor(x[3],y[2])") @@ -72,21 +102,21 @@ struct EvalCtx { .add({{"x","2"},{"y","1"}}, 3) .add({{"x","1"},{"y","2"}}, -5)); } - Value::UP make_tensor_apply_lhs() { + Value::UP make_tensor_join_lhs() { return engine.from_spec( TensorSpec("tensor(x{},y{})") .add({{"x","1"},{"y","1"}}, 1) .add({{"x","2"},{"y","1"}}, 3) .add({{"x","1"},{"y","2"}}, 5)); } - Value::UP make_tensor_apply_rhs() { + Value::UP make_tensor_join_rhs() { return engine.from_spec( TensorSpec("tensor(y{},z{})") .add({{"y","1"},{"z","1"}}, 7) .add({{"y","2"},{"z","1"}}, 11) .add({{"y","1"},{"z","2"}}, 13)); } - Value::UP make_tensor_apply_output() { + Value::UP make_tensor_join_output() { return engine.from_spec( TensorSpec("tensor(x{},y{},z{})") .add({{"x","1"},{"y","1"},{"z","1"}}, 7) @@ -108,10 +138,20 @@ void verify_equal(const Value &expect, const Value &value) { EXPECT_EQUAL(expect_spec, value_spec); } +TEST("require that const_value works") { + EvalCtx ctx(SimpleTensorEngine::ref()); + Value::UP my_const = ctx.make_tensor_matrix(); + Value::UP expect = ctx.make_tensor_matrix(); + const auto &fun = const_value(*my_const, ctx.stash); + EXPECT_EQUAL(expect->type(), fun.result_type()); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect, ctx.eval(prog))); +} + TEST("require that tensor injection works") { EvalCtx ctx(SimpleTensorEngine::ref()); - size_t a_id = ctx.add_tensor(ctx.make_tensor_inject()); - Value::UP expect = ctx.make_tensor_inject(); + size_t a_id = ctx.add_tensor(ctx.make_tensor_matrix()); + Value::UP expect = ctx.make_tensor_matrix(); const auto &fun = inject(ValueType::from_spec("tensor(x[2],y[2])"), a_id, ctx.stash); EXPECT_EQUAL(expect->type(), fun.result_type()); const auto &prog = ctx.compile(fun); @@ -151,9 +191,9 @@ TEST("require that tensor map works") { TEST("require that tensor join works") { EvalCtx ctx(SimpleTensorEngine::ref()); - size_t a_id = ctx.add_tensor(ctx.make_tensor_apply_lhs()); - size_t b_id = ctx.add_tensor(ctx.make_tensor_apply_rhs()); - Value::UP expect = ctx.make_tensor_apply_output(); + size_t a_id = ctx.add_tensor(ctx.make_tensor_join_lhs()); + size_t b_id = ctx.add_tensor(ctx.make_tensor_join_rhs()); + Value::UP expect = ctx.make_tensor_join_output(); const auto &fun = join(inject(ValueType::from_spec("tensor(x{},y{})"), a_id, ctx.stash), inject(ValueType::from_spec("tensor(y{},z{})"), b_id, ctx.stash), operation::Mul::f, ctx.stash); @@ -162,13 +202,73 @@ TEST("require that tensor join works") { TEST_DO(verify_equal(*expect, ctx.eval(prog))); } +TEST("require that tensor concat works") { + EvalCtx ctx(SimpleTensorEngine::ref()); + size_t a_id = ctx.add_tensor(ctx.make_tensor_matrix_first_half()); + size_t b_id = ctx.add_tensor(ctx.make_tensor_matrix_second_half()); + Value::UP expect = ctx.make_tensor_matrix(); + const auto &fun = concat(inject(ValueType::from_spec("tensor(x[2])"), a_id, ctx.stash), + inject(ValueType::from_spec("tensor(x[2])"), b_id, ctx.stash), + "y", ctx.stash); + EXPECT_EQUAL(expect->type(), fun.result_type()); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect, ctx.eval(prog))); +} + +TEST("require that tensor rename works") { + EvalCtx ctx(SimpleTensorEngine::ref()); + size_t a_id = ctx.add_tensor(ctx.make_tensor_matrix()); + Value::UP expect = ctx.make_tensor_matrix_renamed(); + const auto &fun = rename(inject(ValueType::from_spec("tensor(x[2],y[2])"), a_id, ctx.stash), + {"x"}, {"z"}, ctx.stash); + EXPECT_EQUAL(expect->type(), fun.result_type()); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect, ctx.eval(prog))); +} + +TEST("require that if_node works") { + EvalCtx ctx(SimpleTensorEngine::ref()); + size_t a_id = ctx.add_tensor(ctx.make_true()); + size_t b_id = ctx.add_tensor(ctx.make_tensor_matrix_first_half()); + size_t c_id = ctx.add_tensor(ctx.make_tensor_matrix_second_half()); + Value::UP expect_true = ctx.make_tensor_matrix_first_half(); + Value::UP expect_false = ctx.make_tensor_matrix_second_half(); + const auto &fun = if_node(inject(ValueType::double_type(), a_id, ctx.stash), + inject(ValueType::from_spec("tensor(x[2])"), b_id, ctx.stash), + inject(ValueType::from_spec("tensor(x[2])"), c_id, ctx.stash), ctx.stash); + EXPECT_EQUAL(expect_true->type(), fun.result_type()); + const auto &prog = ctx.compile(fun); + TEST_DO(verify_equal(*expect_true, ctx.eval(prog))); + ctx.replace_tensor(a_id, ctx.make_false()); + TEST_DO(verify_equal(*expect_false, ctx.eval(prog))); +} + +TEST("require that if_node gets expected result type") { + Stash stash; + const Node &a = inject(DoubleValue::double_type(), 0, stash); + const Node &b = inject(ValueType::from_spec("tensor(x[2])"), 0, stash); + const Node &c = inject(ValueType::from_spec("tensor(x[3])"), 0, stash); + const Node &d = inject(ValueType::from_spec("tensor(x[])"), 0, stash); + const Node &e = inject(ValueType::from_spec("tensor(y[3])"), 0, stash); + const Node &if_same = if_node(a, b, b, stash); + const Node &if_similar = if_node(a, b, c, stash); + const Node &if_subtype = if_node(a, b, d, stash); + const Node &if_different = if_node(a, b, e, stash); + EXPECT_EQUAL(if_same.result_type(), ValueType::from_spec("tensor(x[2])")); + EXPECT_EQUAL(if_similar.result_type(), ValueType::from_spec("any")); + EXPECT_EQUAL(if_subtype.result_type(), ValueType::from_spec("any")); + EXPECT_EQUAL(if_different.result_type(), ValueType::from_spec("any")); +} + TEST("require that push_children works") { Stash stash; std::vector<Node::Child::CREF> refs; - const Node &a = inject(ValueType::double_type(), 0, stash); - const Node &b = inject(ValueType::double_type(), 1, stash); + const Node &a = inject(DoubleValue::double_type(), 0, stash); + const Node &b = inject(DoubleValue::double_type(), 1, stash); + const Node &c = const_value(stash.create<DoubleValue>(1.0), stash); a.push_children(refs); b.push_children(refs); + c.push_children(refs); ASSERT_EQUAL(refs.size(), 0u); //------------------------------------------------------------------------- reduce(a, Aggr::SUM, {}, stash).push_children(refs); @@ -184,6 +284,21 @@ TEST("require that push_children works") { EXPECT_EQUAL(&refs[2].get().get(), &a); EXPECT_EQUAL(&refs[3].get().get(), &b); //------------------------------------------------------------------------- + concat(a, b, "x", stash).push_children(refs); + ASSERT_EQUAL(refs.size(), 6u); + EXPECT_EQUAL(&refs[4].get().get(), &a); + EXPECT_EQUAL(&refs[5].get().get(), &b); + //------------------------------------------------------------------------- + rename(c, {}, {}, stash).push_children(refs); + ASSERT_EQUAL(refs.size(), 7u); + EXPECT_EQUAL(&refs[6].get().get(), &c); + //------------------------------------------------------------------------- + if_node(a, b, c, stash).push_children(refs); + ASSERT_EQUAL(refs.size(), 10u); + EXPECT_EQUAL(&refs[7].get().get(), &a); + EXPECT_EQUAL(&refs[8].get().get(), &b); + EXPECT_EQUAL(&refs[9].get().get(), &c); + //------------------------------------------------------------------------- } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp index 3463bee3447..8b4b1497243 100644 --- a/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_dot_product_function/dense_dot_product_function_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/eval/eval/tensor_function.h> +#include <vespa/eval/tensor/default_tensor_engine.h> #include <vespa/eval/tensor/dense/dense_dot_product_function.h> #include <vespa/eval/tensor/dense/dense_tensor.h> #include <vespa/eval/tensor/dense/dense_tensor_builder.h> @@ -78,7 +79,7 @@ struct Fixture ~Fixture(); double eval() const { Stash stash; - const Value &result = function.eval(input.get(), stash); + const Value &result = function.eval(DefaultTensorEngine::ref(), input.get(), stash); ASSERT_TRUE(result.is_double()); LOG(info, "eval(): (%s) * (%s) = %f", input.param(0).type().to_spec().c_str(), diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp index f27a2073159..5e18df10921 100644 --- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp +++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp @@ -52,14 +52,14 @@ void verify_result(const TensorSpec &v, const TensorSpec &m, bool happy) { prod_vec->type().dimensions()[0].size, expect.type().dimensions()[0].size, happy); - const Value &actual1 = fun1.eval(wrap({*prod_vec, *prod_mat}), stash); + const Value &actual1 = fun1.eval(prod_engine, wrap({*prod_vec, *prod_mat}), stash); TEST_DO(verify_equal(expect, actual1)); DenseXWProductFunction fun2(expect.type(), 1, 0, prod_vec->type().dimensions()[0].size, expect.type().dimensions()[0].size, happy); - const Value &actual2 = fun2.eval(wrap({*prod_mat, *prod_vec}), stash); + const Value &actual2 = fun2.eval(prod_engine, wrap({*prod_mat, *prod_vec}), stash); TEST_DO(verify_equal(expect, actual2)); } diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt index 468b73a509b..3816780d4d9 100644 --- a/eval/src/vespa/eval/eval/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_add_library(eval_eval OBJECT interpreted_function.cpp key_gen.cpp lazy_params.cpp + make_tensor_function.cpp node_types.cpp operation.cpp operator_nodes.cpp diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp index 0fb5860eaf0..13ab6fe5676 100644 --- a/eval/src/vespa/eval/eval/interpreted_function.cpp +++ b/eval/src/vespa/eval/eval/interpreted_function.cpp @@ -113,7 +113,7 @@ void op_tensor_concat(State &state, uint64_t param) { void op_tensor_function(State &state, uint64_t param) { const TensorFunction &fun = unwrap_param<TensorFunction>(param); - state.stack.push_back(fun.eval(*state.params, state.stash)); + state.stack.push_back(fun.eval(state.engine, *state.params, state.stash)); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp new file mode 100644 index 00000000000..d28c4812a31 --- /dev/null +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -0,0 +1,305 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "make_tensor_function.h" +#include "tensor_function.h" +#include "node_visitor.h" +#include "node_traverser.h" +#include "tensor_spec.h" +#include "operation.h" +#include "node_types.h" +#include "tensor_engine.h" +#include <vespa/eval/eval/llvm/compile_cache.h> + +namespace vespalib::eval { + +namespace { + +using namespace nodes; +using map_fun_t = double (*)(double); +using join_fun_t = double (*)(double, double); + +//----------------------------------------------------------------------------- + +bool step_labels(std::vector<double> &labels, const ValueType &type) { + for (size_t idx = labels.size(); idx-- > 0; ) { + labels[idx] += 1.0; + if (size_t(labels[idx]) < type.dimensions()[idx].size) { + return true; + } else { + labels[idx] = 0.0; + } + } + return false; +} + +//----------------------------------------------------------------------------- + +struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { + Stash &stash; + const TensorEngine &tensor_engine; + const NodeTypes &types; + std::vector<tensor_function::Node::CREF> stack; + + TensorFunctionBuilder(Stash &stash_in, const TensorEngine &tensor_engine_in, const NodeTypes &types_in) + : stash(stash_in), tensor_engine(tensor_engine_in), types(types_in), stack() {} + + //------------------------------------------------------------------------- + + void make_const(const Node &, const Value &value) { + stack.emplace_back(tensor_function::const_value(value, stash)); + } + + void make_inject(const Node &node, size_t param_idx) { + const ValueType &type = types.get_type(node); + stack.emplace_back(tensor_function::inject(type, param_idx, stash)); + } + + void make_reduce(const Node &, Aggr aggr, const std::vector<vespalib::string> &dimensions) { + assert(stack.size() >= 1); + const auto &a = stack.back(); + stack.back() = tensor_function::reduce(a, aggr, dimensions, stash); + } + + void make_map(const Node &, map_fun_t function) { + assert(stack.size() >= 1); + const auto &a = stack.back(); + stack.back() = tensor_function::map(a, function, stash); + } + + void make_join(const Node &, join_fun_t function) { + assert(stack.size() >= 2); + const auto &b = stack.back(); + stack.pop_back(); + const auto &a = stack.back(); + stack.back() = tensor_function::join(a, b, function, stash); + } + + void make_concat(const Node &, const vespalib::string &dimension) { + assert(stack.size() >= 2); + const auto &b = stack.back(); + stack.pop_back(); + const auto &a = stack.back(); + stack.back() = tensor_function::concat(a, b, dimension, stash); + } + + void make_rename(const Node &, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to) { + assert(stack.size() >= 1); + const auto &a = stack.back(); + stack.back() = tensor_function::rename(a, from, to, stash); + } + + void make_if(const Node &) { + assert(stack.size() >= 3); + const auto &c = stack.back(); + stack.pop_back(); + const auto &b = stack.back(); + stack.pop_back(); + const auto &a = stack.back(); + stack.back() = tensor_function::if_node(a, b, c, stash); + } + + //------------------------------------------------------------------------- + + void visit(const Number &node) override { + make_const(node, stash.create<DoubleValue>(node.value())); + } + void visit(const Symbol &node) override { + make_inject(node, node.id()); + } + void visit(const String &node) override { + make_const(node, stash.create<DoubleValue>(node.hash())); + } + void visit(const In &node) override { + auto my_in = std::make_unique<In>(std::make_unique<Symbol>(0)); + for (size_t i = 0; i < node.num_entries(); ++i) { + my_in->add_entry(std::make_unique<Number>(node.get_entry(i).get_const_value())); + } + Function my_fun(std::move(my_in), {"x"}); + const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(my_fun, PassParams::SEPARATE)); + make_map(node, token.get()->get().get_function<1>()); + } + void visit(const Neg &node) override { + make_map(node, operation::Neg::f); + } + void visit(const Not &node) override { + make_map(node, operation::Not::f); + } + void visit(const If &node) override { + make_if(node); + } + void visit(const Error &node) override { + make_const(node, ErrorValue::instance); + } + void visit(const TensorMap &node) override { + const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); + make_map(node, token.get()->get().get_function<1>()); + } + void visit(const TensorJoin &node) override { + const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::SEPARATE)); + make_join(node, token.get()->get().get_function<2>()); + } + void visit(const TensorReduce &node) override { + make_reduce(node, node.aggr(), node.dimensions()); + } + void visit(const TensorRename &node) override { + make_rename(node, node.from(), node.to()); + } + void visit(const TensorLambda &node) override { + const auto &type = node.type(); + TensorSpec spec(type.to_spec()); + const auto &token = stash.create<CompileCache::Token::UP>(CompileCache::compile(node.lambda(), PassParams::ARRAY)); + auto fun = token.get()->get().get_function(); + std::vector<double> params(type.dimensions().size(), 0.0); + assert(token.get()->get().num_params() == params.size()); + do { + TensorSpec::Address addr; + for (size_t i = 0; i < params.size(); ++i) { + addr.emplace(type.dimensions()[i].name, size_t(params[i])); + } + spec.add(addr, fun(¶ms[0])); + } while (step_labels(params, type)); + make_const(node, *stash.create<Value::UP>(tensor_engine.from_spec(spec))); + } + void visit(const TensorConcat &node) override { + make_concat(node, node.dimension()); + } + void visit(const Add &node) override { + make_join(node, operation::Add::f); + } + void visit(const Sub &node) override { + make_join(node, operation::Sub::f); + } + void visit(const Mul &node) override { + make_join(node, operation::Mul::f); + } + void visit(const Div &node) override { + make_join(node, operation::Div::f); + } + void visit(const Mod &node) override { + make_join(node, operation::Mod::f); + } + void visit(const Pow &node) override { + make_join(node, operation::Pow::f); + } + void visit(const Equal &node) override { + make_join(node, operation::Equal::f); + } + void visit(const NotEqual &node) override { + make_join(node, operation::NotEqual::f); + } + void visit(const Approx &node) override { + make_join(node, operation::Approx::f); + } + void visit(const Less &node) override { + make_join(node, operation::Less::f); + } + void visit(const LessEqual &node) override { + make_join(node, operation::LessEqual::f); + } + void visit(const Greater &node) override { + make_join(node, operation::Greater::f); + } + void visit(const GreaterEqual &node) override { + make_join(node, operation::GreaterEqual::f); + } + void visit(const And &node) override { + make_join(node, operation::And::f); + } + void visit(const Or &node) override { + make_join(node, operation::Or::f); + } + void visit(const Cos &node) override { + make_map(node, operation::Cos::f); + } + void visit(const Sin &node) override { + make_map(node, operation::Sin::f); + } + void visit(const Tan &node) override { + make_map(node, operation::Tan::f); + } + void visit(const Cosh &node) override { + make_map(node, operation::Cosh::f); + } + void visit(const Sinh &node) override { + make_map(node, operation::Sinh::f); + } + void visit(const Tanh &node) override { + make_map(node, operation::Tanh::f); + } + void visit(const Acos &node) override { + make_map(node, operation::Acos::f); + } + void visit(const Asin &node) override { + make_map(node, operation::Asin::f); + } + void visit(const Atan &node) override { + make_map(node, operation::Atan::f); + } + void visit(const Exp &node) override { + make_map(node, operation::Exp::f); + } + void visit(const Log10 &node) override { + make_map(node, operation::Log10::f); + } + void visit(const Log &node) override { + make_map(node, operation::Log::f); + } + void visit(const Sqrt &node) override { + make_map(node, operation::Sqrt::f); + } + void visit(const Ceil &node) override { + make_map(node, operation::Ceil::f); + } + void visit(const Fabs &node) override { + make_map(node, operation::Fabs::f); + } + void visit(const Floor &node) override { + make_map(node, operation::Floor::f); + } + void visit(const Atan2 &node) override { + make_join(node, operation::Atan2::f); + } + void visit(const Ldexp &node) override { + make_join(node, operation::Ldexp::f); + } + void visit(const Pow2 &node) override { + make_join(node, operation::Pow::f); + } + void visit(const Fmod &node) override { + make_join(node, operation::Mod::f); + } + void visit(const Min &node) override { + make_join(node, operation::Min::f); + } + void visit(const Max &node) override { + make_join(node, operation::Max::f); + } + void visit(const IsNan &node) override { + make_map(node, operation::IsNan::f); + } + void visit(const Relu &node) override { + make_map(node, operation::Relu::f); + } + void visit(const Sigmoid &node) override { + make_map(node, operation::Sigmoid::f); + } + void visit(const Elu &node) override { + make_map(node, operation::Elu::f); + } + + //------------------------------------------------------------------------- + + bool open(const Node &) override { return true; } + void close(const Node &node) override { node.accept(*this); } +}; + +} // namespace vespalib::eval::<unnamed> + +const TensorFunction &make_tensor_function(const TensorEngine &engine, const nodes::Node &root, const NodeTypes &types, Stash &stash) { + TensorFunctionBuilder builder(stash, engine, types); + root.traverse(builder); + assert(builder.stack.size() == 1); + return builder.stack[0]; +} + +} // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/make_tensor_function.h b/eval/src/vespa/eval/eval/make_tensor_function.h new file mode 100644 index 00000000000..f9f415b9b9f --- /dev/null +++ b/eval/src/vespa/eval/eval/make_tensor_function.h @@ -0,0 +1,17 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib { class Stash; } + +namespace vespalib::eval { + +class TensorEngine; +class NodeTypes; +class TensorFunction; + +namespace nodes { class Node; } + +const TensorFunction &make_tensor_function(const TensorEngine &engine, const nodes::Node &root, const NodeTypes &types, Stash &stash); + +} // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp index c3c1d66c645..1a60dd0e898 100644 --- a/eval/src/vespa/eval/eval/tensor_function.cpp +++ b/eval/src/vespa/eval/eval/tensor_function.cpp @@ -11,15 +11,6 @@ namespace vespalib { namespace eval { namespace tensor_function { -const TensorEngine &infer_engine(const std::initializer_list<Value::CREF> &values) { - for (const Value &value: values) { - if (auto tensor = value.as_tensor()) { - return tensor->engine(); - } - } - return SimpleTensorEngine::ref(); -} - //----------------------------------------------------------------------------- void @@ -43,7 +34,7 @@ Op2::push_children(std::vector<Child::CREF> &children) const //----------------------------------------------------------------------------- const Value & -ConstValue::eval(const LazyParams &, Stash &) const +ConstValue::eval(const TensorEngine &, const LazyParams &, Stash &) const { return _value; } @@ -51,7 +42,7 @@ ConstValue::eval(const LazyParams &, Stash &) const //----------------------------------------------------------------------------- const Value & -Inject::eval(const LazyParams ¶ms, Stash &stash) const +Inject::eval(const TensorEngine &, const LazyParams ¶ms, Stash &stash) const { return params.resolve(_param_idx, stash); } @@ -59,52 +50,47 @@ Inject::eval(const LazyParams ¶ms, Stash &stash) const //----------------------------------------------------------------------------- const Value & -Reduce::eval(const LazyParams ¶ms, Stash &stash) const +Reduce::eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const { - const Value &a = child().eval(params, stash); - const TensorEngine &engine = infer_engine({a}); + const Value &a = child().eval(engine, params, stash); return engine.reduce(a, _aggr, _dimensions, stash); } //----------------------------------------------------------------------------- const Value & -Map::eval(const LazyParams ¶ms, Stash &stash) const +Map::eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const { - const Value &a = child().eval(params, stash); - const TensorEngine &engine = infer_engine({a}); + const Value &a = child().eval(engine, params, stash); return engine.map(a, _function, stash); } //----------------------------------------------------------------------------- const Value & -Join::eval(const LazyParams ¶ms, Stash &stash) const +Join::eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const { - const Value &a = lhs().eval(params, stash); - const Value &b = rhs().eval(params, stash); - const TensorEngine &engine = infer_engine({a,b}); + const Value &a = lhs().eval(engine, params, stash); + const Value &b = rhs().eval(engine, params, stash); return engine.join(a, b, _function, stash); } //----------------------------------------------------------------------------- const Value & -Concat::eval(const LazyParams ¶ms, Stash &stash) const +Concat::eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const { - const Value &a = lhs().eval(params, stash); - const Value &b = rhs().eval(params, stash); - const TensorEngine &engine = infer_engine({a,b}); + const Value &a = lhs().eval(engine, params, stash); + const Value &b = rhs().eval(engine, params, stash); return engine.concat(a, b, _dimension, stash); } //----------------------------------------------------------------------------- const Value & -Rename::eval(const LazyParams ¶ms, Stash &stash) const +Rename::eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const { - const Value &a = child().eval(params, stash); - const TensorEngine &engine = infer_engine({a}); + const Value &a = child().eval(engine, params, stash); return engine.rename(a, _from, _to, stash); } @@ -119,11 +105,11 @@ If::push_children(std::vector<Child::CREF> &children) const } const Value & -If::eval(const LazyParams ¶ms, Stash &stash) const +If::eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const { - return (cond().eval(params, stash).as_bool() - ? true_child().eval(params, stash) - : false_child().eval(params, stash)); + return (cond().eval(engine, params, stash).as_bool() + ? true_child().eval(engine, params, stash) + : false_child().eval(engine, params, stash)); } //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h index ed19b6dce4a..d9ee5cc068c 100644 --- a/eval/src/vespa/eval/eval/tensor_function.h +++ b/eval/src/vespa/eval/eval/tensor_function.h @@ -6,6 +6,7 @@ #include <vector> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/arrayref.h> +#include "make_tensor_function.h" #include "lazy_params.h" #include "value_type.h" #include "value.h" @@ -80,10 +81,11 @@ struct TensorFunction * valid. The return value must conform to 'result_type'. * * @return result of evaluating this tensor function + * @param engine the tensor engine we are using for evaluation * @param params external values needed to evaluate this function * @param stash heterogeneous object store **/ - virtual const Value &eval(const LazyParams ¶ms, Stash &stash) const = 0; + virtual const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const = 0; virtual ~TensorFunction() {} }; @@ -105,6 +107,7 @@ class Node : public TensorFunction private: ValueType _result_type; public: + using CREF = std::reference_wrapper<const Node>; Node(const ValueType &result_type_in) : _result_type(result_type_in) {} const ValueType &result_type() const final override { return _result_type; } }; @@ -153,7 +156,7 @@ private: const Value &_value; public: ConstValue(const Value &value_in) : Leaf(value_in.type()), _value(value_in) {} - const Value &eval(const LazyParams ¶ms, Stash &) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &) const final override; }; //----------------------------------------------------------------------------- @@ -166,7 +169,7 @@ public: Inject(const ValueType &result_type_in, size_t param_idx_in) : Leaf(result_type_in), _param_idx(param_idx_in) {} size_t param_idx() const { return _param_idx; } - const Value &eval(const LazyParams ¶ms, Stash &) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &) const final override; }; //----------------------------------------------------------------------------- @@ -184,7 +187,7 @@ public: : Op1(result_type_in, child_in), _aggr(aggr_in), _dimensions(dimensions_in) {} Aggr aggr() const { return _aggr; } const std::vector<vespalib::string> &dimensions() const { return _dimensions; } - const Value &eval(const LazyParams ¶ms, Stash &stash) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const final override; }; //----------------------------------------------------------------------------- @@ -199,7 +202,7 @@ public: map_fun_t function_in) : Op1(result_type_in, child_in), _function(function_in) {} map_fun_t function() const { return _function; } - const Value &eval(const LazyParams ¶ms, Stash &stash) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const final override; }; //----------------------------------------------------------------------------- @@ -215,7 +218,7 @@ public: join_fun_t function_in) : Op2(result_type_in, lhs_in, rhs_in), _function(function_in) {} join_fun_t function() const { return _function; } - const Value &eval(const LazyParams ¶ms, Stash &stash) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const final override; }; //----------------------------------------------------------------------------- @@ -231,7 +234,7 @@ public: const vespalib::string &dimension_in) : Op2(result_type_in, lhs_in, rhs_in), _dimension(dimension_in) {} const vespalib::string &dimension() const { return _dimension; } - const Value &eval(const LazyParams ¶ms, Stash &stash) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const final override; }; //----------------------------------------------------------------------------- @@ -249,7 +252,7 @@ public: : Op1(result_type_in, child_in), _from(from_in), _to(to_in) {} const std::vector<vespalib::string> &from() const { return _from; } const std::vector<vespalib::string> &to() const { return _to; } - const Value &eval(const LazyParams ¶ms, Stash &stash) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const final override; }; //----------------------------------------------------------------------------- @@ -270,7 +273,7 @@ public: const TensorFunction &true_child() const { return _true_child.get(); } const TensorFunction &false_child() const { return _false_child.get(); } void push_children(std::vector<Child::CREF> &children) const final override; - const Value &eval(const LazyParams ¶ms, Stash &stash) const final override; + const Value &eval(const TensorEngine &engine, const LazyParams ¶ms, Stash &stash) const final override; }; //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp index 6fa2fc2574d..8fe0732f3c4 100644 --- a/eval/src/vespa/eval/eval/test/tensor_conformance.cpp +++ b/eval/src/vespa/eval/eval/test/tensor_conformance.cpp @@ -278,7 +278,7 @@ struct RetainedReduce : Eval { ValueType expect_type = ir.result_type(); const auto &fun = engine.compile(ir, stash); Input input(engine.from_spec(a)); - return Result(engine, check_type(fun.eval(input.get(), stash), expect_type)); + return Result(engine, check_type(fun.eval(engine, input.get(), stash), expect_type)); } }; @@ -293,7 +293,7 @@ struct RetainedMap : Eval { ValueType expect_type = ir.result_type(); const auto &fun = engine.compile(ir, stash); Input input(engine.from_spec(a)); - return Result(engine, check_type(fun.eval(input.get(), stash), expect_type)); + return Result(engine, check_type(fun.eval(engine, input.get(), stash), expect_type)); } }; @@ -311,7 +311,7 @@ struct RetainedJoin : Eval { ValueType expect_type = ir.result_type(); const auto &fun = engine.compile(ir, stash); Input input(engine.from_spec(a), engine.from_spec(b)); - return Result(engine, check_type(fun.eval(input.get(), stash), expect_type)); + return Result(engine, check_type(fun.eval(engine, input.get(), stash), expect_type)); } }; diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp index de9963a0c9a..9f09940806b 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp @@ -29,7 +29,7 @@ getCellsRef(const eval::Value &value) } const eval::Value & -DenseDotProductFunction::eval(const eval::LazyParams ¶ms, Stash &stash) const +DenseDotProductFunction::eval(const eval::TensorEngine &, const eval::LazyParams ¶ms, Stash &stash) const { DenseTensorView::CellsRef lhsCells = getCellsRef(params.resolve(_lhsTensorId, stash)); DenseTensorView::CellsRef rhsCells = getCellsRef(params.resolve(_rhsTensorId, stash)); diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h index 1bca0ce4c8d..4e3a54ca18d 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h @@ -23,7 +23,7 @@ public: size_t rhsTensorId() const { return _rhsTensorId; } const eval::ValueType &result_type() const override { return eval::DoubleValue::double_type(); } void push_children(std::vector<Child::CREF> &) const override {} - const eval::Value &eval(const eval::LazyParams ¶ms, Stash &stash) const override; + const eval::Value &eval(const eval::TensorEngine &engine, const eval::LazyParams ¶ms, Stash &stash) const override; }; } diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp index caf8f0de1d7..50ab6efc931 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp @@ -72,7 +72,7 @@ getCellsRef(const eval::Value &value) } // namespace <unnamed> const eval::Value & -DenseXWProductFunction::eval(const eval::LazyParams ¶ms, Stash &stash) const +DenseXWProductFunction::eval(const eval::TensorEngine &, const eval::LazyParams ¶ms, Stash &stash) const { DenseTensorView::CellsRef vectorCells = getCellsRef(params.resolve(_vectorId, stash)); DenseTensorView::CellsRef matrixCells = getCellsRef(params.resolve(_matrixId, stash)); diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h index bc0a63bc79e..c6a466dc527 100644 --- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h +++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h @@ -47,7 +47,7 @@ public: const eval::ValueType &result_type() const override { return _resultType; } void push_children(std::vector<Child::CREF> &) const override {} - const eval::Value &eval(const eval::LazyParams ¶ms, Stash &stash) const override; + const eval::Value &eval(const eval::TensorEngine &engine, const eval::LazyParams ¶ms, Stash &stash) const override; }; } diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index d9699b700d0..6bfa4362f65 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -58,6 +58,18 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> </dependency> + <dependency> + <groupId>io.airlift</groupId> + <artifactId>airline</artifactId> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> </dependencies> <build> @@ -67,6 +79,34 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java new file mode 100644 index 00000000000..b50416ac159 --- /dev/null +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClient.java @@ -0,0 +1,236 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.filedistribution.status; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.airlift.airline.Command; +import io.airlift.airline.HelpOption; +import io.airlift.airline.Option; +import io.airlift.airline.SingleCommand; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; + +/** + * Tool for getting file distribution status + * + * @author hmusum + */ +public class FileDistributionStatusClient { + + private static final String statusUnknown = "UNKNOWN"; + private static final String statusInProgress = "IN_PROGRESS"; + private static final String statusFinished = "FINISHED"; + + + private final String tenantName; + private final String applicationName; + private final String instanceName; + private final String environment; + private final String region; + private final double timeout; + private final boolean debug; + + FileDistributionStatusClient(CommandLineArguments arguments) { + tenantName = arguments.getTenantName(); + applicationName = arguments.getApplicationName(); + instanceName = arguments.getInstanceName(); + environment = arguments.getEnvironment(); + region = arguments.getRegion(); + timeout = arguments.getTimeout(); + debug = arguments.getDebugFlag(); + } + + public static void main(String[] args) { + try { + new FileDistributionStatusClient(CommandLineArguments.build(args)).run(); + } catch (Exception e) { + System.err.println(e.getMessage()); + System.exit(1); + } + } + + public void run() { + String json = doHttpRequest(); + System.out.println(parseAndGenerateOutput(json)); + } + + private String doHttpRequest() { + int timeoutInMillis = (int) (timeout * 1000); + RequestConfig config = RequestConfig.custom() + .setConnectTimeout(timeoutInMillis) + .setConnectionRequestTimeout(timeoutInMillis) + .setSocketTimeout(timeoutInMillis) + .build(); + CloseableHttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).build(); + URI statusUri = createStatusApiUri(); + if (debug) + System.out.println("URI:" + statusUri); + try { + CloseableHttpResponse response = httpClient.execute(new HttpGet(statusUri)); + String content = EntityUtils.toString(response.getEntity()); + if (debug) + System.out.println("response:" + content); + if (response.getStatusLine().getStatusCode() == 200) { + return content; + } else { + throw new RuntimeException("Failed to get status for request " + statusUri + ": " + + response.getStatusLine() + ": " + content); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + String parseAndGenerateOutput(String json) { + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode; + try { + jsonNode = objectMapper.readTree(json); + } catch (IOException e) { + throw new RuntimeException(e); + } + String status = jsonNode.get("status").asText(); + switch (status) { + case statusUnknown: + return "File distribution status unknown: " + jsonNode.get("message").asText(); + case statusInProgress: + return "File distribution in progress:\n" + inProgressOutput(jsonNode.get("hosts")); + case statusFinished: + return "File distribution finished"; + default: + throw new RuntimeException("Unknown status " + status); + } + } + + private URI createStatusApiUri() { + String path = String.format("/application/v2/tenant/%s/application/%s/environment/%s/region/%s/instance/%s/filedistributionstatus", + tenantName, applicationName, environment, region, instanceName); + try { + return new URIBuilder() + .setScheme("http") + .setHost("localhost") + .setPort(19071) + .setPath(path) + .addParameter("timeout", String.valueOf(timeout)) + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + private String inProgressOutput(JsonNode hosts) { + ArrayList<String> statusPerHost = new ArrayList<>(); + for (JsonNode host : hosts) { + StringBuilder sb = new StringBuilder(); + String status = host.get("status").asText(); + sb.append(host.get("hostname").asText()).append(": ").append(status); + if (status.equals(statusUnknown)) + sb.append(" (").append(host.get("message").asText()).append(")"); + else if (status.equals(statusInProgress)) { + JsonNode fileReferencesArray = host.get("fileReferences"); + int size = fileReferencesArray.size(); + int finished = 0; + for (JsonNode element : fileReferencesArray) { + for (Iterator<Map.Entry<String, JsonNode>> it = element.fields(); it.hasNext(); ) { + Map.Entry<String, JsonNode> fileReferenceStatus = it.next(); + if (fileReferenceStatus.getValue().asDouble() == 1.0) + finished++; + } + } + sb.append(" (" + finished + " of " + size + " finished)"); + } + statusPerHost.add(sb.toString()); + } + return String.join("\n", statusPerHost); + } + + @Command(name = "vespa-status-filedistribution", description = "Tool for getting file distribution status.") + public static class CommandLineArguments { + + static CommandLineArguments build(String[] args) { + CommandLineArguments arguments = null; + try { + arguments = SingleCommand.singleCommand(CommandLineArguments.class).parse(args); + } catch (Exception e) { + System.err.println(e.getMessage()); + System.err.println("Use --help to show usage.\n"); + System.exit(1); + } + + if (arguments.helpOption.showHelpIfRequested()) { + System.exit(0); + } + + if (arguments.getTenantName() == null) { + System.err.println("'--tenant' not set."); + System.exit(1); + } + + if (arguments.getApplicationName() == null) { + System.err.println("'--application' not set."); + System.exit(1); + } + + return arguments; + } + + @Inject + HelpOption helpOption; + + @Option(name = {"--tenant"}, + description = "tenant name") + private String tenantNameArg; + + @Option(name = {"--application"}, + description = "application name") + private String applicationNameArg; + + @Option(name = {"--instance"}, + description = "instance name") + private String instanceNameArg = "default"; + + @Option(name = {"--environment"}, + description = "environment name") + private String environmentArg = "prod"; + + @Option(name = {"--region"}, + description = "region name") + private String regionArg = "default"; + + @Option(name = {"--timeout"}, + description = "The timeout (in seconds).") + private double timeoutArg = 5; + + @Option(name = {"--debug"}, + description = "Print debug log.") + private boolean debugArg; + + public String getTenantName() { return tenantNameArg; } + + public String getApplicationName() { return applicationNameArg; } + + public String getInstanceName() { return instanceNameArg; } + + public String getEnvironment() { return environmentArg; } + + public String getRegion() { return regionArg; } + + public double getTimeout() { return timeoutArg; } + + public boolean getDebugFlag() { return debugArg; } + } + +} diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java new file mode 100644 index 00000000000..fcbe880bfc7 --- /dev/null +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/status/FileDistributionStatusClientTest.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.filedistribution.status; + +import org.junit.Test; + +import static com.yahoo.vespa.filedistribution.status.FileDistributionStatusClient.CommandLineArguments; +import static org.junit.Assert.assertEquals; + +public class FileDistributionStatusClientTest { + + private static final CommandLineArguments arguments = createArguments("--tenant", "foo", "--application", "bar"); + private final FileDistributionStatusClient client = new FileDistributionStatusClient(arguments); + + @Test + public void finishedForAllHosts() { + String output = client.parseAndGenerateOutput("{\"status\":\"FINISHED\"}"); + assertEquals("File distribution finished", output); + } + + @Test + public void unknownForAllHosts() { + String output = client.parseAndGenerateOutput("{\"status\":\"UNKNOWN\", \"message\":\"Something went wrong\"}"); + assertEquals("File distribution status unknown: Something went wrong", output); + } + + @Test + public void manyHostsVariousStates() { + String statusForTwoHosts = createStatusForTwoHosts(); + System.out.println(statusForTwoHosts); + String output = client.parseAndGenerateOutput(statusForTwoHosts); + assertEquals("File distribution in progress:\nlocalhost1: IN_PROGRESS (1 of 2 finished)\nlocalhost2: UNKNOWN (Connection timed out)", output); + } + + private static CommandLineArguments createArguments(String... args) { + return CommandLineArguments.build(args); + } + + private String createStatusForTwoHosts() { + return "{\"status\":\"IN_PROGRESS\"," + + "\"hosts\":[" + createInProgressStatusForHost("localhost1") + "," + createUnknownStatusForHost("localhost2") + "]" + + "}"; + } + + private String createInProgressStatusForHost(String hostname) { + return "{\"hostname\":\"" + hostname + "\"," + + "\"status\":\"IN_PROGRESS\"," + + "\"message\":\"\"," + + "\"fileReferences\":[" + + "{\"1234\":0.2}, {\"abcd\":1.0}]}"; + } + + private String createUnknownStatusForHost(String hostname) { + return "{\"hostname\":\"" + hostname + "\"," + + "\"status\":\"UNKNOWN\"," + + "\"message\":\"Connection timed out\"}"; + } + +} diff --git a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java index 15611c85b45..1e95ca15c3d 100644 --- a/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java +++ b/node-maintainer/src/main/java/com/yahoo/vespa/hosted/node/maintainer/Maintainer.java @@ -190,6 +190,7 @@ public class Maintainer { private static HttpClient createHttpClient(Duration timeout) { int timeoutInMillis = (int) timeout.toMillis(); return HttpClientBuilder.create() + .setUserAgent("node-maintainer") .setDefaultRequestConfig(RequestConfig.custom() .setConnectTimeout(timeoutInMillis) .setConnectionRequestTimeout(timeoutInMillis) diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/AthenzIdentityVerifier.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzIdentityVerifier.java index a73bbb7ed8c..6bec4bc9a82 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/AthenzIdentityVerifier.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzIdentityVerifier.java @@ -1,7 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.athenz.utils; +package com.yahoo.vespa.athenz.tls; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLPeerUnverifiedException; diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/AthenzIdentityVerifierTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/AthenzIdentityVerifierTest.java index dabfc16b024..57f38c3a114 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/AthenzIdentityVerifierTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/AthenzIdentityVerifierTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.utils; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x509.BasicConstraints; import org.bouncycastle.asn1.x509.Extension; |