add json-repl (and interactive) modes to vespa-eval-expr

2 files changed, 274 insertions, 0 deletions

diff --git a/eval/src/tests/apps/eval_expr/CMakeLists.txt b/eval/src/tests/apps/eval_expr/CMakeLists.txt
new file mode 100644
index 00000000000..84f90f61f49
--- /dev/null
+++ b/eval/src/tests/apps/eval_expr/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(eval_eval_expr_test_app TEST
+    SOURCES
+    eval_expr_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME eval_eval_expr_test_app COMMAND eval_eval_expr_test_app
+               DEPENDS eval_eval_expr_test_app eval_eval_expr_app)
diff --git a/eval/src/tests/apps/eval_expr/eval_expr_test.cpp b/eval/src/tests/apps/eval_expr/eval_expr_test.cpp
new file mode 100644
index 00000000000..8e0b3286421
--- /dev/null
+++ b/eval/src/tests/apps/eval_expr/eval_expr_test.cpp
@@ -0,0 +1,265 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/testkit/time_bomb.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/util/child_process.h>
+#include <vespa/vespalib/data/input.h>
+#include <vespa/vespalib/data/output.h>
+#include <vespa/vespalib/data/simple_buffer.h>
+#include <vespa/vespalib/util/size_literals.h>
+
+using namespace vespalib;
+using vespalib::make_string_short::fmt;
+using vespalib::slime::JsonFormat;
+using vespalib::slime::Inspector;
+
+vespalib::string module_build_path("../../../../");
+vespalib::string binary = module_build_path + "src/apps/eval_expr/vespa-eval-expr";
+vespalib::string server_cmd = binary + " --verbose json-repl";
+
+//-----------------------------------------------------------------------------
+
+void read_until_eof(Input &input) {
+    for (auto mem = input.obtain(); mem.size > 0; mem = input.obtain()) {
+        input.evict(mem.size);
+    }
+}
+
+// copied from vespalib::eval::test
+//
+// It seems that linking with the eval library contaminates the
+// process proxy in such a way that valgrind will fail. The working
+// theory is that some static state gets initialized before the proxy
+// is forked. This state conflicts with itself when the eval library
+// is loaded again when doing exec on vespa-eval-expr. This test
+// bypasses the issue by linking with vespalib instead. A more robust
+// solution would be to reverse the roles of the process proxy and the
+// program; letting the proxy start the program. This could also be
+// combined with the ability to send open file descriptors on unix
+// domain sockets to avoid indirection for stdin/stdout streams.
+
+void write_compact(const Slime &slime, Output &out) {
+    JsonFormat::encode(slime, out, true);
+    out.reserve(1).data[0] = '\n';
+    out.commit(1);
+}
+
+// Output adapter used to write to stdin of a child process
+class ChildIn : public Output {
+    ChildProcess &_child;
+    SimpleBuffer _output;
+public:
+    ChildIn(ChildProcess &child) : _child(child) {}
+    WritableMemory reserve(size_t bytes) override {
+        return _output.reserve(bytes);
+    }
+    Output &commit(size_t bytes) override {
+        _output.commit(bytes);
+        Memory buf = _output.obtain();
+        ASSERT_TRUE(_child.write(buf.data, buf.size));
+        _output.evict(buf.size);
+        return *this;
+    }
+};
+
+// Input adapter used to read from stdout of a child process
+class ChildOut : public Input {
+    ChildProcess &_child;
+    SimpleBuffer _input;
+public:
+    ChildOut(ChildProcess &child)
+      : _child(child)
+    {
+        EXPECT_TRUE(_child.running());
+        EXPECT_TRUE(!_child.failed());
+    }
+    Memory obtain() override {
+        if ((_input.get().size == 0) && !_child.eof()) {
+            WritableMemory buf = _input.reserve(4_Ki);
+            uint32_t res = _child.read(buf.data, buf.size);
+            ASSERT_TRUE((res > 0) || _child.eof());
+            _input.commit(res);
+        }
+        return _input.obtain();
+    }
+    Input &evict(size_t bytes) override {
+        _input.evict(bytes);
+        return *this;
+    }
+};
+
+//-----------------------------------------------------------------------------
+
+struct Result {
+    vespalib::string error;
+    vespalib::string result;
+    std::vector<std::pair<vespalib::string, vespalib::string>> steps;
+
+    Result(const Inspector &obj)
+      : error(obj["error"].asString().make_string()),
+        result(obj["result"].asString().make_string()),
+        steps()
+    {
+        const auto &arr = obj["steps"];
+        for (size_t i = 0; i < arr.entries(); ++i) {
+            steps.emplace_back(arr[i]["class"].asString().make_string(),
+                               arr[i]["symbol"].asString().make_string());
+        }
+    }
+    void verify_result(const vespalib::string &expect) {
+        EXPECT_EQUAL(error, "");
+        EXPECT_EQUAL(result, expect);
+    }
+    void verify_error(const vespalib::string &expect) {
+        EXPECT_EQUAL(steps.size(), 0u);
+        EXPECT_EQUAL(result, "");
+        fprintf(stderr, "... does error '%s' contain message '%s'?\n",
+                error.c_str(), expect.c_str());
+        EXPECT_TRUE(error.find(expect) != error.npos);
+    }
+    ~Result();
+};
+Result::~Result() = default;
+
+void dump_message(const char *prefix, const Slime &slime) {
+    SimpleBuffer buf;
+    slime::JsonFormat::encode(slime, buf, true);
+    auto str = buf.get().make_string();
+    fprintf(stderr, "%s%s\n", prefix, str.c_str());
+}
+
+class Server {
+private:
+    TimeBomb _bomb;
+    ChildProcess _child;
+    ChildIn _child_stdin;
+    ChildOut _child_stdout;
+public:
+    Server()
+      : _bomb(60),
+        _child(server_cmd.c_str()),
+        _child_stdin(_child),
+        _child_stdout(_child) {}
+    ~Server();
+    Slime invoke(const Slime &req) {
+        dump_message("request --> ", req);
+        write_compact(req, _child_stdin);
+        Slime reply;
+        ASSERT_TRUE(JsonFormat::decode(_child_stdout, reply));
+        dump_message("  reply <-- ", reply);
+        return reply;
+    }
+    Result eval(const vespalib::string &expr, const vespalib::string &name = {}, bool verbose = false) {
+        Slime req;
+        auto &obj = req.setObject();
+        obj.setString("expr", expr.c_str());
+        if (!name.empty()) {
+            obj.setString("name", name.c_str());
+        }
+        if (verbose) {
+            obj.setBool("verbose", true);
+        }
+        Slime reply = invoke(req);
+        return {reply.get()};
+    }
+};
+Server::~Server() {
+    _child.close();
+    read_until_eof(_child_stdout);
+    ASSERT_TRUE(_child.wait());
+    ASSERT_TRUE(!_child.running());
+    ASSERT_TRUE(!_child.failed());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST("print server command") {
+    fprintf(stderr, "server cmd: %s\n", server_cmd.c_str());
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_F("require that simple evaluation works", Server()) {
+    TEST_DO(f1.eval("2+2").verify_result("4"));
+}
+
+TEST_F("require that multiple dependent expressions work", Server()) {
+    TEST_DO(f1.eval("2+2", "a").verify_result("4"));
+    TEST_DO(f1.eval("a+2", "b").verify_result("6"));
+    TEST_DO(f1.eval("a+b").verify_result("10"));
+}
+
+TEST_F("require that symbols can be overwritten", Server()) {
+    TEST_DO(f1.eval("1", "a").verify_result("1"));
+    TEST_DO(f1.eval("a+1", "a").verify_result("2"));
+    TEST_DO(f1.eval("a+1", "a").verify_result("3"));
+    TEST_DO(f1.eval("a+1", "a").verify_result("4"));
+}
+
+TEST_F("require that tensor result is returned in verbose verbatim form", Server()) {
+    TEST_DO(f1.eval("1", "a").verify_result("1"));
+    TEST_DO(f1.eval("2", "b").verify_result("2"));
+    TEST_DO(f1.eval("3", "c").verify_result("3"));
+    TEST_DO(f1.eval("tensor(x[3]):[a,b,c]").verify_result("tensor(x[3]):{{x:0}:1,{x:1}:2,{x:2}:3}"));
+}
+
+TEST_F("require that execution steps can be extracted", Server()) {
+    TEST_DO(f1.eval("1", "a").verify_result("1"));
+    TEST_DO(f1.eval("2", "b").verify_result("2"));
+    TEST_DO(f1.eval("3", "c").verify_result("3"));
+    auto res1 = f1.eval("a+b+c");
+    auto res2 = f1.eval("a+b+c", "", true);
+    EXPECT_EQUAL(res1.steps.size(), 0u);
+    EXPECT_EQUAL(res2.steps.size(), 5u);
+    for (const auto &step: res2.steps) {
+        fprintf(stderr, "step:\n  class: %s\n    symbol: %s\n",
+                step.first.c_str(), step.second.c_str());
+    }
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_F("require that operation batching works", Server()) {
+    Slime req;
+    auto &arr = req.setArray();
+    auto &req1 = arr.addObject();
+    req1.setString("expr", "2+2");
+    req1.setString("name", "a");
+    auto &req2 = arr.addObject();
+    req2.setString("expr", "a+2");
+    req2.setString("name", "b");
+    auto &req3 = arr.addObject();
+    req3.setString("expr", "this does not parse");
+    auto &req4 = arr.addObject();
+    req4.setString("expr", "a+b");
+    Slime reply = f1.invoke(req);
+    EXPECT_EQUAL(reply.get().entries(), 4u);
+    EXPECT_TRUE(reply[2]["error"].asString().size > 0);
+    EXPECT_EQUAL(reply[3]["result"].asString().make_string(), "10");
+}
+
+TEST_F("require that empty operation batch works", Server()) {
+    Slime req;
+    req.setArray();
+    Slime reply = f1.invoke(req);
+    EXPECT_TRUE(reply.get().type().getId() == slime::ARRAY::ID);
+    EXPECT_EQUAL(reply.get().entries(), 0u);
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_F("require that empty expression produces error", Server()) {
+    auto res = f1.eval("");
+    TEST_DO(res.verify_error("missing expression"));
+}
+
+TEST_F("require that parse error produces error", Server()) {
+    auto res = f1.eval("this does not parse");
+    TEST_DO(res.verify_error("expression parsing failed"));
+}
+
+//-----------------------------------------------------------------------------
+
+TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); }