diff options
Diffstat (limited to 'searchlib/src/tests/fef')
72 files changed, 2946 insertions, 0 deletions
diff --git a/searchlib/src/tests/fef/.gitignore b/searchlib/src/tests/fef/.gitignore new file mode 100644 index 00000000000..ff604ccaf00 --- /dev/null +++ b/searchlib/src/tests/fef/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +fef_test +searchlib_fef_test_app diff --git a/searchlib/src/tests/fef/CMakeLists.txt b/searchlib/src/tests/fef/CMakeLists.txt new file mode 100644 index 00000000000..a239ba972c3 --- /dev/null +++ b/searchlib/src/tests/fef/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_fef_test_app + SOURCES + fef_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_fef_test_app COMMAND searchlib_fef_test_app) diff --git a/searchlib/src/tests/fef/DESC b/searchlib/src/tests/fef/DESC new file mode 100644 index 00000000000..431ee7a1a1f --- /dev/null +++ b/searchlib/src/tests/fef/DESC @@ -0,0 +1 @@ +fef test. Take a look at fef.cpp for details. diff --git a/searchlib/src/tests/fef/FILES b/searchlib/src/tests/fef/FILES new file mode 100644 index 00000000000..7e6752e501e --- /dev/null +++ b/searchlib/src/tests/fef/FILES @@ -0,0 +1 @@ +fef.cpp diff --git a/searchlib/src/tests/fef/attributecontent/.gitignore b/searchlib/src/tests/fef/attributecontent/.gitignore new file mode 100644 index 00000000000..dd57ee57362 --- /dev/null +++ b/searchlib/src/tests/fef/attributecontent/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +attributecontent_test +searchlib_attributecontent_test_app diff --git a/searchlib/src/tests/fef/attributecontent/CMakeLists.txt b/searchlib/src/tests/fef/attributecontent/CMakeLists.txt new file mode 100644 index 00000000000..84cdb3d4fce --- /dev/null +++ b/searchlib/src/tests/fef/attributecontent/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_attributecontent_test_app + SOURCES + attributecontent_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_attributecontent_test_app COMMAND searchlib_attributecontent_test_app) diff --git a/searchlib/src/tests/fef/attributecontent/DESC b/searchlib/src/tests/fef/attributecontent/DESC new file mode 100644 index 00000000000..fa1c457c573 --- /dev/null +++ b/searchlib/src/tests/fef/attributecontent/DESC @@ -0,0 +1 @@ +attributecontent test. Take a look at attributecontent.cpp for details. diff --git a/searchlib/src/tests/fef/attributecontent/FILES b/searchlib/src/tests/fef/attributecontent/FILES new file mode 100644 index 00000000000..4325e907b45 --- /dev/null +++ b/searchlib/src/tests/fef/attributecontent/FILES @@ -0,0 +1 @@ +attributecontent.cpp diff --git a/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp b/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp new file mode 100644 index 00000000000..66430994016 --- /dev/null +++ b/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("attributecontent_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/integerbase.h> + +#include <vespa/searchlib/attribute/attributevector.hpp> + +using namespace search::attribute; + +namespace search { +namespace fef { + +class Test : public vespalib::TestApp { +private: + void testWriteAndRead(); + void testFill(); + +public: + int Main(); +}; + +void +Test::testWriteAndRead() +{ + typedef search::attribute::AttributeContent<uint32_t> UintContent; + UintContent buf; + EXPECT_EQUAL(buf.capacity(), 16u); + EXPECT_EQUAL(buf.size(), 0u); + + uint32_t i; + uint32_t * data; + const uint32_t * itr; + for (i = 0, data = buf.data(); i < 16; ++i, ++data) { + *data = i; + } + buf.setSize(16); + EXPECT_EQUAL(buf.size(), 16u); + for (i = 0, itr = buf.begin(); itr != buf.end(); ++i, ++itr) { + EXPECT_EQUAL(*itr, i); + EXPECT_EQUAL(buf[i], i); + } + EXPECT_EQUAL(i, 16u); + + buf.allocate(10); + EXPECT_EQUAL(buf.capacity(), 16u); + EXPECT_EQUAL(buf.size(), 16u); + buf.allocate(32); + EXPECT_EQUAL(buf.capacity(), 32u); + EXPECT_EQUAL(buf.size(), 0u); + + for (i = 0, data = buf.data(); i < 32; ++i, ++data) { + *data = i; + } + buf.setSize(32); + EXPECT_EQUAL(buf.size(), 32u); + for (i = 0, itr = buf.begin(); itr != buf.end(); ++i, ++itr) { + EXPECT_EQUAL(*itr, i); + EXPECT_EQUAL(buf[i], i); + } + EXPECT_EQUAL(i, 32u); +} + +void +Test::testFill() +{ + Config cfg(BasicType::INT32, CollectionType::ARRAY); + AttributeVector::SP av = AttributeFactory::createAttribute("aint32", cfg); + av->addDocs(2); + IntegerAttribute * ia = static_cast<IntegerAttribute *>(av.get()); + ia->append(0, 10, 0); + ia->append(1, 20, 0); + ia->append(1, 30, 0); + av->commit(); + const IAttributeVector & iav = *av.get(); + IntegerContent buf; + buf.fill(iav, 0); + EXPECT_EQUAL(1u, buf.size()); + EXPECT_EQUAL(10, buf[0]); + buf.fill(iav, 1); + EXPECT_EQUAL(2u, buf.size()); + EXPECT_EQUAL(20, buf[0]); + EXPECT_EQUAL(30, buf[1]); + buf.fill(iav, 0); + EXPECT_EQUAL(1u, buf.size()); + EXPECT_EQUAL(10, buf[0]); +} + +int +Test::Main() +{ + TEST_INIT("attributecontent_test"); + + testWriteAndRead(); + testFill(); + + TEST_DONE(); +} + +} // namespace fef +} // namespace search + +TEST_APPHOOK(search::fef::Test); diff --git a/searchlib/src/tests/fef/featurenamebuilder/.gitignore b/searchlib/src/tests/fef/featurenamebuilder/.gitignore new file mode 100644 index 00000000000..781f49956a9 --- /dev/null +++ b/searchlib/src/tests/fef/featurenamebuilder/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +featurenamebuilder_test +searchlib_featurenamebuilder_test_app diff --git a/searchlib/src/tests/fef/featurenamebuilder/CMakeLists.txt b/searchlib/src/tests/fef/featurenamebuilder/CMakeLists.txt new file mode 100644 index 00000000000..167642c1337 --- /dev/null +++ b/searchlib/src/tests/fef/featurenamebuilder/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_featurenamebuilder_test_app + SOURCES + featurenamebuilder_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_featurenamebuilder_test_app COMMAND searchlib_featurenamebuilder_test_app) diff --git a/searchlib/src/tests/fef/featurenamebuilder/DESC b/searchlib/src/tests/fef/featurenamebuilder/DESC new file mode 100644 index 00000000000..38abf1af794 --- /dev/null +++ b/searchlib/src/tests/fef/featurenamebuilder/DESC @@ -0,0 +1 @@ +featurenamebuilder test. Take a look at featurenamebuilder.cpp for details. diff --git a/searchlib/src/tests/fef/featurenamebuilder/FILES b/searchlib/src/tests/fef/featurenamebuilder/FILES new file mode 100644 index 00000000000..71df1d1033f --- /dev/null +++ b/searchlib/src/tests/fef/featurenamebuilder/FILES @@ -0,0 +1 @@ +featurenamebuilder.cpp diff --git a/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp b/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp new file mode 100644 index 00000000000..0e574c776b5 --- /dev/null +++ b/searchlib/src/tests/fef/featurenamebuilder/featurenamebuilder_test.cpp @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("featurenamebuilder_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/fef/featurenamebuilder.h> + +using namespace search::fef; + +typedef FeatureNameBuilder B; + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("featurenamebuilder_test"); + + // normal cases + EXPECT_EQUAL(B().baseName("foo").buildName(), "foo"); + EXPECT_EQUAL(B().baseName("foo").output("out").buildName(), "foo.out"); + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").buildName(), "foo(a,b)"); + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").output("out").buildName(), "foo(a,b).out"); + + // empty base = empty name + EXPECT_EQUAL(B().baseName("").buildName(), ""); + EXPECT_EQUAL(B().baseName("").output("out").buildName(), ""); + EXPECT_EQUAL(B().baseName("").parameter("a").parameter("b").buildName(), ""); + EXPECT_EQUAL(B().baseName("").parameter("a").parameter("b").output("out").buildName(), ""); + + // quoting + EXPECT_EQUAL(B().baseName("foo").parameter("a,b").output("out").buildName(), "foo(\"a,b\").out"); + EXPECT_EQUAL(B().baseName("foo").parameter("a\\").output("out").buildName(), "foo(\"a\\\\\").out"); + EXPECT_EQUAL(B().baseName("foo").parameter("a)").output("out").buildName(), "foo(\"a)\").out"); + EXPECT_EQUAL(B().baseName("foo").parameter(" ").output("out").buildName(), "foo(\" \").out"); + EXPECT_EQUAL(B().baseName("foo").parameter("\"").output("out").buildName(), "foo(\"\\\"\").out"); + EXPECT_EQUAL(B().baseName("foo").parameter("\\\t\n\r\f\x15").output("out").buildName(), "foo(\"\\\\\\t\\n\\r\\f\\x15\").out"); + EXPECT_EQUAL(B().baseName("foo").parameter("\\\t\n\r\f\x20").output("out").buildName(), "foo(\"\\\\\\t\\n\\r\\f \").out"); + + // empty parameters + EXPECT_EQUAL(B().baseName("foo").parameter("").output("out").buildName(), "foo().out"); + EXPECT_EQUAL(B().baseName("foo").parameter("").parameter("").output("out").buildName(), "foo(,).out"); + EXPECT_EQUAL(B().baseName("foo").parameter("").parameter("").parameter("").output("out").buildName(), "foo(,,).out"); + EXPECT_EQUAL(B().baseName("foo").parameter("").parameter("x").parameter("").output("out").buildName(), "foo(,x,).out"); + + // test change components + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").output("out").buildName(), "foo(a,b).out"); + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").output("out").baseName("bar").buildName(), "bar(a,b).out"); + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").output("out").clearParameters().buildName(), "foo.out"); + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").output("out").clearParameters().parameter("x").buildName(), "foo(x).out"); + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").output("out").output("").buildName(), "foo(a,b)"); + EXPECT_EQUAL(B().baseName("foo").parameter("a").parameter("b").output("out").output("len").buildName(), "foo(a,b).len"); + + // test exact quote vs non-quote + EXPECT_EQUAL(B().baseName("foo").parameter("a").buildName(), "foo(a)"); + EXPECT_EQUAL(B().baseName("foo").parameter(" a").buildName(), "foo(\" a\")"); + EXPECT_EQUAL(B().baseName("foo").parameter("a.out").buildName(), "foo(a.out)"); + EXPECT_EQUAL(B().baseName("foo").parameter(" a.out").buildName(), "foo(\" a.out\")"); + EXPECT_EQUAL(B().baseName("foo").parameter("bar(a,b)").buildName(), "foo(bar(a,b))"); + EXPECT_EQUAL(B().baseName("foo").parameter("bar(a, b)").buildName(), "foo(\"bar(a, b)\")"); + EXPECT_EQUAL(B().baseName("foo").parameter("bar(a,b).out").buildName(), "foo(bar(a,b).out)"); + EXPECT_EQUAL(B().baseName("foo").parameter("bar(a, b).out").buildName(), "foo(\"bar(a, b).out\")"); + + // test non-exact quote vs non-quote + EXPECT_EQUAL(B().baseName("foo").parameter(" \t\n\r\f", false).buildName(), "foo()"); + EXPECT_EQUAL(B().baseName("foo").parameter(" \t\n\r\fbar ", false).buildName(), "foo(bar)"); + EXPECT_EQUAL(B().baseName("foo").parameter(" bar ", false).buildName(), "foo(bar)"); + EXPECT_EQUAL(B().baseName("foo").parameter(" a b ", false).buildName(), "foo(\" a b \")"); + EXPECT_EQUAL(B().baseName("foo").parameter("a%", false).buildName(), "foo(\"a%\")"); + EXPECT_EQUAL(B().baseName("foo").parameter("foo\"\\", false).buildName(), "foo(\"foo\\\"\\\\\")"); + EXPECT_EQUAL(B().baseName("foo").parameter(" a . out ", false).buildName(), "foo(a.out)"); + EXPECT_EQUAL(B().baseName("foo").parameter(" bar ( a , b ) ", false).buildName(), "foo(bar(a,b))"); + EXPECT_EQUAL(B().baseName("foo").parameter(" bar ( a , b ) . out ", false).buildName(), "foo(bar(a,b).out)"); + EXPECT_EQUAL(B().baseName("foo").parameter(" bar ( a , b ) . out.2 ", false).buildName(), "foo(bar(a,b).out.2)"); + EXPECT_EQUAL(B().baseName("foo").parameter(" bar ( a , b ) . out . 2 ", false).buildName(), "foo(\" bar ( a , b ) . out . 2 \")"); + + TEST_DONE(); +} diff --git a/searchlib/src/tests/fef/featurenameparser/.gitignore b/searchlib/src/tests/fef/featurenameparser/.gitignore new file mode 100644 index 00000000000..f16080e9791 --- /dev/null +++ b/searchlib/src/tests/fef/featurenameparser/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +featurenameparser_test +searchlib_featurenameparser_test_app diff --git a/searchlib/src/tests/fef/featurenameparser/CMakeLists.txt b/searchlib/src/tests/fef/featurenameparser/CMakeLists.txt new file mode 100644 index 00000000000..e313ee24deb --- /dev/null +++ b/searchlib/src/tests/fef/featurenameparser/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_featurenameparser_test_app + SOURCES + featurenameparser_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_featurenameparser_test_app COMMAND searchlib_featurenameparser_test_app) diff --git a/searchlib/src/tests/fef/featurenameparser/DESC b/searchlib/src/tests/fef/featurenameparser/DESC new file mode 100644 index 00000000000..4c3da4e47a2 --- /dev/null +++ b/searchlib/src/tests/fef/featurenameparser/DESC @@ -0,0 +1 @@ +featurenameparser test. Take a look at featurenameparser.cpp for details. diff --git a/searchlib/src/tests/fef/featurenameparser/FILES b/searchlib/src/tests/fef/featurenameparser/FILES new file mode 100644 index 00000000000..4567d5b7ccc --- /dev/null +++ b/searchlib/src/tests/fef/featurenameparser/FILES @@ -0,0 +1 @@ +featurenameparser.cpp diff --git a/searchlib/src/tests/fef/featurenameparser/featurenameparser_test.cpp b/searchlib/src/tests/fef/featurenameparser/featurenameparser_test.cpp new file mode 100644 index 00000000000..2824f5ef8fc --- /dev/null +++ b/searchlib/src/tests/fef/featurenameparser/featurenameparser_test.cpp @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("featurenameparser_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/fef/featurenameparser.h> +#include <vector> +#include <string> + +using namespace search::fef; + +struct ParamList { + std::vector<vespalib::string> list; + ParamList() : list() {} + ParamList(const std::vector<vespalib::string> &l) : list(l) {} + ParamList &add(const vespalib::string &str) { + list.push_back(str); + return *this; + } + bool operator==(const ParamList &rhs) const { + return rhs.list == list; + } +}; + +std::ostream &operator<<(std::ostream &os, const ParamList &pl) { + os << std::endl; + for (uint32_t i = 0; i < pl.list.size(); ++i) { + os << " " << pl.list[i] << std::endl; + } + return os; +} + +class Test : public vespalib::TestApp +{ +public: + bool testParse(const vespalib::string &input, bool valid, + const vespalib::string &base, ParamList pl, + const vespalib::string &output); + void testFile(const vespalib::string &name); + int Main(); +}; + +bool +Test::testParse(const vespalib::string &input, bool valid, + const vespalib::string &base, ParamList pl, + const vespalib::string &output) +{ + bool ok = true; + FeatureNameParser parser(input); + if (!parser.valid()) { + LOG(warning, "parse error: input:'%s', rest:'%s'", + input.c_str(), input.substr(parser.parsedBytes()).c_str()); + } + ok &= EXPECT_EQUAL(parser.valid(), valid); + ok &= EXPECT_EQUAL(parser.baseName(), base); + ok &= EXPECT_EQUAL(ParamList(parser.parameters()), pl); + ok &= EXPECT_EQUAL(parser.output(), output); + return ok; +} + +void +Test::testFile(const vespalib::string &name) +{ + char buf[4096]; + uint32_t lineN = 0; + FILE *f = fopen(name.c_str(), "r"); + ASSERT_TRUE(f != 0); + while (fgets(buf, sizeof(buf), f) != NULL) { + ++lineN; + vespalib::string line(buf); + if (*line.rbegin() == '\n') { + line.resize(line.size() - 1); + } + if (line.empty() || line[0] == '#') { + continue; + } + uint32_t idx = line.find("<=>"); + if (!EXPECT_TRUE(idx < line.size())) { + LOG(error, "(%s:%u): malformed line: '%s'", + name.c_str(), lineN, line.c_str()); + } else { + vespalib::string input = line.substr(0, idx); + vespalib::string expect = line.substr(idx + strlen("<=>")); + if (!EXPECT_EQUAL(FeatureNameParser(input).featureName(), expect)) { + LOG(error, "(%s:%u): test failed: '%s'", + name.c_str(), lineN, line.c_str()); + } + } + } + ASSERT_TRUE(!ferror(f)); + fclose(f); +} + +int +Test::Main() +{ + TEST_INIT("featurenameparser_test"); + + // normal cases + EXPECT_TRUE(testParse("foo", true, "foo", ParamList(), "")); + EXPECT_TRUE(testParse("foo.out", true, "foo", ParamList(), "out")); + EXPECT_TRUE(testParse("foo(a)", true, "foo", ParamList().add("a"), "")); + EXPECT_TRUE(testParse("foo(a,b)", true, "foo", ParamList().add("a").add("b"), "")); + EXPECT_TRUE(testParse("foo(a,b).out", true, "foo", ParamList().add("a").add("b"), "out")); + + // @ in feature name (for macros) + EXPECT_TRUE(testParse("foo@", true, "foo@", ParamList(), "")); + EXPECT_TRUE(testParse("foo@.out", true, "foo@", ParamList(), "out")); + EXPECT_TRUE(testParse("foo@(a)", true, "foo@", ParamList().add("a"), "")); + EXPECT_TRUE(testParse("foo@(a,b)", true, "foo@", ParamList().add("a").add("b"), "")); + EXPECT_TRUE(testParse("foo@(a,b).out", true, "foo@", ParamList().add("a").add("b"), "out")); + + // $ in feature name (for macros) + EXPECT_TRUE(testParse("foo$", true, "foo$", ParamList(), "")); + EXPECT_TRUE(testParse("foo$.out", true, "foo$", ParamList(), "out")); + EXPECT_TRUE(testParse("foo$(a)", true, "foo$", ParamList().add("a"), "")); + EXPECT_TRUE(testParse("foo$(a,b)", true, "foo$", ParamList().add("a").add("b"), "")); + EXPECT_TRUE(testParse("foo$(a,b).out", true, "foo$", ParamList().add("a").add("b"), "out")); + + // de-quoting of parameters + EXPECT_TRUE(testParse("foo(a,\"b\")", true, "foo", ParamList().add("a").add("b"), "")); + EXPECT_TRUE(testParse("foo(a,\" b \")", true, "foo", ParamList().add("a").add(" b "), "")); + EXPECT_TRUE(testParse("foo( \"a\" , \" b \" )", true, "foo", ParamList().add("a").add(" b "), "")); + EXPECT_TRUE(testParse("foo(\"\\\"\\\\\\t\\n\\r\\f\\x20\")", true, "foo", ParamList().add("\"\\\t\n\r\f "), "")); + + // only default output if '.' not specified + EXPECT_TRUE(testParse("foo.", false, "", ParamList(), "")); + EXPECT_TRUE(testParse("foo(a,b).", false, "", ParamList(), "")); + + // string cannot end in parameter list + EXPECT_TRUE(testParse("foo(", false, "", ParamList(), "")); + EXPECT_TRUE(testParse("foo(a", false, "", ParamList(), "")); + EXPECT_TRUE(testParse("foo(a\\", false, "", ParamList(), "")); + EXPECT_TRUE(testParse("foo(a\\)", false, "", ParamList(), "")); + EXPECT_TRUE(testParse("foo(a,", false, "", ParamList(), "")); + EXPECT_TRUE(testParse("foo(a,b", false, "", ParamList(), "")); + + // empty parameters + EXPECT_TRUE(testParse("foo()", true, "foo", ParamList().add(""), "")); + EXPECT_TRUE(testParse("foo(,)", true, "foo", ParamList().add("").add(""), "")); + EXPECT_TRUE(testParse("foo(,,)", true, "foo", ParamList().add("").add("").add(""), "")); + EXPECT_TRUE(testParse("foo(,x,)", true, "foo", ParamList().add("").add("x").add(""), "")); + EXPECT_TRUE(testParse("foo( )", true, "foo", ParamList().add(""), "")); + EXPECT_TRUE(testParse("foo( , , )", true, "foo", ParamList().add("").add("").add(""), "")); + EXPECT_TRUE(testParse("foo( \t , \n , \r , \f )", true, "foo", ParamList().add("").add("").add("").add(""), "")); + + testFile("parsetest.txt"); + TEST_DONE(); +} + +TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/fef/featurenameparser/parsetest.txt b/searchlib/src/tests/fef/featurenameparser/parsetest.txt new file mode 100644 index 00000000000..ce9db595eca --- /dev/null +++ b/searchlib/src/tests/fef/featurenameparser/parsetest.txt @@ -0,0 +1,55 @@ +# This file is used to test feature name parsing. The file format is +# as follows: Empty lines and lines starting with '#' will be +# ignored. Other lines must be on the form +# "<input>'<=>'<expected_output>". The parser will be run on the +# input, and the normalized feature name will be compared to the +# expected output. If they match the test passes, if they don't match +# the test fails. The normalized feature name in the case of a parse +# error is the empty string. When parsing this file, no whitespace +# skipping is allowed inside the input or the expected output. To +# simplify things, the byte sequence '<=>' may not be used anywhere +# else than as a separator between the input and the expected +# output. Malformed lines will result in a failed test. + +# basic normalization + foo . out <=>foo.out + foo ( a , b ) . out <=>foo(a,b).out + foo ( a , b , "") . out <=>foo(a,b,).out + foo ( bar ( a ) , b , "") . out <=>foo(bar(a),b,).out + +# basic parse errors +<=> + <=> +foo(<=> +foo(,<=> +foo().<=> +foo(a b)<=> +foo(bar(a b))<=> +foo . a . b<=> + +#quoting +foo("a b")<=>foo("a b") +foo(bar("a b"))<=>foo(bar("a b")) +foo("\"bar\"")<=>foo("\"bar\"") +foo( "bar(x)" )<=>foo(bar(x)) +foo( "bar( x )" )<=>foo("bar( x )") +foo("xyz")<=>foo(xyz) +foo("\\\t\n\r\f\x10")<=>foo("\\\t\n\r\f\x10") +foo("\y")<=> +foo("\x05")<=>foo("\x05") +foo("\x00")<=> +foo("\")<=> +foo("abc<=> +foo("\x5")<=> +foo("\x31\x32\x33")<=>foo(123) + +# my current favorite pair :) +foo("bar(\"x\")")<=>foo("bar(\"x\")") +foo("bar(\"x \")")<=>foo(bar("x ")) + +# might want to disallow non-printables inside quotes... +foo(" ")<=>foo("\t") + +#some more fancy normalization tests + foo ( a , b ) . out <=>foo(a,b).out + foo ( "", bar ( baz ( a, "" ) , "" ) , b , " ") . out <=>foo(,bar(baz(a,),),b," ").out diff --git a/searchlib/src/tests/fef/featureoverride/.gitignore b/searchlib/src/tests/fef/featureoverride/.gitignore new file mode 100644 index 00000000000..35285582ceb --- /dev/null +++ b/searchlib/src/tests/fef/featureoverride/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +featureoverride_test +searchlib_featureoverride_test_app diff --git a/searchlib/src/tests/fef/featureoverride/CMakeLists.txt b/searchlib/src/tests/fef/featureoverride/CMakeLists.txt new file mode 100644 index 00000000000..23370d51d22 --- /dev/null +++ b/searchlib/src/tests/fef/featureoverride/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_featureoverride_test_app + SOURCES + featureoverride.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_featureoverride_test_app COMMAND searchlib_featureoverride_test_app) diff --git a/searchlib/src/tests/fef/featureoverride/DESC b/searchlib/src/tests/fef/featureoverride/DESC new file mode 100644 index 00000000000..1605959dae6 --- /dev/null +++ b/searchlib/src/tests/fef/featureoverride/DESC @@ -0,0 +1 @@ +featureoverride test. Take a look at featureoverride.cpp for details. diff --git a/searchlib/src/tests/fef/featureoverride/FILES b/searchlib/src/tests/fef/featureoverride/FILES new file mode 100644 index 00000000000..864ca65657a --- /dev/null +++ b/searchlib/src/tests/fef/featureoverride/FILES @@ -0,0 +1 @@ +featureoverride.cpp diff --git a/searchlib/src/tests/fef/featureoverride/featureoverride.cpp b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp new file mode 100644 index 00000000000..b0929f50fa9 --- /dev/null +++ b/searchlib/src/tests/fef/featureoverride/featureoverride.cpp @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("featureoverride_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/fef/fef.h> + +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/searchlib/fef/test/plugin/double.h> +#include <vespa/searchlib/fef/test/plugin/sum.h> +#include <vespa/searchlib/features/valuefeature.h> + +using namespace search::fef; +using namespace search::fef::test; +using namespace search::features; +using search::feature_t; + +typedef FeatureExecutor::LP FESP; +typedef Blueprint::SP BPSP; + +struct Fixture +{ + MatchDataLayout mdl; + std::vector<FeatureExecutor::LP> executors; + MatchData::UP md; + Fixture() : mdl(), executors(), md() {} + Fixture &add(FeatureExecutor::LP &executor, size_t outCnt) { + executor->inputs_done(); + for (uint32_t outIdx = 0; outIdx < outCnt; ++outIdx) { + executor->bindOutput(mdl.allocFeature()); + } + executor->outputs_done(); + executors.push_back(executor); + return *this; + } + Fixture &run() { + md = mdl.createMatchData(); + for (const auto &executor : executors) { + executor->execute(*md); + } + return *this; + } + feature_t resolveFeature(FeatureHandle handle) { + return *md->resolveFeature(handle); + } + FESP createValueExecutor() { + std::vector<feature_t> values; + values.push_back(1.0); + values.push_back(2.0); + values.push_back(3.0); + return FESP(new ValueExecutor(values)); + } +}; + +TEST_F("test decorator - single override", Fixture) +{ + FESP fe = f.createValueExecutor(); + fe = FESP(new FeatureOverrider(fe, 1, 50.0)); + f.add(fe, 3).run(); + EXPECT_EQUAL(fe->outputs().size(), 3u); + + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[0]), 1.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[1]), 50.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[2]), 3.0); +} + +TEST_F("test decorator - multiple overrides", Fixture) +{ + FESP fe = f.createValueExecutor(); + fe = FESP(new FeatureOverrider(fe, 0, 50.0)); + fe = FESP(new FeatureOverrider(fe, 2, 100.0)); + f.add(fe, 3).run(); + EXPECT_EQUAL(fe->outputs().size(), 3u); + + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[0]), 50.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[1]), 2.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[2]), 100.0); +} + +TEST_F("test decorator - non-existing override", Fixture) +{ + FESP fe = f.createValueExecutor(); + fe = FESP(new FeatureOverrider(fe, 1000, 50.0)); + f.add(fe, 3).run(); + EXPECT_EQUAL(fe->outputs().size(), 3u); + + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[0]), 1.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[1]), 2.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[2]), 3.0); +} + +TEST_F("test decorator - transitive override", Fixture) +{ + FeatureExecutor::SharedInputs inputs; + FESP fe = f.createValueExecutor(); + fe = FESP(new FeatureOverrider(fe, 1, 50.0)); + f.add(fe, 3); + EXPECT_EQUAL(fe->outputs().size(), 3u); + + FESP fe2 = FESP(new DoubleExecutor(3)); + fe2->bind_shared_inputs(inputs); + fe2->addInput(fe->outputs()[0]); + fe2->addInput(fe->outputs()[1]); + fe2->addInput(fe->outputs()[2]); + fe2 = FESP(new FeatureOverrider(fe2, 2, 10.0)); + f.add(fe2, 3).run(); + EXPECT_EQUAL(fe2->outputs().size(), 3u); + + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[0]), 1.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[1]), 50.0); + EXPECT_EQUAL(f.resolveFeature(fe->outputs()[2]), 3.0); + EXPECT_EQUAL(f.resolveFeature(fe2->outputs()[0]), 2.0); + EXPECT_EQUAL(f.resolveFeature(fe2->outputs()[1]), 100.0); + EXPECT_EQUAL(f.resolveFeature(fe2->outputs()[2]), 10.0); +} + +TEST("test overrides") +{ + BlueprintFactory bf; + bf.addPrototype(BPSP(new ValueBlueprint())); + bf.addPrototype(BPSP(new DoubleBlueprint())); + bf.addPrototype(BPSP(new SumBlueprint())); + + IndexEnvironment idxEnv; + RankSetup rs(bf, idxEnv); + + rs.addDumpFeature("value(1,2,3)"); + rs.addDumpFeature("double(value(1))"); + rs.addDumpFeature("double(value(2))"); + rs.addDumpFeature("double(value(3))"); + rs.addDumpFeature("mysum(value(2),value(2))"); + rs.addDumpFeature("mysum(value(1),value(2),value(3))"); + EXPECT_TRUE(rs.compile()); + + RankProgram::UP rankProgram = rs.create_dump_program(); + + MatchDataLayout mdl; + QueryEnvironment queryEnv; + Properties overrides; + + overrides.add("value(2)", "20.0"); + overrides.add("value(1,2,3).1", "4.0"); + overrides.add("value(1,2,3).2", "6.0"); + overrides.add("bogus(feature)", "10.0"); + + rankProgram->setup(mdl, queryEnv, overrides); + rankProgram->run(2); + + std::map<vespalib::string, feature_t> res = Utils::getAllFeatures(*rankProgram); + + EXPECT_EQUAL(res.size(), 20u); + EXPECT_APPROX(res["value(1)"], 1.0, 1e-6); + EXPECT_APPROX(res["value(1).0"], 1.0, 1e-6); + EXPECT_APPROX(res["value(2)"], 20.0, 1e-6); + EXPECT_APPROX(res["value(2).0"], 20.0, 1e-6); + EXPECT_APPROX(res["value(3)"], 3.0, 1e-6); + EXPECT_APPROX(res["value(3).0"], 3.0, 1e-6); + EXPECT_APPROX(res["value(1,2,3)"], 1.0, 1e-6); + EXPECT_APPROX(res["value(1,2,3).0"], 1.0, 1e-6); + EXPECT_APPROX(res["value(1,2,3).1"], 4.0, 1e-6); + EXPECT_APPROX(res["value(1,2,3).2"], 6.0, 1e-6); + EXPECT_APPROX(res["mysum(value(2),value(2))"], 40.0, 1e-6); + EXPECT_APPROX(res["mysum(value(2),value(2)).out"], 40.0, 1e-6); + EXPECT_APPROX(res["mysum(value(1),value(2),value(3))"], 24.0, 1e-6); + EXPECT_APPROX(res["mysum(value(1),value(2),value(3)).out"], 24.0, 1e-6); + EXPECT_APPROX(res["double(value(1))"], 2.0, 1e-6); + EXPECT_APPROX(res["double(value(1)).0"], 2.0, 1e-6); + EXPECT_APPROX(res["double(value(2))"], 40.0, 1e-6); + EXPECT_APPROX(res["double(value(2)).0"], 40.0, 1e-6); + EXPECT_APPROX(res["double(value(3))"], 6.0, 1e-6); + EXPECT_APPROX(res["double(value(3)).0"], 6.0, 1e-6); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp new file mode 100644 index 00000000000..b3107e57fae --- /dev/null +++ b/searchlib/src/tests/fef/fef_test.cpp @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("fef_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/fef/fef.h> +#include <vespa/searchlib/fef/objectstore.h> + +using namespace search::fef; +using std::shared_ptr; +using search::feature_t; + +class Test : public vespalib::TestApp +{ +public: + void testLayout(); + void testObjectStore(); + void testTermFieldMatchDataAppend(); + int Main(); +}; + +void +Test::testLayout() +{ + { + TermFieldMatchData tmd; + EXPECT_EQUAL(IllegalFieldId, tmd.getFieldId()); + EXPECT_EQUAL(TermFieldMatchData::invalidId(), tmd.getDocId()); + } + MatchDataLayout mdl; + EXPECT_EQUAL(mdl.allocTermField(0), 0u); + EXPECT_EQUAL(mdl.allocTermField(42), 1u); + EXPECT_EQUAL(mdl.allocTermField(IllegalFieldId), 2u); + EXPECT_EQUAL(mdl.allocFeature(), 0u); + EXPECT_EQUAL(mdl.allocFeature(), 1u); + EXPECT_EQUAL(mdl.allocFeature(), 2u); + + MatchData::UP md = mdl.createMatchData(); + EXPECT_EQUAL(TermFieldMatchData::invalidId(), md->getDocId()); + EXPECT_EQUAL(md->getNumTermFields(), 3u); + EXPECT_EQUAL(md->getNumFeatures(), 3u); + TermFieldMatchData *t0 = md->resolveTermField(0); + TermFieldMatchData *t1 = md->resolveTermField(1); + TermFieldMatchData *t2 = md->resolveTermField(2); + EXPECT_EQUAL(t1, t0 + 1); + EXPECT_EQUAL(t2, t1 + 1); + EXPECT_EQUAL(0u, t0->getFieldId()); + EXPECT_EQUAL(42u, t1->getFieldId()); + EXPECT_EQUAL(IllegalFieldId, t2->getFieldId()); + feature_t *f0 = md->resolveFeature(0); + feature_t *f1 = md->resolveFeature(1); + feature_t *f2 = md->resolveFeature(2); + EXPECT_EQUAL(f1, f0 + 1); + EXPECT_EQUAL(f2, f1 + 1); + EXPECT_TRUE((void*)t2 < (void*)f0 || (void*)f2 < (void*)t0); +} + +void +Test::testObjectStore() +{ + ObjectStore s; + class Object : public Anything { + }; + Anything::UP u1(new Object()); + Anything::UP u11(new Object()); + Anything::UP u2(new Object()); + const Anything * o1(u1.get()); + const Anything * o11(u11.get()); + const Anything * o2(u2.get()); + EXPECT_TRUE(nullptr == s.get("a")); + s.add("a", std::move(u1)); + EXPECT_EQUAL(o1, s.get("a")); + EXPECT_TRUE(nullptr == s.get("b")); + s.add("b", std::move(u2)); + EXPECT_EQUAL(o1, s.get("a")); + EXPECT_EQUAL(o2, s.get("b")); + s.add("a", std::move(u11)); + EXPECT_EQUAL(o11, s.get("a")); +} + +void +Test::testTermFieldMatchDataAppend() +{ + TermFieldMatchData tmd; + EXPECT_EQUAL(0u, tmd.size()); + EXPECT_EQUAL(1u, tmd.capacity()); + TermFieldMatchDataPosition pos; + tmd.appendPosition(pos); + EXPECT_EQUAL(1u, tmd.size()); + EXPECT_EQUAL(1u, tmd.capacity()); + tmd.appendPosition(pos); + EXPECT_EQUAL(2u, tmd.size()); + EXPECT_EQUAL(2u, tmd.capacity()); + for (size_t i(2); i < std::numeric_limits<uint16_t>::max(); i++) { + EXPECT_EQUAL(i, tmd.size()); + EXPECT_EQUAL(std::min(size_t(std::numeric_limits<uint16_t>::max()), vespalib::roundUp2inN(i)), tmd.capacity()); + tmd.appendPosition(pos); + } + EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.size()); + EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.capacity()); + tmd.appendPosition(pos); + EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.size()); + EXPECT_EQUAL(std::numeric_limits<uint16_t>::max(), tmd.capacity()); +} + +int +Test::Main() +{ + TEST_INIT("fef_test"); + testLayout(); + testObjectStore(); + testTermFieldMatchDataAppend(); + TEST_DONE(); +} + +TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/fef/object_passing/.gitignore b/searchlib/src/tests/fef/object_passing/.gitignore new file mode 100644 index 00000000000..64b250201a8 --- /dev/null +++ b/searchlib/src/tests/fef/object_passing/.gitignore @@ -0,0 +1 @@ +searchlib_object_passing_test_app diff --git a/searchlib/src/tests/fef/object_passing/CMakeLists.txt b/searchlib/src/tests/fef/object_passing/CMakeLists.txt new file mode 100644 index 00000000000..2334711f015 --- /dev/null +++ b/searchlib/src/tests/fef/object_passing/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_object_passing_test_app + SOURCES + object_passing_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_object_passing_test_app COMMAND searchlib_object_passing_test_app) diff --git a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp new file mode 100644 index 00000000000..69c681d8f60 --- /dev/null +++ b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp @@ -0,0 +1,128 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/searchlib/features/valuefeature.h> +#include <vespa/searchlib/fef/blueprintfactory.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/searchlib/fef/test/plugin/sum.h> +#include <vespa/searchlib/fef/rank_program.h> +#include <vespa/searchlib/fef/verify_feature.h> +#include <vespa/vespalib/eval/value_type.h> +#include <vespa/searchlib/fef/feature_type.h> + +using namespace search::fef; +using namespace search::fef::test; +using namespace search::features; +using vespalib::eval::ValueType; + +struct ProxyExecutor : FeatureExecutor { + double number_value; + vespalib::eval::Value::UP object_value; + ProxyExecutor() : number_value(0.0), object_value() {} + bool isPure() override { return true; } + void execute(search::fef::MatchData &md) override { + double was_object = 0.0; + if (md.feature_is_object(inputs()[0])) { + was_object = 1.0; + number_value = md.resolve_object_feature(inputs()[0])->get().as_double(); + object_value.reset(new vespalib::eval::DoubleValue(number_value)); + } else { + number_value = *md.resolveFeature(inputs()[0]); + object_value.reset(new vespalib::eval::DoubleValue(number_value)); + } + if (md.feature_is_object(outputs()[0])) { + *md.resolve_object_feature(outputs()[0]) = *object_value; + } else { + *md.resolveFeature(outputs()[0]) = number_value; + } + *md.resolveFeature(outputs()[1]) = was_object; + } +}; + +struct ProxyBlueprint : Blueprint { + vespalib::string name; + AcceptInput accept_input; + bool object_output; + ProxyBlueprint(const vespalib::string &name_in, AcceptInput accept_input_in, bool object_output_in) + : Blueprint(name_in), name(name_in), accept_input(accept_input_in), object_output(object_output_in) {} + void visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const override {} + Blueprint::UP createInstance() const override { + return Blueprint::UP(new ProxyBlueprint(name, accept_input, object_output)); + } + bool setup(const IIndexEnvironment &, const std::vector<vespalib::string> ¶ms) override { + ASSERT_EQUAL(1u, params.size()); + defineInput(params[0], accept_input); + describeOutput("value", "the value", object_output ? FeatureType::object(ValueType::double_type()) : FeatureType::number()); + describeOutput("was_object", "whether input was object", FeatureType::number()); + return true; + } + FeatureExecutor::LP createExecutor(const IQueryEnvironment &) const override { + return FeatureExecutor::LP(new ProxyExecutor()); + } +}; + +struct Fixture { + BlueprintFactory factory; + IndexEnvironment indexEnv; + + explicit Fixture() { + factory.addPrototype(Blueprint::SP(new ValueBlueprint())); + factory.addPrototype(Blueprint::SP(new ProxyBlueprint("box", Blueprint::AcceptInput::NUMBER, true))); + factory.addPrototype(Blueprint::SP(new ProxyBlueprint("maybe_box", Blueprint::AcceptInput::ANY, true))); + factory.addPrototype(Blueprint::SP(new ProxyBlueprint("unbox", Blueprint::AcceptInput::OBJECT, false))); + factory.addPrototype(Blueprint::SP(new ProxyBlueprint("maybe_unbox", Blueprint::AcceptInput::ANY, false))); + } + + double eval(const vespalib::string &feature) { + BlueprintResolver::SP resolver(new BlueprintResolver(factory, indexEnv)); + resolver->addSeed(feature); + if (!resolver->compile()) { + return vespalib::eval::error_value; + } + MatchDataLayout mdl; + QueryEnvironment queryEnv(&indexEnv); + Properties overrides; + RankProgram program(resolver); + program.setup(mdl, queryEnv, overrides); + program.run(1); + std::vector<vespalib::string> names; + std::vector<FeatureHandle> handles; + program.get_seed_handles(names, handles); + EXPECT_EQUAL(1u, names.size()); + EXPECT_EQUAL(names.size(), handles.size()); + const auto &md = program.match_data(); + EXPECT_TRUE(!md.feature_is_object(handles[0])); // verifies auto-unboxing + return *md.resolveFeature(handles[0]); + } + + bool verify(const vespalib::string &feature) { + return verifyFeature(factory, indexEnv, feature, "unit test"); + } +}; + +TEST_F("require that values can be boxed and unboxed", Fixture()) { + EXPECT_EQUAL(3.0, f1.eval("box(value(3))")); + EXPECT_EQUAL(0.0, f1.eval("box(value(3)).was_object")); + EXPECT_EQUAL(3.0, f1.eval("unbox(box(value(3)))")); + EXPECT_EQUAL(1.0, f1.eval("unbox(box(value(3))).was_object")); + EXPECT_EQUAL(3.0, f1.eval("box(unbox(box(value(3))))")); + EXPECT_EQUAL(0.0, f1.eval("box(unbox(box(value(3)))).was_object")); +} + +TEST_F("require that output features may be either objects or numbers", Fixture()) { + EXPECT_TRUE(f1.verify("value(3)")); + EXPECT_TRUE(f1.verify("box(value(3))")); +} + +TEST_F("require that feature input/output types must be compatible", Fixture()) { + EXPECT_TRUE(!f1.verify("unbox(value(3))")); + EXPECT_TRUE(f1.verify("maybe_unbox(value(3))")); + EXPECT_TRUE(f1.verify("unbox(box(value(3)))")); + EXPECT_TRUE(!f1.verify("unbox(box(box(value(3))))")); + EXPECT_TRUE(f1.verify("unbox(maybe_box(box(value(3))))")); + EXPECT_TRUE(f1.verify("unbox(box(unbox(box(value(3)))))")); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/parameter/.gitignore b/searchlib/src/tests/fef/parameter/.gitignore new file mode 100644 index 00000000000..17cf6c69953 --- /dev/null +++ b/searchlib/src/tests/fef/parameter/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +parameter_test +searchlib_parameter_test_app diff --git a/searchlib/src/tests/fef/parameter/CMakeLists.txt b/searchlib/src/tests/fef/parameter/CMakeLists.txt new file mode 100644 index 00000000000..dcd45390ce3 --- /dev/null +++ b/searchlib/src/tests/fef/parameter/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_parameter_test_app + SOURCES + parameter_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_parameter_test_app NO_VALGRIND COMMAND searchlib_parameter_test_app) diff --git a/searchlib/src/tests/fef/parameter/DESC b/searchlib/src/tests/fef/parameter/DESC new file mode 100644 index 00000000000..738e0dbd512 --- /dev/null +++ b/searchlib/src/tests/fef/parameter/DESC @@ -0,0 +1 @@ +parameter test. Take a look at parameter.cpp for details. diff --git a/searchlib/src/tests/fef/parameter/FILES b/searchlib/src/tests/fef/parameter/FILES new file mode 100644 index 00000000000..20c9e0c9ba0 --- /dev/null +++ b/searchlib/src/tests/fef/parameter/FILES @@ -0,0 +1 @@ +parameter.cpp diff --git a/searchlib/src/tests/fef/parameter/parameter_test.cpp b/searchlib/src/tests/fef/parameter/parameter_test.cpp new file mode 100644 index 00000000000..4d6741937d5 --- /dev/null +++ b/searchlib/src/tests/fef/parameter/parameter_test.cpp @@ -0,0 +1,267 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("parameter_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchlib/fef/parametervalidator.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> + +using namespace search::fef::test; + +namespace search { +namespace fef { + +class StringList : public std::vector<vespalib::string> { +public: + StringList & add(const vespalib::string & str) { push_back(str); return *this; } +}; + +class ParameterTest : public vespalib::TestApp { +private: + typedef ParameterDescriptions PDS; + typedef ParameterType PT; + typedef Parameter P; + typedef StringList SL; + typedef ParameterValidator::Result PVR; + + bool assertParameter(const Parameter & exp, const Parameter & act); + bool validate(const IIndexEnvironment & env, + const std::vector<vespalib::string> & params, + const ParameterDescriptions & descs); + bool validate(const IIndexEnvironment & env, + const std::vector<vespalib::string> & params, + const ParameterDescriptions & descs, + const ParameterValidator::Result & result); + + void testDescriptions(); + void testValidator(); + void testParameters(); + +public: + int Main(); +}; + +bool +ParameterTest::assertParameter(const Parameter & exp, const Parameter & act) +{ + bool retval = true; + if (!EXPECT_EQUAL(exp.getType(), act.getType())) retval = false; + if (!EXPECT_EQUAL(exp.getValue(), act.getValue())) retval = false; + if (!EXPECT_EQUAL(exp.asDouble(), act.asDouble())) retval = false; + if (!EXPECT_EQUAL(exp.asInteger(), act.asInteger())) retval = false; + if (!EXPECT_EQUAL(exp.asField(), act.asField())) retval = false; + return retval; +} + +bool +ParameterTest::validate(const IIndexEnvironment & env, + const std::vector<vespalib::string> & params, + const ParameterDescriptions & descs) +{ + ParameterValidator pv(env, params, descs); + ParameterValidator::Result result = pv.validate(); + LOG(info, "validate(%s)", result.getError().c_str()); + return result.valid(); +} + +bool +ParameterTest::validate(const IIndexEnvironment & env, + const std::vector<vespalib::string> & params, + const ParameterDescriptions & descs, + const ParameterValidator::Result & result) +{ + if (!validate(env, params, descs)) return false; + ParameterValidator pv(env, params, descs); + ParameterValidator::Result actual = pv.validate(); + if (!EXPECT_EQUAL(result.getTag(), actual.getTag())) return false; + if (!EXPECT_EQUAL(result.getParameters().size(), actual.getParameters().size())) return false; + bool retval = true; + for (size_t i = 0; i < result.getParameters().size(); ++i) { + if (!assertParameter(result.getParameters()[i], actual.getParameters()[i])) retval = false; + } + return retval; +} + +void +ParameterTest::testDescriptions() +{ + PDS descs = PDS(). + desc().indexField(ParameterCollection::SINGLE).indexField(ParameterCollection::ARRAY).indexField(ParameterCollection::WEIGHTEDSET).attribute(ParameterCollection::ANY).attributeField(ParameterCollection::ANY).field(). + desc(5).feature().number().string().attribute(ParameterCollection::ANY). + desc().string().number().repeat(2); + const PDS::DescriptionVector & v = descs.getDescriptions(); + EXPECT_EQUAL(v.size(), 3u); + EXPECT_EQUAL(v[0].getTag(), 0u); + EXPECT_TRUE(!v[0].hasRepeat()); + EXPECT_EQUAL(v[0].getParams().size(), 6u); + EXPECT_EQUAL(v[0].getParam(0).type, ParameterType::INDEX_FIELD); + EXPECT_EQUAL(v[0].getParam(1).type, ParameterType::INDEX_FIELD); + EXPECT_EQUAL(v[0].getParam(2).type, ParameterType::INDEX_FIELD); + EXPECT_EQUAL(v[0].getParam(3).type, ParameterType::ATTRIBUTE); + EXPECT_EQUAL(v[0].getParam(4).type, ParameterType::ATTRIBUTE_FIELD); + EXPECT_EQUAL(v[0].getParam(5).type, ParameterType::FIELD); + EXPECT_EQUAL(v[0].getParam(0).collection, ParameterCollection::SINGLE); + EXPECT_EQUAL(v[0].getParam(1).collection, ParameterCollection::ARRAY); + EXPECT_EQUAL(v[0].getParam(2).collection, ParameterCollection::WEIGHTEDSET); + EXPECT_EQUAL(v[0].getParam(3).collection, ParameterCollection::ANY); + EXPECT_EQUAL(v[0].getParam(4).collection, ParameterCollection::ANY); + EXPECT_EQUAL(v[0].getParam(5).collection, ParameterCollection::ANY); + + EXPECT_EQUAL(v[1].getTag(), 5u); + EXPECT_TRUE(!v[1].hasRepeat()); + EXPECT_EQUAL(v[1].getParams().size(), 4u); + EXPECT_EQUAL(v[1].getParam(0).type, ParameterType::FEATURE); + EXPECT_EQUAL(v[1].getParam(1).type, ParameterType::NUMBER); + EXPECT_EQUAL(v[1].getParam(2).type, ParameterType::STRING); + EXPECT_EQUAL(v[1].getParam(3).type, ParameterType::ATTRIBUTE); + + EXPECT_EQUAL(v[2].getTag(), 6u); + EXPECT_TRUE(v[2].hasRepeat()); + EXPECT_EQUAL(v[2].getParams().size(), 2u); + EXPECT_EQUAL(v[2].getParam(0).type, ParameterType::STRING); + EXPECT_EQUAL(v[2].getParam(1).type, ParameterType::NUMBER); + EXPECT_EQUAL(v[2].getParam(2).type, ParameterType::STRING); + EXPECT_EQUAL(v[2].getParam(3).type, ParameterType::NUMBER); + EXPECT_EQUAL(v[2].getParam(4).type, ParameterType::STRING); + EXPECT_EQUAL(v[2].getParam(5).type, ParameterType::NUMBER); +} + +void +ParameterTest::testValidator() +{ + IndexEnvironment env; + IndexEnvironmentBuilder builder(env); + builder.addField(FieldType::INDEX, CollectionType::SINGLE, "foo") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar") + .addField(FieldType::INDEX, CollectionType::ARRAY, "afoo") + .addField(FieldType::INDEX, CollectionType::WEIGHTEDSET, "wfoo") + .addField(FieldType::INDEX, CollectionType::SINGLE, "hybrid"); + env.getFields().back().addAttribute(); // 'hybrid' field can also be accessed as an attribute + + // valid + EXPECT_TRUE(validate(env, SL(), PDS().desc())); + EXPECT_TRUE(validate(env, SL().add("foo"), PDS().desc().field())); + EXPECT_TRUE(validate(env, SL().add("bar"), PDS().desc().field())); + EXPECT_TRUE(validate(env, SL().add("foo"), PDS().desc().indexField(ParameterCollection::SINGLE))); + EXPECT_TRUE(validate(env, SL().add("afoo"), PDS().desc().indexField(ParameterCollection::ARRAY))); + EXPECT_TRUE(validate(env, SL().add("wfoo"), PDS().desc().indexField(ParameterCollection::WEIGHTEDSET))); + EXPECT_TRUE(validate(env, SL().add("foo"), PDS().desc().indexField(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("afoo"), PDS().desc().indexField(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("wfoo"), PDS().desc().indexField(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("bar"), PDS().desc().attribute(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("bar"), PDS().desc().attributeField(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("hybrid"), PDS().desc().attribute(ParameterCollection::ANY))); + EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().feature())); + EXPECT_TRUE(validate(env, SL().add("123"), PDS().desc().number())); + EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().string())); + // first fail but second pass + EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc().field().desc().string())); + + // not valid + EXPECT_FALSE(validate(env, SL().add("baz"), PDS().desc().string().string())); + EXPECT_FALSE(validate(env, SL().add("baz").add("baz"), PDS().desc().string())); + EXPECT_FALSE(validate(env, SL().add("baz"), PDS().desc().field())); + EXPECT_FALSE(validate(env, SL().add("bar"), PDS().desc().indexField(ParameterCollection::SINGLE))); + EXPECT_FALSE(validate(env, SL().add("foo"), PDS().desc().indexField(ParameterCollection::NONE))); + EXPECT_FALSE(validate(env, SL().add("foo"), PDS().desc().indexField(ParameterCollection::ARRAY))); + EXPECT_FALSE(validate(env, SL().add("foo"), PDS().desc().indexField(ParameterCollection::WEIGHTEDSET))); + EXPECT_FALSE(validate(env, SL().add("afoo"), PDS().desc().indexField(ParameterCollection::NONE))); + EXPECT_FALSE(validate(env, SL().add("afoo"), PDS().desc().indexField(ParameterCollection::SINGLE))); + EXPECT_FALSE(validate(env, SL().add("afoo"), PDS().desc().indexField(ParameterCollection::WEIGHTEDSET))); + EXPECT_FALSE(validate(env, SL().add("wfoo"), PDS().desc().indexField(ParameterCollection::NONE))); + EXPECT_FALSE(validate(env, SL().add("wfoo"), PDS().desc().indexField(ParameterCollection::SINGLE))); + EXPECT_FALSE(validate(env, SL().add("wfoo"), PDS().desc().indexField(ParameterCollection::ARRAY))); + EXPECT_FALSE(validate(env, SL().add("unknown"), PDS().desc().attribute(ParameterCollection::ANY))); + EXPECT_FALSE(validate(env, SL().add("unknown"), PDS().desc().attributeField(ParameterCollection::ANY))); + EXPECT_FALSE(validate(env, SL().add("foo"), PDS().desc().attribute(ParameterCollection::ANY))); + EXPECT_FALSE(validate(env, SL().add("foo"), PDS().desc().attributeField(ParameterCollection::ANY))); + EXPECT_FALSE(validate(env, SL().add("hybrid"), PDS().desc().attributeField(ParameterCollection::ANY))); + EXPECT_FALSE(validate(env, SL().add("12a"), PDS().desc().number())); + EXPECT_FALSE(validate(env, SL().add("a12"), PDS().desc().number())); + + // test repeat + PDS d1 = PDS().desc().field().repeat(); + EXPECT_TRUE(validate(env, SL(), d1)); + EXPECT_TRUE(validate(env, SL().add("foo"), d1)); + EXPECT_TRUE(validate(env, SL().add("foo").add("bar"), d1)); + EXPECT_TRUE(!validate(env, SL().add("foo").add("bar").add("baz"), d1)); + PDS d2 = PDS().desc().string().attribute(ParameterCollection::ANY).indexField(ParameterCollection::SINGLE).repeat(2); + EXPECT_TRUE(validate(env, SL().add("str"), d2)); + EXPECT_TRUE(validate(env, SL().add("str").add("bar").add("foo"), d2)); + EXPECT_TRUE(validate(env, SL().add("str").add("bar").add("foo").add("bar").add("foo"), d2)); + EXPECT_TRUE(!validate(env, SL().add("str").add("bar"), d2)); + EXPECT_TRUE(!validate(env, SL().add("str").add("bar").add("foo").add("bar"), d2)); +} + +void +ParameterTest::testParameters() +{ + IndexEnvironment env; + IndexEnvironmentBuilder builder(env); + builder.addField(FieldType::INDEX, CollectionType::SINGLE, "foo") + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar") + .addField(FieldType::INDEX, CollectionType::ARRAY, "afoo") + .addField(FieldType::INDEX, CollectionType::WEIGHTEDSET, "wfoo"); + + const FieldInfo * foo = env.getFieldByName("foo"); + const FieldInfo * bar = env.getFieldByName("bar"); + const FieldInfo * afoo = env.getFieldByName("afoo"); + const FieldInfo * wfoo = env.getFieldByName("wfoo"); + + EXPECT_TRUE(validate(env, SL().add("foo"), PDS().desc().field(), + PVR().addParameter(P(PT::FIELD, "foo").setField(foo)))); // field + EXPECT_TRUE(validate(env, SL().add("foo"), PDS().desc().indexField(ParameterCollection::SINGLE), + PVR().addParameter(P(PT::INDEX_FIELD, "foo").setField(foo)))); // index field + EXPECT_TRUE(validate(env, SL().add("foo"), PDS().desc().indexField(ParameterCollection::ANY), + PVR().addParameter(P(PT::INDEX_FIELD, "foo").setField(foo)))); // index field + EXPECT_TRUE(validate(env, SL().add("afoo"), PDS().desc().indexField(ParameterCollection::ARRAY), + PVR().addParameter(P(PT::INDEX_FIELD, "afoo").setField(afoo)))); // index field + EXPECT_TRUE(validate(env, SL().add("afoo"), PDS().desc().indexField(ParameterCollection::ANY), + PVR().addParameter(P(PT::INDEX_FIELD, "afoo").setField(afoo)))); // index field + EXPECT_TRUE(validate(env, SL().add("wfoo"), PDS().desc().indexField(ParameterCollection::WEIGHTEDSET), + PVR().addParameter(P(PT::INDEX_FIELD, "wfoo").setField(wfoo)))); // index field + EXPECT_TRUE(validate(env, SL().add("wfoo"), PDS().desc().indexField(ParameterCollection::ANY), + PVR().addParameter(P(PT::INDEX_FIELD, "wfoo").setField(wfoo)))); // index field + EXPECT_TRUE(validate(env, SL().add("bar"), PDS().desc().attribute(ParameterCollection::ANY), + PVR().addParameter(P(PT::ATTRIBUTE, "bar").setField(bar)))); // attribute field + EXPECT_TRUE(validate(env, SL().add("feature"), PDS().desc().feature(), + PVR().addParameter(P(PT::FEATURE, "feature")))); // feature + EXPECT_TRUE(validate(env, SL().add("string"), PDS().desc().string(), + PVR().addParameter(P(PT::STRING, "string")))); // string + + // numbers + EXPECT_TRUE(validate(env, SL().add("-100"), PDS().desc().number(), + PVR().addParameter(P(PT::NUMBER, "-100").setDouble(-100).setInteger(-100)))); + EXPECT_TRUE(validate(env, SL().add("100"), PDS().desc().number(), + PVR().addParameter(P(PT::NUMBER, "100").setDouble(100).setInteger(100)))); + EXPECT_TRUE(validate(env, SL().add("100.16"), PDS().desc().number(), + PVR().addParameter(P(PT::NUMBER, "100.16").setDouble(100.16).setInteger(100)))); + + EXPECT_TRUE(validate(env, SL(), PDS().desc(), PVR())); // no param + EXPECT_TRUE(validate(env, SL().add("foo").add("bar"), PDS().desc().string().string(), + PVR().addParameter(P(PT::STRING, "foo")).addParameter(P(PT::STRING, "bar")))); // multiple params + EXPECT_TRUE(validate(env, SL().add("foo").add("bar"), PDS().desc().string().repeat(), + PVR().addParameter(P(PT::STRING, "foo")).addParameter(P(PT::STRING, "bar")))); // repeat + EXPECT_TRUE(validate(env, SL().add("baz"), PDS().desc(10).field().desc(20).string(), + PVR(20).addParameter(P(PT::STRING, "baz")))); // second desc matching +} + +int +ParameterTest::Main() +{ + TEST_INIT("parameter_test"); + + testDescriptions(); + testValidator(); + testParameters(); + + TEST_DONE(); +} + +} +} + +TEST_APPHOOK(search::fef::ParameterTest); + diff --git a/searchlib/src/tests/fef/phrasesplitter/.gitignore b/searchlib/src/tests/fef/phrasesplitter/.gitignore new file mode 100644 index 00000000000..418f9961840 --- /dev/null +++ b/searchlib/src/tests/fef/phrasesplitter/.gitignore @@ -0,0 +1,6 @@ +.depend +Makefile +benchmark +phrasesplitter_test +searchlib_phrasesplitter_test_app +searchlib_benchmark_app diff --git a/searchlib/src/tests/fef/phrasesplitter/CMakeLists.txt b/searchlib/src/tests/fef/phrasesplitter/CMakeLists.txt new file mode 100644 index 00000000000..aa16f3e0a0d --- /dev/null +++ b/searchlib/src/tests/fef/phrasesplitter/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_phrasesplitter_test_app + SOURCES + phrasesplitter_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_phrasesplitter_test_app COMMAND searchlib_phrasesplitter_test_app) +vespa_add_executable(searchlib_benchmark_app + SOURCES + benchmark.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_benchmark_app COMMAND searchlib_benchmark_app BENCHMARK) diff --git a/searchlib/src/tests/fef/phrasesplitter/DESC b/searchlib/src/tests/fef/phrasesplitter/DESC new file mode 100644 index 00000000000..fba49bdb8c0 --- /dev/null +++ b/searchlib/src/tests/fef/phrasesplitter/DESC @@ -0,0 +1 @@ +phrasesplitter test. Take a look at phrasesplitter.cpp for details. diff --git a/searchlib/src/tests/fef/phrasesplitter/FILES b/searchlib/src/tests/fef/phrasesplitter/FILES new file mode 100644 index 00000000000..be37941d0c8 --- /dev/null +++ b/searchlib/src/tests/fef/phrasesplitter/FILES @@ -0,0 +1 @@ +phrasesplitter.cpp diff --git a/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp new file mode 100644 index 00000000000..ca90b1de261 --- /dev/null +++ b/searchlib/src/tests/fef/phrasesplitter/benchmark.cpp @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("phrasesplitter_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <iomanip> +#include <vespa/searchlib/fef/matchdatalayout.h> +#include <vespa/searchlib/fef/phrasesplitter.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> + +namespace search { +namespace fef { + +class Benchmark : public vespalib::TestApp +{ +private: + FastOS_Time _timer; + double _sample; + + void start() { _timer.SetNow(); } + void sample() { _sample = _timer.MilliSecsToNow(); } + void run(size_t numRuns, size_t numPositions); + +public: + Benchmark() : _timer(), _sample(0) {} + int Main(); +}; + +void +Benchmark::run(size_t numRuns, size_t numPositions) +{ + test::QueryEnvironment qe; + std::vector<SimpleTermData> &terms = qe.getTerms(); + MatchDataLayout mdl; + terms.push_back(SimpleTermData()); + terms.back().setUniqueId(1); + terms.back().setPhraseLength(3); // phrase with 3 terms + terms.back().addField(0).setHandle(mdl.allocTermField(0)); + MatchData::UP md = mdl.createMatchData(); + TermFieldMatchData *tmd = md->resolveTermField(terms[0].lookupField(0)->getHandle()); + for (size_t i = 0; i < numPositions; ++i) { + tmd->appendPosition(TermFieldMatchDataPosition(0, i, 0, numPositions)); + } + + PhraseSplitter ps(qe, 0); + + std::cout << "Start benchmark with numRuns(" << numRuns << ") and numPositions(" << numPositions << ")" << std::endl; + + start(); + + for (size_t i = 0; i < numRuns; ++i) { + ps.update(*md); + } + + sample(); +} + +int +Benchmark::Main() +{ + + TEST_INIT("benchmark"); + + if (_argc != 3) { + std::cout << "Must specify <numRuns> and <numPositions>" << std::endl; + return 0; + } + + size_t numRuns = strtoull(_argv[1], NULL, 10); + size_t numPositions = strtoull(_argv[2], NULL, 10); + + run(numRuns, numPositions); + + std::cout << "TET: " << _sample << " (ms)" << std::endl; + std::cout << "ETPD: " << std::fixed << std::setprecision(10) << _sample / numRuns << " (ms)" << std::endl; + + TEST_DONE(); +} + +} +} + +TEST_APPHOOK(search::fef::Benchmark); diff --git a/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp b/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp new file mode 100644 index 00000000000..0fa6f27022e --- /dev/null +++ b/searchlib/src/tests/fef/phrasesplitter/phrasesplitter_test.cpp @@ -0,0 +1,242 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("phrasesplitter_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchlib/fef/matchdatalayout.h> +#include <vespa/searchlib/fef/phrasesplitter.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> + +namespace search { +namespace fef { + +class PhraseSplitterTest : public vespalib::TestApp +{ +private: + void assertTermData(const ITermData * td, uint32_t uniqueId, uint32_t numTerms, + uint32_t fieldId, uint32_t termHandle); + void testCopyTermFieldMatchData(); + void testSplitter(); + void testSplitterUpdate(); + +public: + int Main(); +}; + +void +PhraseSplitterTest::assertTermData(const ITermData *td, uint32_t uniqueId, uint32_t numTerms, + uint32_t fieldId, uint32_t tfHandle) +{ + // fprintf(stderr, "checking uid=%d numterms=%d field=%d handle=%d\n", uniqueId, numTerms, fieldId, tfHandle); + EXPECT_EQUAL(uniqueId, td->getUniqueId()); + EXPECT_EQUAL(numTerms, td->getPhraseLength()); + EXPECT_EQUAL(tfHandle, td->lookupField(fieldId)->getHandle()); +} + +void +PhraseSplitterTest::testCopyTermFieldMatchData() +{ + TermFieldMatchData src; + src.reset(1); + src.appendPosition(TermFieldMatchDataPosition(0, 5, 0, 1000)); + src.appendPosition(TermFieldMatchDataPosition(0, 15, 0, 1000)); + + SimpleTermData td; + TermFieldMatchData dst; + dst.reset(0); + // dst.setTermData(&td); + dst.appendPosition(TermFieldMatchDataPosition(0, 10, 0, 1000)); + { + FieldPositionsIterator itr = dst.getIterator(); + EXPECT_EQUAL(itr.getPosition(), 10u); + itr.next(); + ASSERT_TRUE(!itr.valid()); + } + + PhraseSplitter::copyTermFieldMatchData(dst, src, 2); + + EXPECT_EQUAL(dst.getDocId(), 1u); + { + TermFieldMatchData::PositionsIterator itr = dst.begin(); + EXPECT_EQUAL(itr->getPosition(), 7u); + ++itr; + EXPECT_EQUAL(itr->getPosition(), 17u); + ++itr; + ASSERT_TRUE(itr == dst.end()); + } + { + FieldPositionsIterator itr = dst.getIterator(); + EXPECT_EQUAL(itr.getPosition(), 7u); + itr.next(); + EXPECT_EQUAL(itr.getPosition(), 17u); + itr.next(); + ASSERT_TRUE(!itr.valid()); + } +} + +void +PhraseSplitterTest::testSplitter() +{ + { // single term + test::QueryEnvironment qe; + std::vector<SimpleTermData> &terms = qe.getTerms(); + MatchDataLayout mdl; + terms.push_back(SimpleTermData()); + terms.back().addField(0).setHandle(mdl.allocTermField(0)); + MatchData::UP md = mdl.createMatchData(); + PhraseSplitter ps(qe, 0); + ASSERT_TRUE(ps.getNumTerms() == 1); + ps.update(*md); + // check that nothing is served from the splitter + EXPECT_EQUAL(ps.getTerm(0), &terms[0]); + TermFieldHandle handle = terms[0].lookupField(0)->getHandle(); + EXPECT_EQUAL(ps.resolveTermField(handle), md->resolveTermField(handle)); + } + { // single phrase + test::QueryEnvironment qe; + std::vector<SimpleTermData> & terms = qe.getTerms(); + MatchDataLayout mdl; + terms.push_back(SimpleTermData()); + terms.back().setUniqueId(1); + terms.back().setPhraseLength(3); + terms.back().addField(0).setHandle(mdl.allocTermField(0)); + terms.back().addField(7).setHandle(mdl.allocTermField(7)); + MatchData::UP md = mdl.createMatchData(); + PhraseSplitter ps(qe, 7); + ASSERT_TRUE(ps.getNumTerms() == 3); + ps.update(*md); + // check that all is served from the splitter + for (size_t i = 0; i < 3; ++i) { + // fprintf(stderr, "checking term %d\n", (int)i); + const ITermData *td = ps.getTerm(i); + EXPECT_NOT_EQUAL(td, &terms[0]); + EXPECT_NOT_EQUAL(td->lookupField(7), (ITermFieldData *)0); + EXPECT_EQUAL(td->lookupField(0), (ITermFieldData *)0); + TEST_DO(assertTermData(td, 1, 1, 7, i + 4)); // skipHandles = 4 + EXPECT_NOT_EQUAL(td->lookupField(7)->getHandle(), + terms[0].lookupField(7)->getHandle()); + EXPECT_NOT_EQUAL(ps.resolveTermField(td->lookupField(7)->getHandle()), + md->resolveTermField(terms[0].lookupField(7)->getHandle())); + } + } + { // combination + test::QueryEnvironment qe; + std::vector<SimpleTermData> &terms = qe.getTerms(); + MatchDataLayout mdl; + for (size_t i = 0; i < 3; ++i) { + terms.push_back(SimpleTermData()); + terms.back().setUniqueId(i); + terms.back().setPhraseLength(1); + terms.back().addField(4).setHandle(mdl.allocTermField(4)); + terms.back().addField(7).setHandle(mdl.allocTermField(7)); + // fprintf(stderr, "setup B term %p #f %zd\n", &terms.back(), terms.back().numFields()); + } + terms[1].setPhraseLength(3); + MatchData::UP md = mdl.createMatchData(); + PhraseSplitter ps(qe, 4); + ASSERT_TRUE(ps.getNumTerms() == 5); + ps.update(*md); + { // first term + // fprintf(stderr, "first term\n"); + EXPECT_EQUAL(ps.getTerm(0), &terms[0]); + TEST_DO(assertTermData(ps.getTerm(0), 0, 1, 4, 0)); + TEST_DO(assertTermData(ps.getTerm(0), 0, 1, 7, 1)); + + TermFieldHandle handle = terms[0].lookupField(4)->getHandle(); + EXPECT_EQUAL(ps.resolveTermField(handle), md->resolveTermField(handle)); + handle = terms[0].lookupField(7)->getHandle(); + EXPECT_EQUAL(ps.resolveTermField(handle), md->resolveTermField(handle)); + } + for (size_t i = 0; i < 3; ++i) { // phrase + // fprintf(stderr, "phrase term %zd\n", i); + const ITermData *td = ps.getTerm(i + 1); + EXPECT_NOT_EQUAL(td, &terms[1]); + TEST_DO(assertTermData(td, 1, 1, 4, i + 11)); // skipHandles == 11 + EXPECT_EQUAL(td->lookupField(7), (ITermFieldData *)0); + EXPECT_NOT_EQUAL(ps.resolveTermField(td->lookupField(4)->getHandle()), + md->resolveTermField(terms[1].lookupField(4)->getHandle())); + } + { // last term + // fprintf(stderr, "last term\n"); + EXPECT_EQUAL(ps.getTerm(4), &terms[2]); + TEST_DO(assertTermData(ps.getTerm(4), 2, 1, 4, 4)); + TEST_DO(assertTermData(ps.getTerm(4), 2, 1, 7, 5)); + + // fprintf(stderr, "inspect term %p #f %zd\n", &terms[2], terms[2].numFields()); + fflush(stderr); + TermFieldHandle handle = terms[2].lookupField(4)->getHandle(); + EXPECT_EQUAL(ps.resolveTermField(handle), md->resolveTermField(handle)); + } + } +} + +void +PhraseSplitterTest::testSplitterUpdate() +{ + { + test::QueryEnvironment qe; + std::vector<SimpleTermData> &terms = qe.getTerms(); + MatchDataLayout mdl; + for (size_t i = 0; i < 3; ++i) { + terms.push_back(SimpleTermData()); + terms.back().setUniqueId(i); + terms.back().setPhraseLength(1); + terms.back().addField(0).setHandle(mdl.allocTermField(0)); + } + terms[0].setPhraseLength(2); + terms[2].setPhraseLength(2); + MatchData::UP md = mdl.createMatchData(); + PhraseSplitter ps(qe, 0); + ASSERT_TRUE(ps.getNumTerms() == 5); + { // first phrase + TermFieldMatchData * tmd = md->resolveTermField(terms[0].lookupField(0)->getHandle()); + tmd->appendPosition(TermFieldMatchDataPosition(0, 10, 0, 1000)); + } + { // first term + TermFieldMatchData * tmd = md->resolveTermField(terms[1].lookupField(0)->getHandle()); + tmd->appendPosition(TermFieldMatchDataPosition(0, 20, 0, 1000)); + } + { // second phrase + TermFieldMatchData * tmd = md->resolveTermField(terms[2].lookupField(0)->getHandle()); + tmd->appendPosition(TermFieldMatchDataPosition(0, 30, 0, 1000)); + } + ps.update(*md); + for (size_t i = 0; i < 2; ++i) { // first phrase + const TermFieldMatchData * tmd = ps.resolveTermField(ps.getTerm(i)->lookupField(0)->getHandle()); + TermFieldMatchData::PositionsIterator itr = tmd->begin(); + EXPECT_EQUAL((itr++)->getPosition(), 10 + i); + ASSERT_TRUE(itr == tmd->end()); + } + { // first term + TermFieldMatchData * tmd = md->resolveTermField(ps.getTerm(2)->lookupField(0)->getHandle()); + TermFieldMatchData::PositionsIterator itr = tmd->begin(); + EXPECT_EQUAL((itr++)->getPosition(), 20u); + ASSERT_TRUE(itr == tmd->end()); + } + for (size_t i = 0; i < 2; ++i) { // second phrase + const TermFieldMatchData * tmd = ps.resolveTermField(ps.getTerm(i + 3)->lookupField(0)->getHandle()); + TermFieldMatchData::PositionsIterator itr = tmd->begin(); + EXPECT_EQUAL((itr++)->getPosition(), 30 + i); + ASSERT_TRUE(itr == tmd->end()); + } + } +} + +int +PhraseSplitterTest::Main() +{ + + TEST_INIT("phrasesplitter_test"); + + testCopyTermFieldMatchData(); + testSplitter(); + testSplitterUpdate(); + + TEST_DONE(); +} + +} +} + +TEST_APPHOOK(search::fef::PhraseSplitterTest); diff --git a/searchlib/src/tests/fef/properties/.gitignore b/searchlib/src/tests/fef/properties/.gitignore new file mode 100644 index 00000000000..00f94794fa3 --- /dev/null +++ b/searchlib/src/tests/fef/properties/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +properties_test +searchlib_properties_test_app diff --git a/searchlib/src/tests/fef/properties/CMakeLists.txt b/searchlib/src/tests/fef/properties/CMakeLists.txt new file mode 100644 index 00000000000..0b74b10cb31 --- /dev/null +++ b/searchlib/src/tests/fef/properties/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_properties_test_app + SOURCES + properties_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_properties_test_app COMMAND searchlib_properties_test_app) diff --git a/searchlib/src/tests/fef/properties/DESC b/searchlib/src/tests/fef/properties/DESC new file mode 100644 index 00000000000..02faa4cb727 --- /dev/null +++ b/searchlib/src/tests/fef/properties/DESC @@ -0,0 +1 @@ +properties test. Take a look at properties.cpp for details. diff --git a/searchlib/src/tests/fef/properties/FILES b/searchlib/src/tests/fef/properties/FILES new file mode 100644 index 00000000000..61054fa62c2 --- /dev/null +++ b/searchlib/src/tests/fef/properties/FILES @@ -0,0 +1 @@ +properties.cpp diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp new file mode 100644 index 00000000000..a08d511b418 --- /dev/null +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -0,0 +1,425 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/fef/indexproperties.h> +#include <vespa/searchlib/fef/properties.h> +#include <limits> + +using namespace search::fef; +using namespace search::fef::indexproperties; + +struct CopyVisitor : public IPropertiesVisitor +{ + Properties &dst; + CopyVisitor(Properties &p) : dst(p) {} + virtual void visitProperty(const Property::Value &key, + const Property &values) + { + for (uint32_t i = 0; i < values.size(); ++i) { + dst.add(key, values.getAt(i)); + } + } +}; + +Properties make_props(std::initializer_list<std::pair<const char *, std::initializer_list<const char *> > > entries) { + Properties props; + for (const auto &entry: entries) { + vespalib::string key = entry.first; + for (vespalib::string value: entry.second) { + props.add(key, value); + } + } + return props; +} + +TEST("require that namespace visitation works") { + Properties props = make_props({ {"foo", {"outside"}}, + {"foo.a", {"a_value"}}, + {"foo.b", {"b_value"}}, + {"foo.", {"outside"}} + }); + Properties result; + CopyVisitor copy_visitor(result); + props.visitNamespace("foo", copy_visitor); + EXPECT_EQUAL(2u, result.numKeys()); + EXPECT_EQUAL(result.lookup("a").get(), Property::Value("a_value")); + EXPECT_EQUAL(result.lookup("b").get(), Property::Value("b_value")); +} + +TEST("test stuff") { + { // empty lookup result + Property p; + + EXPECT_EQUAL(p.found(), false); + EXPECT_EQUAL(p.get(), Property::Value("")); + EXPECT_EQUAL(p.get("fb"), Property::Value("fb")); + EXPECT_EQUAL(p.size(), 0u); + EXPECT_EQUAL(p.getAt(0), Property::Value("")); + } + { // add / count / remove + Properties p = make_props({ {"a", {"a1", "a2", "a3"}}, + {"b", {"b1", "b2"}}, + {"c", {"c1"}} + }); + const Properties &pc = p; + + EXPECT_EQUAL(pc.numKeys(), 3u); + EXPECT_EQUAL(pc.numValues(), 6u); + EXPECT_EQUAL(pc.count("a"), 3u); + EXPECT_EQUAL(pc.count("b"), 2u); + EXPECT_EQUAL(pc.count("c"), 1u); + EXPECT_EQUAL(pc.count("d"), 0u); + + p.remove("d"); + + EXPECT_EQUAL(pc.numKeys(), 3u); + EXPECT_EQUAL(pc.numValues(), 6u); + EXPECT_EQUAL(pc.count("a"), 3u); + EXPECT_EQUAL(pc.count("b"), 2u); + EXPECT_EQUAL(pc.count("c"), 1u); + EXPECT_EQUAL(pc.count("d"), 0u); + + p.remove("c"); + + EXPECT_EQUAL(pc.numKeys(), 2u); + EXPECT_EQUAL(pc.numValues(), 5u); + EXPECT_EQUAL(pc.count("a"), 3u); + EXPECT_EQUAL(pc.count("b"), 2u); + EXPECT_EQUAL(pc.count("c"), 0u); + EXPECT_EQUAL(pc.count("d"), 0u); + + p.remove("b"); + + EXPECT_EQUAL(pc.numKeys(), 1u); + EXPECT_EQUAL(pc.numValues(), 3u); + EXPECT_EQUAL(pc.count("a"), 3u); + EXPECT_EQUAL(pc.count("b"), 0u); + EXPECT_EQUAL(pc.count("c"), 0u); + EXPECT_EQUAL(pc.count("d"), 0u); + + p.remove("a"); + + EXPECT_EQUAL(pc.numKeys(), 0u); + EXPECT_EQUAL(pc.numValues(), 0u); + EXPECT_EQUAL(pc.count("a"), 0u); + EXPECT_EQUAL(pc.count("b"), 0u); + EXPECT_EQUAL(pc.count("c"), 0u); + EXPECT_EQUAL(pc.count("d"), 0u); + } + { // lookup / import / visit / compare / hash + Properties p; + + p.add("x", "x1"); + p.add("a.x", "x2"); + p.add("a.b.x", "x3"); + p.add("a.b.c.x", "x4"); + + p.add("list", "e1").add("list", "e2").add("list", "e3"); + + EXPECT_EQUAL(p.numKeys(), 5u); + EXPECT_EQUAL(p.numValues(), 7u); + + EXPECT_EQUAL(p.lookup("x").found(), true); + EXPECT_EQUAL(p.lookup("a.x").found(), true); + EXPECT_EQUAL(p.lookup("a.b.x").found(), true); + EXPECT_EQUAL(p.lookup("a.b.c.x").found(), true); + EXPECT_EQUAL(p.lookup("list").found(), true); + EXPECT_EQUAL(p.lookup("y").found(), false); + + EXPECT_EQUAL(p.lookup("x").get(), Property::Value("x1")); + EXPECT_EQUAL(p.lookup("a.x").get(), Property::Value("x2")); + EXPECT_EQUAL(p.lookup("a.b.x").get(), Property::Value("x3")); + EXPECT_EQUAL(p.lookup("a.b.c.x").get(), Property::Value("x4")); + EXPECT_EQUAL(p.lookup("list").get(), Property::Value("e1")); + EXPECT_EQUAL(p.lookup("y").get(), Property::Value("")); + + EXPECT_EQUAL(p.lookup("x").get(), Property::Value("x1")); + EXPECT_EQUAL(p.lookup("a", "x").get(), Property::Value("x2")); + EXPECT_EQUAL(p.lookup("a", "b", "x").get(), Property::Value("x3")); + EXPECT_EQUAL(p.lookup("a", "b", "c", "x").get(), Property::Value("x4")); + + EXPECT_EQUAL(p.lookup("x").get("fallback"), Property::Value("x1")); + EXPECT_EQUAL(p.lookup("y").get("fallback"), Property::Value("fallback")); + + EXPECT_EQUAL(p.lookup("y").size(), 0u); + EXPECT_EQUAL(p.lookup("x").size(), 1u); + EXPECT_EQUAL(p.lookup("list").size(), 3u); + EXPECT_EQUAL(p.lookup("list").getAt(0), Property::Value("e1")); + EXPECT_EQUAL(p.lookup("list").getAt(1), Property::Value("e2")); + EXPECT_EQUAL(p.lookup("list").getAt(2), Property::Value("e3")); + EXPECT_EQUAL(p.lookup("list").getAt(3), Property::Value("")); + + Properties p2; + + p2.add("x", "new_x"); + p2.add("y", "y1"); + p2.add("list", "foo").add("list", "bar"); + + EXPECT_EQUAL(p2.numKeys(), 3u); + EXPECT_EQUAL(p2.numValues(), 4u); + + p.import(p2); + + EXPECT_EQUAL(p.numKeys(), 6u); + EXPECT_EQUAL(p.numValues(), 7u); + + EXPECT_EQUAL(p.lookup("y").size(), 1u); + EXPECT_EQUAL(p.lookup("y").get(), Property::Value("y1")); + + EXPECT_EQUAL(p.lookup("x").size(), 1u); + EXPECT_EQUAL(p.lookup("x").get(), Property::Value("new_x")); + + EXPECT_EQUAL(p.lookup("z").size(), 0u); + + EXPECT_EQUAL(p.lookup("a", "x").size(), 1u); + EXPECT_EQUAL(p.lookup("a", "x").get(), Property::Value("x2")); + + EXPECT_EQUAL(p.lookup("list").size(), 2u); + EXPECT_EQUAL(p.lookup("list").getAt(0), Property::Value("foo")); + EXPECT_EQUAL(p.lookup("list").getAt(1), Property::Value("bar")); + EXPECT_EQUAL(p.lookup("list").getAt(2), Property::Value("")); + + Properties p3; + + EXPECT_TRUE(!(p == p2)); + EXPECT_TRUE(!(p == p3)); + EXPECT_TRUE(!(p2 == p)); + EXPECT_TRUE(!(p3 == p)); + EXPECT_TRUE(!(p2 == p3)); + EXPECT_TRUE(!(p3 == p2)); + + CopyVisitor cv(p3); + p.visitProperties(cv); + + EXPECT_EQUAL(p3.numKeys(), 6u); + EXPECT_EQUAL(p3.numValues(), 7u); + + EXPECT_TRUE(p == p3); + EXPECT_TRUE(p3 == p); + EXPECT_EQUAL(p.hashCode(), p3.hashCode()); + + p.clear(); + EXPECT_EQUAL(p.numKeys(), 0u); + EXPECT_EQUAL(p.numValues(), 0u); + EXPECT_TRUE(!(p == p3)); + EXPECT_TRUE(!(p3 == p)); + + Properties p4; + CopyVisitor cv2(p4); + p.visitProperties(cv); + EXPECT_EQUAL(p4.numKeys(), 0u); + EXPECT_EQUAL(p4.numValues(), 0u); + EXPECT_TRUE(p == p4); + EXPECT_TRUE(p4 == p); + EXPECT_EQUAL(p.hashCode(), p4.hashCode()); + } + + { // test index properties known by the framework + { // vespa.rank.firstphase + EXPECT_EQUAL(rank::FirstPhase::NAME, vespalib::string("vespa.rank.firstphase")); + EXPECT_EQUAL(rank::FirstPhase::DEFAULT_VALUE, vespalib::string("nativeRank")); + Properties p; + EXPECT_EQUAL(rank::FirstPhase::lookup(p), vespalib::string("nativeRank")); + p.add("vespa.rank.firstphase", "specialrank"); + EXPECT_EQUAL(rank::FirstPhase::lookup(p), vespalib::string("specialrank")); + } + { // vespa.rank.secondphase + EXPECT_EQUAL(rank::SecondPhase::NAME, vespalib::string("vespa.rank.secondphase")); + EXPECT_EQUAL(rank::SecondPhase::DEFAULT_VALUE, vespalib::string("")); + Properties p; + EXPECT_EQUAL(rank::SecondPhase::lookup(p), vespalib::string("")); + p.add("vespa.rank.secondphase", "specialrank"); + EXPECT_EQUAL(rank::SecondPhase::lookup(p), vespalib::string("specialrank")); + } + { // vespa.dump.feature + EXPECT_EQUAL(dump::Feature::NAME, vespalib::string("vespa.dump.feature")); + EXPECT_EQUAL(dump::Feature::DEFAULT_VALUE.size(), 0u); + Properties p; + EXPECT_EQUAL(dump::Feature::lookup(p).size(), 0u); + p.add("vespa.dump.feature", "foo"); + p.add("vespa.dump.feature", "bar"); + std::vector<vespalib::string> a = dump::Feature::lookup(p); + ASSERT_TRUE(a.size() == 2); + EXPECT_EQUAL(a[0], vespalib::string("foo")); + EXPECT_EQUAL(a[1], vespalib::string("bar")); + } + { // vespa.dump.ignoredefaultfeatures + EXPECT_EQUAL(dump::IgnoreDefaultFeatures::NAME, vespalib::string("vespa.dump.ignoredefaultfeatures")); + EXPECT_EQUAL(dump::IgnoreDefaultFeatures::DEFAULT_VALUE, "false"); + Properties p; + EXPECT_TRUE(!dump::IgnoreDefaultFeatures::check(p)); + p.add("vespa.dump.ignoredefaultfeatures", "true"); + EXPECT_TRUE(dump::IgnoreDefaultFeatures::check(p)); + } + { // vespa.matching.termwise_limit + EXPECT_EQUAL(matching::TermwiseLimit::NAME, vespalib::string("vespa.matching.termwise_limit")); + EXPECT_EQUAL(matching::TermwiseLimit::DEFAULT_VALUE, 1.0); + Properties p; + EXPECT_EQUAL(matching::TermwiseLimit::lookup(p), 1.0); + p.add("vespa.matching.termwise_limit", "0.05"); + EXPECT_EQUAL(matching::TermwiseLimit::lookup(p), 0.05); + } + { // vespa.matching.numthreads + EXPECT_EQUAL(matching::NumThreadsPerSearch::NAME, vespalib::string("vespa.matching.numthreadspersearch")); + EXPECT_EQUAL(matching::NumThreadsPerSearch::DEFAULT_VALUE, std::numeric_limits<uint32_t>::max()); + Properties p; + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), std::numeric_limits<uint32_t>::max()); + p.add("vespa.matching.numthreadspersearch", "50"); + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + } + { + EXPECT_EQUAL(matching::NumSearchPartitions::NAME, vespalib::string("vespa.matching.numsearchpartitions")); + EXPECT_EQUAL(matching::NumSearchPartitions::DEFAULT_VALUE, 1u); + Properties p; + EXPECT_EQUAL(matching::NumSearchPartitions::lookup(p), 1u); + p.add("vespa.matching.numsearchpartitions", "50"); + EXPECT_EQUAL(matching::NumSearchPartitions::lookup(p), 50u); + } + { // vespa.matchphase.degradation.attribute + EXPECT_EQUAL(matchphase::DegradationAttribute::NAME, vespalib::string("vespa.matchphase.degradation.attribute")); + EXPECT_EQUAL(matchphase::DegradationAttribute::DEFAULT_VALUE, ""); + Properties p; + EXPECT_EQUAL(matchphase::DegradationAttribute::lookup(p), ""); + p.add("vespa.matchphase.degradation.attribute", "foobar"); + EXPECT_EQUAL(matchphase::DegradationAttribute::lookup(p), "foobar"); + } + { // vespa.matchphase.degradation.ascending + EXPECT_EQUAL(matchphase::DegradationAscendingOrder::NAME, vespalib::string("vespa.matchphase.degradation.ascendingorder")); + EXPECT_EQUAL(matchphase::DegradationAscendingOrder::DEFAULT_VALUE, false); + Properties p; + EXPECT_EQUAL(matchphase::DegradationAscendingOrder::lookup(p), false); + p.add("vespa.matchphase.degradation.ascendingorder", "true"); + EXPECT_EQUAL(matchphase::DegradationAscendingOrder::lookup(p), true); + } + { // vespa.matchphase.degradation.maxhits + EXPECT_EQUAL(matchphase::DegradationMaxHits::NAME, vespalib::string("vespa.matchphase.degradation.maxhits")); + EXPECT_EQUAL(matchphase::DegradationMaxHits::DEFAULT_VALUE, 0u); + Properties p; + EXPECT_EQUAL(matchphase::DegradationMaxHits::lookup(p), 0u); + p.add("vespa.matchphase.degradation.maxhits", "123789"); + EXPECT_EQUAL(matchphase::DegradationMaxHits::lookup(p), 123789u); + } + { // vespa.matchphase.degradation.samplepercentage + EXPECT_EQUAL(matchphase::DegradationSamplePercentage::NAME, vespalib::string("vespa.matchphase.degradation.samplepercentage")); + EXPECT_EQUAL(matchphase::DegradationSamplePercentage::DEFAULT_VALUE, 0.2); + Properties p; + EXPECT_EQUAL(matchphase::DegradationSamplePercentage::lookup(p), 0.2); + p.add("vespa.matchphase.degradation.samplepercentage", "0.9"); + EXPECT_EQUAL(matchphase::DegradationSamplePercentage::lookup(p), 0.9); + } + { // vespa.matchphase.degradation.maxfiltercoverage + EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::NAME, vespalib::string("vespa.matchphase.degradation.maxfiltercoverage")); + EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::DEFAULT_VALUE, 1.0); + Properties p; + EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::lookup(p), 1.0); + p.add("vespa.matchphase.degradation.maxfiltercoverage", "0.076"); + EXPECT_EQUAL(matchphase::DegradationMaxFilterCoverage::lookup(p), 0.076); + } + { // vespa.matchphase.degradation.postfiltermultiplier + EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::NAME, vespalib::string("vespa.matchphase.degradation.postfiltermultiplier")); + EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::DEFAULT_VALUE, 1.0); + Properties p; + EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::lookup(p), 1.0); + p.add("vespa.matchphase.degradation.postfiltermultiplier", "0.9"); + EXPECT_EQUAL(matchphase::DegradationPostFilterMultiplier::lookup(p), 0.9); + } + { // vespa.matchphase.diversity.attribute + EXPECT_EQUAL(matchphase::DiversityAttribute::NAME, vespalib::string("vespa.matchphase.diversity.attribute")); + EXPECT_EQUAL(matchphase::DiversityAttribute::DEFAULT_VALUE, ""); + Properties p; + EXPECT_EQUAL(matchphase::DiversityAttribute::lookup(p), ""); + p.add("vespa.matchphase.diversity.attribute", "foobar"); + EXPECT_EQUAL(matchphase::DiversityAttribute::lookup(p), "foobar"); + } + { // vespa.matchphase.diversity.mingroups + EXPECT_EQUAL(matchphase::DiversityMinGroups::NAME, vespalib::string("vespa.matchphase.diversity.mingroups")); + EXPECT_EQUAL(matchphase::DiversityMinGroups::DEFAULT_VALUE, 1u); + Properties p; + EXPECT_EQUAL(matchphase::DiversityMinGroups::lookup(p), 1u); + p.add("vespa.matchphase.diversity.mingroups", "5"); + EXPECT_EQUAL(matchphase::DiversityMinGroups::lookup(p), 5u); + } + { // vespa.hitcollector.heapsize + EXPECT_EQUAL(hitcollector::HeapSize::NAME, vespalib::string("vespa.hitcollector.heapsize")); + EXPECT_EQUAL(hitcollector::HeapSize::DEFAULT_VALUE, 100u); + Properties p; + EXPECT_EQUAL(hitcollector::HeapSize::lookup(p), 100u); + p.add("vespa.hitcollector.heapsize", "50"); + EXPECT_EQUAL(hitcollector::HeapSize::lookup(p), 50u); + } + { // vespa.hitcollector.arraysize + EXPECT_EQUAL(hitcollector::ArraySize::NAME, vespalib::string("vespa.hitcollector.arraysize")); + EXPECT_EQUAL(hitcollector::ArraySize::DEFAULT_VALUE, 10000u); + Properties p; + EXPECT_EQUAL(hitcollector::ArraySize::lookup(p), 10000u); + p.add("vespa.hitcollector.arraysize", "50"); + EXPECT_EQUAL(hitcollector::ArraySize::lookup(p), 50u); + } + { // vespa.hitcollector.estimatepoint + EXPECT_EQUAL(hitcollector::EstimatePoint::NAME, vespalib::string("vespa.hitcollector.estimatepoint")); + EXPECT_EQUAL(hitcollector::EstimatePoint::DEFAULT_VALUE, 0xffffffffu); + Properties p; + EXPECT_EQUAL(hitcollector::EstimatePoint::lookup(p), 0xffffffffu); + p.add("vespa.hitcollector.estimatepoint", "50"); + EXPECT_EQUAL(hitcollector::EstimatePoint::lookup(p), 50u); + } + { // vespa.hitcollector.estimatelimit + EXPECT_EQUAL(hitcollector::EstimateLimit::NAME, vespalib::string("vespa.hitcollector.estimatelimit")); + EXPECT_EQUAL(hitcollector::EstimateLimit::DEFAULT_VALUE, 0xffffffffu); + Properties p; + EXPECT_EQUAL(hitcollector::EstimateLimit::lookup(p), 0xffffffffu); + p.add("vespa.hitcollector.estimatelimit", "50"); + EXPECT_EQUAL(hitcollector::EstimateLimit::lookup(p), 50u); + } + { // vespa.hitcollector.rankscoredroplimit + EXPECT_EQUAL(hitcollector::RankScoreDropLimit::NAME, vespalib::string("vespa.hitcollector.rankscoredroplimit")); + search::feature_t got1 = hitcollector::RankScoreDropLimit::DEFAULT_VALUE; + EXPECT_TRUE(got1 != got1); + Properties p; + search::feature_t got2= hitcollector::RankScoreDropLimit::lookup(p); + EXPECT_TRUE(got2 != got2); + p.add("vespa.hitcollector.rankscoredroplimit", "-123456789.12345"); + EXPECT_EQUAL(hitcollector::RankScoreDropLimit::lookup(p), -123456789.12345); + p.clear().add("vespa.hitcollector.rankscoredroplimit", "123456789.12345"); + EXPECT_EQUAL(hitcollector::RankScoreDropLimit::lookup(p), 123456789.12345); + } + { // vespa.fieldweight. + EXPECT_EQUAL(FieldWeight::BASE_NAME, vespalib::string("vespa.fieldweight.")); + EXPECT_EQUAL(FieldWeight::DEFAULT_VALUE, 100u); + Properties p; + EXPECT_EQUAL(FieldWeight::lookup(p, "foo"), 100u); + p.add("vespa.fieldweight.foo", "200"); + EXPECT_EQUAL(FieldWeight::lookup(p, "foo"), 200u); + } + { // vespa.isfilterfield. + EXPECT_EQUAL(IsFilterField::BASE_NAME, "vespa.isfilterfield."); + EXPECT_EQUAL(IsFilterField::DEFAULT_VALUE, "false"); + Properties p; + EXPECT_TRUE(!IsFilterField::check(p, "foo")); + p.add("vespa.isfilterfield.foo", "true"); + EXPECT_TRUE(IsFilterField::check(p, "foo")); + EXPECT_TRUE(!IsFilterField::check(p, "bar")); + IsFilterField::set(p, "bar"); + EXPECT_TRUE(IsFilterField::check(p, "bar")); + } + } +} + +TEST("test attribute type properties") +{ + Properties p; + p.add("vespa.type.attribute.foo", "tensor(x[10])"); + EXPECT_EQUAL("tensor(x[10])", type::Attribute::lookup(p, "foo")); + EXPECT_EQUAL("", type::Attribute::lookup(p, "bar")); +} + +TEST("test query feature type properties") +{ + Properties p; + p.add("vespa.type.query.foo", "tensor(x[10])"); + EXPECT_EQUAL("tensor(x[10])", type::QueryFeature::lookup(p, "foo")); + EXPECT_EQUAL("", type::QueryFeature::lookup(p, "bar")); +} + + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/rank_program/.gitignore b/searchlib/src/tests/fef/rank_program/.gitignore new file mode 100644 index 00000000000..b86a29e139f --- /dev/null +++ b/searchlib/src/tests/fef/rank_program/.gitignore @@ -0,0 +1 @@ +searchlib_rank_program_test_app diff --git a/searchlib/src/tests/fef/rank_program/CMakeLists.txt b/searchlib/src/tests/fef/rank_program/CMakeLists.txt new file mode 100644 index 00000000000..12d971a9421 --- /dev/null +++ b/searchlib/src/tests/fef/rank_program/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_rank_program_test_app + SOURCES + rank_program_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_rank_program_test_app COMMAND searchlib_rank_program_test_app) diff --git a/searchlib/src/tests/fef/rank_program/FILES b/searchlib/src/tests/fef/rank_program/FILES new file mode 100644 index 00000000000..bf6e4665a68 --- /dev/null +++ b/searchlib/src/tests/fef/rank_program/FILES @@ -0,0 +1 @@ +rank_program_test.cpp diff --git a/searchlib/src/tests/fef/rank_program/rank_program_test.cpp b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp new file mode 100644 index 00000000000..baf665c58e8 --- /dev/null +++ b/searchlib/src/tests/fef/rank_program/rank_program_test.cpp @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/searchlib/features/valuefeature.h> +#include <vespa/searchlib/fef/blueprintfactory.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> +#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/searchlib/fef/test/plugin/sum.h> +#include <vespa/searchlib/fef/rank_program.h> + +using namespace search::fef; +using namespace search::fef::test; +using namespace search::features; + +struct ImpureValueExecutor : FeatureExecutor { + double value; + ImpureValueExecutor(double value_in) : value(value_in) {} + bool isPure() override { return false; } + void execute(search::fef::MatchData &md) override { *md.resolveFeature(outputs()[0]) = value; } +}; + +struct ImpureValueBlueprint : Blueprint { + double value; + ImpureValueBlueprint() : Blueprint("ivalue"), value(31212.0) {} + void visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const override {} + Blueprint::UP createInstance() const override { return Blueprint::UP(new ImpureValueBlueprint()); } + bool setup(const IIndexEnvironment &, const std::vector<vespalib::string> ¶ms) override { + ASSERT_EQUAL(1u, params.size()); + value = strtod(params[0].c_str(), nullptr); + describeOutput("out", "the impure value"); + return true; + } + FeatureExecutor::LP createExecutor(const IQueryEnvironment &) const override { + return FeatureExecutor::LP(new ImpureValueExecutor(value)); + } +}; + +struct MySetup { + BlueprintFactory factory; + IndexEnvironment indexEnv; + BlueprintResolver::SP resolver; + Properties overrides; + RankProgram program; + MySetup() : factory(), indexEnv(), resolver(new BlueprintResolver(factory, indexEnv)), + overrides(), program(resolver) + { + factory.addPrototype(Blueprint::SP(new ValueBlueprint())); + factory.addPrototype(Blueprint::SP(new ImpureValueBlueprint())); + factory.addPrototype(Blueprint::SP(new SumBlueprint())); + } + MySetup &add(const vespalib::string &feature) { + resolver->addSeed(feature); + return *this; + } + MySetup &override(const vespalib::string &feature, double value) { + overrides.add(feature, vespalib::make_string("%g", value)); + return *this; + } + MySetup &compile() { + ASSERT_TRUE(resolver->compile()); + MatchDataLayout mdl; + QueryEnvironment queryEnv(&indexEnv); + program.setup(mdl, queryEnv, overrides); + return *this; + } + MySetup &run() { + program.run(1); + return *this; + } + double get() { + std::vector<vespalib::string> names; + std::vector<FeatureHandle> handles; + program.get_seed_handles(names, handles); + EXPECT_EQUAL(1u, names.size()); + EXPECT_EQUAL(names.size(), handles.size()); + return *program.match_data().resolveFeature(handles[0]); + } + double get(const vespalib::string &feature) { + std::vector<vespalib::string> names; + std::vector<FeatureHandle> handles; + program.get_seed_handles(names, handles); + EXPECT_EQUAL(names.size(), handles.size()); + for (size_t i = 0; i < names.size(); ++i) { + if (names[i] == feature) { + return *program.match_data().resolveFeature(handles[i]); + } + } + return 31212.0; + } + std::map<vespalib::string, double> all() { + std::map<vespalib::string, double> result; + std::vector<vespalib::string> names; + std::vector<FeatureHandle> handles; + program.get_seed_handles(names, handles); + EXPECT_EQUAL(names.size(), handles.size()); + for (size_t i = 0; i < names.size(); ++i) { + result[names[i]] = *program.match_data().resolveFeature(handles[i]); + } + return result; + } +}; + +TEST_F("require that match data docid is set by run", MySetup()) { + f1.compile(); + EXPECT_NOT_EQUAL(1u, f1.program.match_data().getDocId()); + f1.run(); + EXPECT_EQUAL(1u, f1.program.match_data().getDocId()); +} + +TEST_F("require that simple program works", MySetup()) { + EXPECT_EQUAL(15.0, f1.add("mysum(value(10),ivalue(5))").compile().run().get()); + EXPECT_EQUAL(3u, f1.program.num_executors()); + EXPECT_EQUAL(2u, f1.program.program_size()); +} + +TEST_F("require that const features are calculated during setup", MySetup()) { + f1.add("mysum(value(10),value(5))").compile(); + EXPECT_EQUAL(15.0, f1.get()); + EXPECT_EQUAL(3u, f1.program.num_executors()); + EXPECT_EQUAL(0u, f1.program.program_size()); +} + +TEST_F("require that non-const features are calculated during run", MySetup()) { + f1.add("mysum(ivalue(10),ivalue(5))").compile(); + EXPECT_EQUAL(0.0, f1.get()); + f1.run(); + EXPECT_EQUAL(15.0, f1.get()); + EXPECT_EQUAL(3u, f1.program.num_executors()); + EXPECT_EQUAL(3u, f1.program.program_size()); +} + +TEST_F("require that a single program can calculate multiple output features", MySetup()) { + f1.add("value(1)").add("ivalue(2)").add("ivalue(3)"); + f1.add("mysum(value(1),value(2),ivalue(3))"); + f1.compile().run(); + EXPECT_EQUAL(5u, f1.program.num_executors()); + EXPECT_EQUAL(3u, f1.program.program_size()); + EXPECT_EQUAL(5u, f1.program.match_data().getNumFeatures()); + auto result = f1.all(); + EXPECT_EQUAL(4u, result.size()); + EXPECT_EQUAL(1.0, result["value(1)"]); + EXPECT_EQUAL(2.0, result["ivalue(2)"]); + EXPECT_EQUAL(3.0, result["ivalue(3)"]); + EXPECT_EQUAL(6.0, result["mysum(value(1),value(2),ivalue(3))"]); +} + +TEST_F("require that a single executor can produce multiple features", MySetup()) { + f1.add("mysum(value(1,2,3).0,value(1,2,3).1,value(1,2,3).2)"); + EXPECT_EQUAL(6.0, f1.compile().run().get()); + EXPECT_EQUAL(2u, f1.program.num_executors()); + EXPECT_EQUAL(0u, f1.program.program_size()); + EXPECT_EQUAL(4u, f1.program.match_data().getNumFeatures()); +} + +TEST_F("require that feature values can be overridden", MySetup()) { + f1.add("value(1)").add("ivalue(2)").add("ivalue(3)"); + f1.add("mysum(value(1),value(2),ivalue(3))"); + f1.override("value(2)", 20.0).override("ivalue(3)", 30.0); + f1.compile().run(); + EXPECT_EQUAL(5u, f1.program.num_executors()); + EXPECT_EQUAL(3u, f1.program.program_size()); + EXPECT_EQUAL(5u, f1.program.match_data().getNumFeatures()); + auto result = f1.all(); + EXPECT_EQUAL(4u, result.size()); + EXPECT_EQUAL(1.0, result["value(1)"]); + EXPECT_EQUAL(2.0, result["ivalue(2)"]); + EXPECT_EQUAL(30.0, result["ivalue(3)"]); + EXPECT_EQUAL(51.0, result["mysum(value(1),value(2),ivalue(3))"]); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/resolver/.gitignore b/searchlib/src/tests/fef/resolver/.gitignore new file mode 100644 index 00000000000..57114e69298 --- /dev/null +++ b/searchlib/src/tests/fef/resolver/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +searchlib_resolver_test_app diff --git a/searchlib/src/tests/fef/resolver/CMakeLists.txt b/searchlib/src/tests/fef/resolver/CMakeLists.txt new file mode 100644 index 00000000000..835a50fd6fb --- /dev/null +++ b/searchlib/src/tests/fef/resolver/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_resolver_test_app + SOURCES + resolver_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_resolver_test_app COMMAND searchlib_resolver_test_app) diff --git a/searchlib/src/tests/fef/resolver/DESC b/searchlib/src/tests/fef/resolver/DESC new file mode 100644 index 00000000000..7d3262ab110 --- /dev/null +++ b/searchlib/src/tests/fef/resolver/DESC @@ -0,0 +1 @@ +resolver test. Take a look at resolver_test.cpp for details. diff --git a/searchlib/src/tests/fef/resolver/FILES b/searchlib/src/tests/fef/resolver/FILES new file mode 100644 index 00000000000..c40c0663848 --- /dev/null +++ b/searchlib/src/tests/fef/resolver/FILES @@ -0,0 +1 @@ +resolver_test.cpp diff --git a/searchlib/src/tests/fef/resolver/resolver_test.cpp b/searchlib/src/tests/fef/resolver/resolver_test.cpp new file mode 100644 index 00000000000..3d791f886e1 --- /dev/null +++ b/searchlib/src/tests/fef/resolver/resolver_test.cpp @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("resolver_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/fef/fef.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> + +namespace search { +namespace fef { + +class BaseBlueprint : public Blueprint { +public: + BaseBlueprint() : Blueprint("base") { } + virtual void visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const {} + virtual Blueprint::UP createInstance() const { return Blueprint::UP(new BaseBlueprint()); } + virtual bool setup(const IIndexEnvironment & indexEnv, + const ParameterList & params) { + (void) indexEnv; (void) params; + describeOutput("foo", "foo"); + describeOutput("bar", "bar"); + describeOutput("baz", "baz"); + return true; + } + virtual FeatureExecutor::LP createExecutor(const IQueryEnvironment &) const { + return FeatureExecutor::LP(NULL); + } +}; + +class CombineBlueprint : public Blueprint { +public: + CombineBlueprint() : Blueprint("combine") { } + virtual void visitDumpFeatures(const IIndexEnvironment &, + IDumpFeatureVisitor &) const {} + virtual Blueprint::UP createInstance() const { return Blueprint::UP(new CombineBlueprint()); } + virtual bool setup(const IIndexEnvironment & indexEnv, + const ParameterList & params) { + (void) indexEnv; (void) params; + defineInput("base.foo"); + defineInput("base.bar"); + defineInput("base.baz"); + describeOutput("out", "out"); + return true; + } + virtual FeatureExecutor::LP createExecutor(const IQueryEnvironment &) const { + return FeatureExecutor::LP(NULL); + } +}; + +class Test : public vespalib::TestApp { +private: + BlueprintFactory _factory; + void requireThatWeGetUniqueBlueprints(); +public: + Test(); + int Main(); +}; + +Test::Test() : + _factory() +{ + _factory.addPrototype(Blueprint::SP(new BaseBlueprint())); + _factory.addPrototype(Blueprint::SP(new CombineBlueprint())); +} + +void +Test::requireThatWeGetUniqueBlueprints() +{ + test::IndexEnvironment ienv; + BlueprintResolver::SP res(new BlueprintResolver(_factory, ienv)); + res->addSeed("combine"); + EXPECT_TRUE(res->compile()); + const BlueprintResolver::ExecutorSpecList & spec = res->getExecutorSpecs(); + EXPECT_EQUAL(2u, spec.size()); + EXPECT_TRUE(dynamic_cast<BaseBlueprint *>(spec[0].blueprint.get()) != NULL); + EXPECT_TRUE(dynamic_cast<CombineBlueprint *>(spec[1].blueprint.get()) != NULL); +} + +int +Test::Main() +{ + TEST_INIT("resolver_test"); + + requireThatWeGetUniqueBlueprints(); + + TEST_DONE(); +} + +} +} + +TEST_APPHOOK(search::fef::Test); diff --git a/searchlib/src/tests/fef/table/.gitignore b/searchlib/src/tests/fef/table/.gitignore new file mode 100644 index 00000000000..b89a30490e0 --- /dev/null +++ b/searchlib/src/tests/fef/table/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +table_test +searchlib_table_test_app diff --git a/searchlib/src/tests/fef/table/CMakeLists.txt b/searchlib/src/tests/fef/table/CMakeLists.txt new file mode 100644 index 00000000000..ca61eb7c365 --- /dev/null +++ b/searchlib/src/tests/fef/table/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_table_test_app + SOURCES + table_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_table_test_app COMMAND searchlib_table_test_app) diff --git a/searchlib/src/tests/fef/table/DESC b/searchlib/src/tests/fef/table/DESC new file mode 100644 index 00000000000..65834ed1305 --- /dev/null +++ b/searchlib/src/tests/fef/table/DESC @@ -0,0 +1 @@ +table test. Take a look at table.cpp for details. diff --git a/searchlib/src/tests/fef/table/FILES b/searchlib/src/tests/fef/table/FILES new file mode 100644 index 00000000000..40be726aeb8 --- /dev/null +++ b/searchlib/src/tests/fef/table/FILES @@ -0,0 +1 @@ +table.cpp diff --git a/searchlib/src/tests/fef/table/table_test.cpp b/searchlib/src/tests/fef/table/table_test.cpp new file mode 100644 index 00000000000..2d05e0c7310 --- /dev/null +++ b/searchlib/src/tests/fef/table/table_test.cpp @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("tablemanager_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <fstream> +#include <limits> +#include <iostream> +#include <vespa/searchlib/fef/filetablefactory.h> +#include <vespa/searchlib/fef/functiontablefactory.h> +#include <vespa/searchlib/fef/table.h> +#include <vespa/searchlib/fef/tablemanager.h> + +namespace search { +namespace fef { + +class TableTest : public vespalib::TestApp +{ +private: + bool assertTable(const Table & act, const Table & exp); + bool assertCreateTable(const ITableFactory & tf, const vespalib::string & name, const Table & exp); + void testTable(); + void testFileTableFactory(); + void testFunctionTableFactory(); + void testTableManager(); + +public: + int Main(); +}; + +bool +TableTest::assertTable(const Table & act, const Table & exp) +{ + if (!EXPECT_EQUAL(act.size(), exp.size())) return false; + for (size_t i = 0; i < act.size(); ++i) { + if (!EXPECT_APPROX(act[i], exp[i], 0.01)) return false; + } + return true; +} + +bool +TableTest::assertCreateTable(const ITableFactory & tf, const vespalib::string & name, const Table & exp) +{ + Table::SP t = tf.createTable(name); + if (!EXPECT_TRUE(t.get() != NULL)) return false; + return assertTable(*t, exp); +} + +void +TableTest::testTable() +{ + Table t; + EXPECT_EQUAL(t.size(), 0u); + EXPECT_EQUAL(t.max(), -std::numeric_limits<double>::max()); + t.add(1).add(2); + EXPECT_EQUAL(t.size(), 2u); + EXPECT_EQUAL(t.max(), 2); + EXPECT_EQUAL(t[0], 1); + EXPECT_EQUAL(t[1], 2); + t.add(10); + EXPECT_EQUAL(t.size(), 3u); + EXPECT_EQUAL(t.max(), 10); + EXPECT_EQUAL(t[2], 10); + t.add(5); + EXPECT_EQUAL(t.size(), 4u); + EXPECT_EQUAL(t.max(), 10); + EXPECT_EQUAL(t[3], 5); +} + +void +TableTest::testFileTableFactory() +{ + { + FileTableFactory ftf("tables1"); + EXPECT_TRUE(assertCreateTable(ftf, "a", Table().add(1.5).add(2.25).add(3))); + EXPECT_TRUE(ftf.createTable("b").get() == NULL); + } + { + FileTableFactory ftf("tables1/"); + EXPECT_TRUE(ftf.createTable("a").get() != NULL); + } +} + +void +TableTest::testFunctionTableFactory() +{ + FunctionTableFactory ftf(2); + EXPECT_TRUE(assertCreateTable(ftf, "expdecay(400,12)", + Table().add(400).add(368.02))); + EXPECT_TRUE(assertCreateTable(ftf, "loggrowth(1000,5000,1)", + Table().add(5000).add(5693.15))); + EXPECT_TRUE(assertCreateTable(ftf, "linear(10,100)", + Table().add(100).add(110))); + // specify table size + EXPECT_TRUE(assertCreateTable(ftf, "expdecay(400,12,3)", + Table().add(400).add(368.02).add(338.60))); + EXPECT_TRUE(assertCreateTable(ftf, "loggrowth(1000,5000,1,3)", + Table().add(5000).add(5693.15).add(6098.61))); + EXPECT_TRUE(assertCreateTable(ftf, "linear(10,100,3)", + Table().add(100).add(110).add(120))); + EXPECT_TRUE(ftf.createTable("expdecay()").get() == NULL); + EXPECT_TRUE(ftf.createTable("expdecay(10)").get() == NULL); + EXPECT_TRUE(ftf.createTable("loggrowth()").get() == NULL); + EXPECT_TRUE(ftf.createTable("linear()").get() == NULL); + EXPECT_TRUE(ftf.createTable("none").get() == NULL); + EXPECT_TRUE(ftf.createTable("none(").get() == NULL); + EXPECT_TRUE(ftf.createTable("none)").get() == NULL); + EXPECT_TRUE(ftf.createTable("none)(").get() == NULL); +} + +void +TableTest::testTableManager() +{ + { + TableManager tm; + tm.addFactory(ITableFactory::SP(new FileTableFactory("tables1"))); + tm.addFactory(ITableFactory::SP(new FileTableFactory("tables2"))); + + { + const Table * t = tm.getTable("a"); // from tables1 + ASSERT_TRUE(t != NULL); + EXPECT_TRUE(assertTable(*t, Table().add(1.5).add(2.25).add(3))); + EXPECT_TRUE(t == tm.getTable("a")); + } + { + const Table * t = tm.getTable("b"); // from tables2 + ASSERT_TRUE(t != NULL); + EXPECT_TRUE(assertTable(*t, Table().add(40).add(50).add(60))); + EXPECT_TRUE(t == tm.getTable("b")); + } + { + EXPECT_TRUE(tm.getTable("c") == NULL); + EXPECT_TRUE(tm.getTable("c") == NULL); + } + } + { + TableManager tm; + ASSERT_TRUE(tm.getTable("a") == NULL); + } +} + +int +TableTest::Main() +{ + TEST_INIT("table_test"); + + testTable(); + testFileTableFactory(); + testFunctionTableFactory(); + testTableManager(); + + TEST_DONE(); +} + +} +} + +TEST_APPHOOK(search::fef::TableTest); diff --git a/searchlib/src/tests/fef/table/tables1/a b/searchlib/src/tests/fef/table/tables1/a new file mode 100644 index 00000000000..c46f4d59a71 --- /dev/null +++ b/searchlib/src/tests/fef/table/tables1/a @@ -0,0 +1,3 @@ +1.5 +2.25 +3 diff --git a/searchlib/src/tests/fef/table/tables2/a b/searchlib/src/tests/fef/table/tables2/a new file mode 100644 index 00000000000..300ed6fcd17 --- /dev/null +++ b/searchlib/src/tests/fef/table/tables2/a @@ -0,0 +1,3 @@ +10 +20 +30 diff --git a/searchlib/src/tests/fef/table/tables2/b b/searchlib/src/tests/fef/table/tables2/b new file mode 100644 index 00000000000..6f98b52f55f --- /dev/null +++ b/searchlib/src/tests/fef/table/tables2/b @@ -0,0 +1,3 @@ +40 +50 +60 diff --git a/searchlib/src/tests/fef/termfieldmodel/.gitignore b/searchlib/src/tests/fef/termfieldmodel/.gitignore new file mode 100644 index 00000000000..0f860efa14a --- /dev/null +++ b/searchlib/src/tests/fef/termfieldmodel/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +searchlib_termfieldmodel_test_app diff --git a/searchlib/src/tests/fef/termfieldmodel/CMakeLists.txt b/searchlib/src/tests/fef/termfieldmodel/CMakeLists.txt new file mode 100644 index 00000000000..c8a678c11bb --- /dev/null +++ b/searchlib/src/tests/fef/termfieldmodel/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_termfieldmodel_test_app + SOURCES + termfieldmodel_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_termfieldmodel_test_app COMMAND searchlib_termfieldmodel_test_app) diff --git a/searchlib/src/tests/fef/termfieldmodel/DESC b/searchlib/src/tests/fef/termfieldmodel/DESC new file mode 100644 index 00000000000..2c8df5a8aab --- /dev/null +++ b/searchlib/src/tests/fef/termfieldmodel/DESC @@ -0,0 +1 @@ +termfieldmodel test. Take a look at termfieldmodel_test.cpp for details. diff --git a/searchlib/src/tests/fef/termfieldmodel/FILES b/searchlib/src/tests/fef/termfieldmodel/FILES new file mode 100644 index 00000000000..b5440335bc6 --- /dev/null +++ b/searchlib/src/tests/fef/termfieldmodel/FILES @@ -0,0 +1 @@ +termfieldmodel_test.cpp diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp new file mode 100644 index 00000000000..26a02d38adf --- /dev/null +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -0,0 +1,209 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("termfieldmodel_test"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/fef/fef.h> +#include <vespa/searchlib/queryeval/searchiterator.h> + +#include <algorithm> + +using namespace search::fef; + +struct State { + SimpleTermData term; + MatchData::UP md; + TermFieldMatchData *f3; + TermFieldMatchData *f5; + TermFieldMatchData *f7; + TermFieldMatchDataArray array; + + State() : term(), md(), f3(0), f5(0), f7(0), array() {} + + void setArray(TermFieldMatchDataArray value) { + array = value; + } +}; + +void testInvalidId() { + const TermFieldMatchData empty; + using search::queryeval::SearchIterator; + + EXPECT_EQUAL(TermFieldMatchData::invalidId(), empty.getDocId()); + EXPECT_TRUE(TermFieldMatchData::invalidId() < (SearchIterator::beginId() + 1 ) || + TermFieldMatchData::invalidId() > (search::endDocId - 1)); +} + +void testSetup(State &state) { + MatchDataLayout layout; + + state.term.addField(3); // docfreq = 1 + state.term.addField(7); // docfreq = 2 + state.term.addField(5); // docfreq = 3 + + typedef search::fef::ITermFieldRangeAdapter FRA; + typedef search::fef::SimpleTermFieldRangeAdapter SFR; + + // lookup terms + { + int i = 1; + for (SFR iter(state.term); iter.valid(); iter.next()) { + iter.get().setDocFreq(0.25 * i++); + } + } + + // reserve handles + { + for (SFR iter(state.term); iter.valid(); iter.next()) { + iter.get().setHandle(layout.allocTermField(iter.get().getFieldId())); + } + } + + state.md = layout.createMatchData(); + + // init match data + { + for (FRA iter(state.term); iter.valid(); iter.next()) { + const ITermFieldData& tfd = iter.get(); + + TermFieldHandle handle = tfd.getHandle(); + TermFieldMatchData *data = state.md->resolveTermField(handle); + switch (tfd.getFieldId()) { + case 3: + state.f3 = data; + break; + case 5: + state.f5 = data; + break; + case 7: + state.f7 = data; + break; + default: + EXPECT_TRUE(false); + } + } + EXPECT_EQUAL(3u, state.f3->getFieldId()); + EXPECT_EQUAL(5u, state.f5->getFieldId()); + EXPECT_EQUAL(7u, state.f7->getFieldId()); + } + + // test that we can setup array + EXPECT_EQUAL(false, state.array.valid()); + state.setArray(TermFieldMatchDataArray().add(state.f3).add(state.f5).add(state.f7)); + EXPECT_EQUAL(true, state.array.valid()); +} + +void testGenerate(State &state) { + // verify array + EXPECT_EQUAL(3u, state.array.size()); + EXPECT_EQUAL(state.f3, state.array[0]); + EXPECT_EQUAL(state.f5, state.array[1]); + EXPECT_EQUAL(state.f7, state.array[2]); + + // stale unpacked data + state.f5->reset(5); + EXPECT_EQUAL(5u, state.f5->getDocId()); + { + TermFieldMatchDataPosition pos; + pos.setPosition(3); + pos.setElementId(0); + pos.setElementLen(10); + state.f5->appendPosition(pos); + EXPECT_EQUAL(1u, state.f5->getIterator().size()); + EXPECT_EQUAL(10u, state.f5->getIterator().getFieldLength()); + } + state.f5->reset(6); + EXPECT_EQUAL(6u, state.f5->getDocId()); + EXPECT_EQUAL(FieldPositionsIterator::UNKNOWN_LENGTH, + state.f5->getIterator().getFieldLength()); + EXPECT_EQUAL(0u, state.f5->getIterator().size()); + + + // fresh unpacked data + state.md->setDocId(10); + state.f3->reset(10); + { + TermFieldMatchDataPosition pos; + pos.setPosition(3); + pos.setElementId(0); + pos.setElementLen(10); + EXPECT_EQUAL(FieldPositionsIterator::UNKNOWN_LENGTH, + state.f3->getIterator().getFieldLength()); + state.f3->appendPosition(pos); + EXPECT_EQUAL(10u, state.f3->getIterator().getFieldLength()); + } + { + TermFieldMatchDataPosition pos; + pos.setPosition(15); + pos.setElementId(1); + pos.setElementLen(20); + state.f3->appendPosition(pos); + EXPECT_EQUAL(20u, state.f3->getIterator().getFieldLength()); + } + { + TermFieldMatchDataPosition pos; + pos.setPosition(1); + pos.setElementId(2); + pos.setElementLen(5); + state.f3->appendPosition(pos); + EXPECT_EQUAL(20u, state.f3->getIterator().getFieldLength()); + } + + // raw score + state.f7->setRawScore(10, 5.0); +} + +void testAnalyze(State &state) { + EXPECT_EQUAL(state.md->getDocId(), state.f3->getDocId()); + EXPECT_NOT_EQUAL(state.md->getDocId(), state.f5->getDocId()); + EXPECT_EQUAL(state.md->getDocId(), state.f7->getDocId()); + + FieldPositionsIterator it = state.f3->getIterator(); + EXPECT_EQUAL(20u, it.getFieldLength()); + EXPECT_EQUAL(3u, it.size()); + EXPECT_TRUE(it.valid()); + EXPECT_EQUAL(3u, it.getPosition()); + EXPECT_EQUAL(0u, it.getElementId()); + EXPECT_EQUAL(10u, it.getElementLen()); + it.next(); + EXPECT_TRUE(it.valid()); + EXPECT_EQUAL(15u, it.getPosition()); + EXPECT_EQUAL(1u, it.getElementId()); + EXPECT_EQUAL(20u, it.getElementLen()); + it.next(); + EXPECT_TRUE(it.valid()); + EXPECT_EQUAL(1u, it.getPosition()); + EXPECT_EQUAL(2u, it.getElementId()); + EXPECT_EQUAL(5u, it.getElementLen()); + it.next(); + EXPECT_TRUE(!it.valid()); + + EXPECT_EQUAL(0.0, state.f3->getRawScore()); + EXPECT_EQUAL(0.0, state.f5->getRawScore()); + EXPECT_EQUAL(5.0, state.f7->getRawScore()); +} + +TEST("term field model") { + State state; + testSetup(state); + testGenerate(state); + testAnalyze(state); + testInvalidId(); +} + +TEST("Access subqueries") { + State state; + testSetup(state); + state.f3->reset(10); + state.f3->setSubqueries(10, 42); + EXPECT_EQUAL(42ULL, state.f3->getSubqueries()); + state.f3->enableRawScore(); + EXPECT_EQUAL(0ULL, state.f3->getSubqueries()); + + state.f3->reset(11); + state.f3->appendPosition(TermFieldMatchDataPosition()); + state.f3->setSubqueries(11, 42); + EXPECT_EQUAL(0ULL, state.f3->getSubqueries()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/fef/termmatchdatamerger/.gitignore b/searchlib/src/tests/fef/termmatchdatamerger/.gitignore new file mode 100644 index 00000000000..64f3f4a4600 --- /dev/null +++ b/searchlib/src/tests/fef/termmatchdatamerger/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +searchlib_termmatchdatamerger_test_app diff --git a/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt b/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt new file mode 100644 index 00000000000..cfb6ae2611f --- /dev/null +++ b/searchlib/src/tests/fef/termmatchdatamerger/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_termmatchdatamerger_test_app + SOURCES + termmatchdatamerger_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_termmatchdatamerger_test_app COMMAND searchlib_termmatchdatamerger_test_app) diff --git a/searchlib/src/tests/fef/termmatchdatamerger/DESC b/searchlib/src/tests/fef/termmatchdatamerger/DESC new file mode 100644 index 00000000000..abacd50b719 --- /dev/null +++ b/searchlib/src/tests/fef/termmatchdatamerger/DESC @@ -0,0 +1 @@ +termmatchdatamerger test. Take a look at termmatchdatamerger.cpp for details. diff --git a/searchlib/src/tests/fef/termmatchdatamerger/FILES b/searchlib/src/tests/fef/termmatchdatamerger/FILES new file mode 100644 index 00000000000..709c15d91b8 --- /dev/null +++ b/searchlib/src/tests/fef/termmatchdatamerger/FILES @@ -0,0 +1 @@ +termmatchdatamerger_test.cpp diff --git a/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp b/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp new file mode 100644 index 00000000000..14b74498f2d --- /dev/null +++ b/searchlib/src/tests/fef/termmatchdatamerger/termmatchdatamerger_test.cpp @@ -0,0 +1,281 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("termmatchdatamerger_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/searchlib/fef/termfieldmatchdata.h> +#include <vespa/searchlib/fef/termfieldmatchdataarray.h> +#include <vespa/searchlib/fef/termmatchdatamerger.h> + +using namespace search::fef; + +typedef TermMatchDataMerger::Input MDMI; +typedef TermMatchDataMerger::Inputs MDMIs; + +namespace { + +TermFieldMatchDataPosition make_pos(uint32_t pos) +{ + return TermFieldMatchDataPosition(0, pos, 1, 1000); +} + +} // namespace <unnamed> + +class Test : public vespalib::TestApp +{ +public: + void testMergeEmptyInput(); + void testMergeSimple(); + void testMergeMultifield(); + void testMergeDuplicates(); + void testMergeFieldLength(); + int Main(); +}; + +void +Test::testMergeEmptyInput() +{ + TermFieldMatchData out; + TermFieldMatchDataArray output; + output.add(&out); + + TermFieldMatchData in; + MDMIs input; + input.push_back(MDMI(&in, 1.0)); + + TermMatchDataMerger merger(input, output); + + uint32_t docid = 5; + in.reset(docid); + merger.merge(docid); + EXPECT_EQUAL(docid, out.getDocId()); + EXPECT_TRUE(out.begin() == out.end()); +} + +void +Test::testMergeSimple() +{ + TermFieldMatchData a; + TermFieldMatchData b; + TermFieldMatchData c; + MDMIs input; + input.push_back(MDMI(&a, 0.5)); + input.push_back(MDMI(&b, 1.0)); + input.push_back(MDMI(&c, 1.5)); + + TermFieldMatchData out; + TermFieldMatchDataArray output; + output.add(&out); + TermMatchDataMerger merger(input, output); + + uint32_t docid = 5; + + a.reset(docid); + a.appendPosition(make_pos(5).setMatchExactness(0.5)); + a.appendPosition(make_pos(10).setMatchExactness(3.0)); + a.appendPosition(make_pos(15).setMatchExactness(2.0)); + + b.reset(docid); + b.appendPosition(make_pos(7).setMatchExactness(0.5)); + b.appendPosition(make_pos(20).setMatchExactness(4.0)); + + c.reset(docid); + c.appendPosition(make_pos(22).setMatchExactness(0.5)); + c.appendPosition(make_pos(27).setMatchExactness(2.0)); + c.appendPosition(make_pos(28).setMatchExactness(5.0)); + + merger.merge(docid); + + EXPECT_EQUAL(docid, out.getDocId()); + EXPECT_EQUAL(8u, out.end() - out.begin()); + + EXPECT_EQUAL( 5u, out.begin()[0].getPosition()); + EXPECT_EQUAL( 7u, out.begin()[1].getPosition()); + EXPECT_EQUAL(10u, out.begin()[2].getPosition()); + EXPECT_EQUAL(15u, out.begin()[3].getPosition()); + EXPECT_EQUAL(20u, out.begin()[4].getPosition()); + EXPECT_EQUAL(22u, out.begin()[5].getPosition()); + EXPECT_EQUAL(27u, out.begin()[6].getPosition()); + EXPECT_EQUAL(28u, out.begin()[7].getPosition()); + + EXPECT_EQUAL(0.25, out.begin()[0].getMatchExactness()); + EXPECT_EQUAL( 0.5, out.begin()[1].getMatchExactness()); + EXPECT_EQUAL( 1.5, out.begin()[2].getMatchExactness()); + EXPECT_EQUAL( 1.0, out.begin()[3].getMatchExactness()); + EXPECT_EQUAL( 4.0, out.begin()[4].getMatchExactness()); + EXPECT_EQUAL(0.75, out.begin()[5].getMatchExactness()); + EXPECT_EQUAL( 3.0, out.begin()[6].getMatchExactness()); + EXPECT_EQUAL( 7.5, out.begin()[7].getMatchExactness()); + + // one stale input + + docid = 10; + a.reset(docid); + a.appendPosition(make_pos(5)); + a.appendPosition(make_pos(10)); + a.appendPosition(make_pos(15)); + + merger.merge(docid); + + EXPECT_EQUAL(docid, out.getDocId()); + EXPECT_EQUAL(3u, out.end() - out.begin()); + + EXPECT_EQUAL( 5u, out.begin()[0].getPosition()); + EXPECT_EQUAL(10u, out.begin()[1].getPosition()); + EXPECT_EQUAL(15u, out.begin()[2].getPosition()); + + // both inputs are stale + + docid = 15; + + merger.merge(docid); + EXPECT_NOT_EQUAL(docid, out.getDocId()); +} + + +void +Test::testMergeMultifield() +{ + TermFieldMatchData a; + TermFieldMatchData b; + TermFieldMatchData c; + MDMIs input; + a.setFieldId(1); + b.setFieldId(2); + c.setFieldId(2); + input.push_back(MDMI(&a, 1.0)); + input.push_back(MDMI(&b, 0.5)); + input.push_back(MDMI(&c, 1.5)); + + TermFieldMatchData out1; + TermFieldMatchData out2; + TermFieldMatchData out3; + TermFieldMatchDataArray output; + out1.setFieldId(1); + out2.setFieldId(2); + out3.setFieldId(3); + output.add(&out1).add(&out2).add(&out3); + + TermMatchDataMerger merger(input, output); + + uint32_t docid = 5; + + a.reset(docid); + a.appendPosition(make_pos(5)); + a.appendPosition(make_pos(15)); + + b.reset(docid); + b.appendPosition(make_pos(7)); + b.appendPosition(make_pos(20)); + + c.reset(docid); + c.appendPosition(make_pos(5)); + c.appendPosition(make_pos(20)); + + merger.merge(docid); + + EXPECT_EQUAL(docid, out1.getDocId()); + EXPECT_EQUAL(docid, out2.getDocId()); + EXPECT_NOT_EQUAL(docid, out3.getDocId()); + + EXPECT_EQUAL(2u, out1.end() - out1.begin()); + EXPECT_EQUAL(3u, out2.end() - out2.begin()); + + EXPECT_EQUAL( 5u, out1.begin()[0].getPosition()); + EXPECT_EQUAL(15u, out1.begin()[1].getPosition()); + + EXPECT_EQUAL( 5u, out2.begin()[0].getPosition()); + EXPECT_EQUAL( 7u, out2.begin()[1].getPosition()); + EXPECT_EQUAL(20u, out2.begin()[2].getPosition()); + + EXPECT_EQUAL(1.0, out1.begin()[0].getMatchExactness()); + EXPECT_EQUAL(1.0, out1.begin()[1].getMatchExactness()); + + EXPECT_EQUAL(1.5, out2.begin()[0].getMatchExactness()); + EXPECT_EQUAL(0.5, out2.begin()[1].getMatchExactness()); + EXPECT_EQUAL(1.5, out2.begin()[2].getMatchExactness()); +} + +void +Test::testMergeDuplicates() +{ + TermFieldMatchData a; + TermFieldMatchData b; + MDMIs input; + input.push_back(MDMI(&a, 0.5)); + input.push_back(MDMI(&b, 1.5)); + + TermFieldMatchData out; + TermFieldMatchDataArray output; + output.add(&out); + TermMatchDataMerger merger(input, output); + + uint32_t docid = 5; + + a.reset(docid); + a.appendPosition(make_pos(5)); + a.appendPosition(make_pos(10)); + a.appendPosition(make_pos(15)); + + b.reset(docid); + b.appendPosition(make_pos(3)); + b.appendPosition(make_pos(10)); + b.appendPosition(make_pos(15)); + b.appendPosition(make_pos(17)); + + merger.merge(docid); + + EXPECT_EQUAL(docid, out.getDocId()); + EXPECT_EQUAL(5u, out.end() - out.begin()); + + EXPECT_EQUAL( 3u, out.begin()[0].getPosition()); + EXPECT_EQUAL(1.5, out.begin()[0].getMatchExactness()); + EXPECT_EQUAL( 5u, out.begin()[1].getPosition()); + EXPECT_EQUAL(0.5, out.begin()[1].getMatchExactness()); + EXPECT_EQUAL(10u, out.begin()[2].getPosition()); + EXPECT_EQUAL(1.5, out.begin()[2].getMatchExactness()); + EXPECT_EQUAL(15u, out.begin()[3].getPosition()); + EXPECT_EQUAL(1.5, out.begin()[3].getMatchExactness()); + EXPECT_EQUAL(17u, out.begin()[4].getPosition()); + EXPECT_EQUAL(1.5, out.begin()[4].getMatchExactness()); +} + +void +Test::testMergeFieldLength() +{ + TermFieldMatchData a; + TermFieldMatchData b; + MDMIs input; + input.push_back(MDMI(&a, 1.0)); + input.push_back(MDMI(&b, 1.0)); + + TermFieldMatchData out; + TermFieldMatchDataArray output; + output.add(&out); + TermMatchDataMerger merger(input, output); + + uint32_t docid = 5; + a.reset(docid); + a.appendPosition(make_pos(1)); + b.reset(docid); + b.appendPosition(make_pos(2)); + merger.merge(docid); + + EXPECT_EQUAL(docid, out.getDocId()); + EXPECT_EQUAL(1000u, out.getIterator().getFieldLength()); +} + +int +Test::Main() +{ + TEST_INIT("termmatchdatamerger_test"); + testMergeEmptyInput(); + testMergeSimple(); + testMergeMultifield(); + testMergeDuplicates(); + testMergeFieldLength(); + TEST_DONE(); +} + +TEST_APPHOOK(Test); |