diff options
author | Arne H Juul <arnej27959@users.noreply.github.com> | 2024-01-24 14:26:13 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-01-24 14:26:13 +0100 |
commit | 330bc45eb4e3ce75374dc82eb3d3177565318c16 (patch) | |
tree | c70e7b6ffdf00dc18f9988b575c32324a1e374ea /searchcore | |
parent | a03840fcf84b5d369158c51ab43f3b0a4a6730d2 (diff) | |
parent | 2b13aee65c77b5a0e0b62149da8a7c506c1fc14d (diff) |
Merge pull request #29976 from vespa-engine/arnej/unit-test-verify-ranksetup-streaming
write vsmfields.cfg and add smoke test
Diffstat (limited to 'searchcore')
-rw-r--r-- | searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp | 69 |
1 files changed, 59 insertions, 10 deletions
diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 159ae339aa9..d2e649bb339 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -8,6 +8,7 @@ #include <string> #include <vector> #include <map> +#include <set> #include <initializer_list> const char *prog = "../../../apps/verify_ranksetup/vespa-verify-ranksetup-bin"; @@ -25,6 +26,8 @@ using search::index::schema::DataType; using vespalib::make_string_short::fmt; +enum class SearchMode { INDEXED, STREAMING, BOTH }; + struct Writer { FILE *file; explicit Writer(const std::string &file_name) { @@ -145,6 +148,39 @@ struct Setup { out.fmt("indexfield[%zu].collectiontype %s\n", i, pos->second.second.c_str()); } } + void write_vsmfield(const Writer &out, size_t idx, std::string name, std::string dataType) { + out.fmt("fieldspec[%zu].name \"%s\"\n", idx, name.c_str()); + if (dataType == "STRING") { + out.fmt("fieldspec[%zu].searchmethod AUTOUTF8\n", idx); + out.fmt("fieldspec[%zu].normalize LOWERCASE\n", idx); + } else { + out.fmt("fieldspec[%zu].searchmethod %s\n", idx, dataType.c_str()); + } + } + void write_vsmfields(const Writer &out) { + std::set<std::string> allFields; + size_t i = 0; + for (const auto & field : indexes) { + write_vsmfield(out, i, field.first, field.second.first); + out.fmt("fieldspec[%zu].fieldtype INDEX\n", i); + i++; + allFields.insert(field.first); + } + for (const auto & field : attributes) { + if (allFields.count(field.first) != 0) continue; + write_vsmfield(out, i, field.first, field.second.dataType); + out.fmt("fieldspec[%zu].fieldtype ATTRIBUTE\n", i); + i++; + allFields.insert(field.first); + } + out.fmt("documenttype[0].name \"foobar\"\n"); + size_t j = 0; + for (const auto & field : allFields) { + out.fmt("documenttype[0].index[%zu].name \"%s\"\n", j, field.c_str()); + out.fmt("documenttype[0].index[%zu].field[0].name \"%s\"\n", j, field.c_str()); + j++; + } + } void write_rank_profiles(const Writer &out) { out.fmt("rankprofile[%zu]\n", extra_profiles.size() + 1); out.fmt("rankprofile[0].name \"default\"\n"); @@ -165,7 +201,7 @@ struct Setup { for (const auto &entry: constants) { out.fmt("constant[%zu].name \"%s\"\n", idx, entry.first.c_str()); out.fmt("constant[%zu].fileref \"12345\"\n", idx); - out.fmt("constant[%zu].type \"%s\"\n", idx, entry.second.c_str()); + out.fmt("constant[%zu].type \"%s\"\n", idx, entry.second.c_str()); ++idx; } } @@ -215,32 +251,45 @@ struct Setup { void generate() { write_attributes(Writer(gen_dir + "/attributes.cfg")); write_indexschema(Writer(gen_dir + "/indexschema.cfg")); + write_vsmfields(Writer(gen_dir + "/vsmfields.cfg")); write_rank_profiles(Writer(gen_dir + "/rank-profiles.cfg")); write_ranking_constants(Writer(gen_dir + "/ranking-constants.cfg")); write_ranking_expressions(Writer(gen_dir + "/ranking-expressions.cfg")); write_onnx_models(Writer(gen_dir + "/onnx-models.cfg")); write_self_cfg(Writer(gen_dir + "/verify-ranksetup.cfg")); } - bool verify() { + bool verify(SearchMode mode = SearchMode::BOTH) { + if (mode == SearchMode::BOTH) { + bool res_indexed = verify_mode(SearchMode::INDEXED); + bool res_streaming = verify_mode(SearchMode::STREAMING); + EXPECT_EQUAL(res_indexed, res_streaming); + return res_indexed; + } else { + return verify_mode(mode); + } + } + bool verify_mode(SearchMode mode) { generate(); - vespalib::Process process(fmt("%s dir:%s", prog, gen_dir.c_str()), true); + vespalib::Process process(fmt("%s dir:%s%s", prog, gen_dir.c_str(), + (mode == SearchMode::STREAMING ? " -S" : "")), + true); for (auto line = process.read_line(); !line.empty(); line = process.read_line()) { fprintf(stderr, "> %s\n", line.c_str()); } return (process.join() == 0); } - void verify_valid(std::initializer_list<std::string> features) { + void verify_valid(std::initializer_list<std::string> features, SearchMode mode = SearchMode::BOTH) { for (const std::string &f: features) { first_phase(f); - if (!EXPECT_TRUE(verify())) { + if (!EXPECT_TRUE(verify(mode))) { fprintf(stderr, "--> feature '%s' was invalid (should be valid)\n", f.c_str()); } } } - void verify_invalid(std::initializer_list<std::string> features) { + void verify_invalid(std::initializer_list<std::string> features, SearchMode mode = SearchMode::BOTH) { for (const std::string &f: features) { first_phase(f); - if (!EXPECT_TRUE(!verify())) { + if (!EXPECT_TRUE(!verify(mode))) { fprintf(stderr, "--> feature '%s' was valid (should be invalid)\n", f.c_str()); } } @@ -346,12 +395,12 @@ TEST_F("require that dump features can break validation", SimpleSetup()) { //----------------------------------------------------------------------------- TEST_F("require that fieldMatch feature requires single value field", SimpleSetup()) { - f.verify_invalid({"fieldMatch(keywords)", "fieldMatch(list)"}); + f.verify_invalid({"fieldMatch(keywords)", "fieldMatch(list)"}, SearchMode::INDEXED); f.verify_valid({"fieldMatch(title)"}); } TEST_F("require that age feature requires attribute parameter", SimpleSetup()) { - f.verify_invalid({"age(unknown)", "age(title)"}); + f.verify_invalid({"age(unknown)", "age(title)"}, SearchMode::INDEXED); f.verify_valid({"age(date)"}); } @@ -361,7 +410,7 @@ TEST_F("require that nativeRank can be used on any valid field", SimpleSetup()) } TEST_F("require that nativeAttributeMatch requires attribute parameter", SimpleSetup()) { - f.verify_invalid({"nativeAttributeMatch(unknown)", "nativeAttributeMatch(title)", "nativeAttributeMatch(title,date)"}); + f.verify_invalid({"nativeAttributeMatch(unknown)", "nativeAttributeMatch(title)", "nativeAttributeMatch(title,date)"}, SearchMode::INDEXED); f.verify_valid({"nativeAttributeMatch", "nativeAttributeMatch(date)"}); } |