diff options
Diffstat (limited to 'searchsummary/src/tests/docsummary')
24 files changed, 253 insertions, 29 deletions
diff --git a/searchsummary/src/tests/docsummary/CMakeLists.txt b/searchsummary/src/tests/docsummary/CMakeLists.txt index 4cd12eb4db6..9f5d767c094 100644 --- a/searchsummary/src/tests/docsummary/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_positionsdfw_test_app TEST SOURCES positionsdfw_test.cpp diff --git a/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt index 22e0d3e6477..cfd65e28bae 100644 --- a/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_annotation_converter_test_app TEST SOURCES annotation_converter_test.cpp diff --git a/searchsummary/src/tests/docsummary/annotation_converter/annotation_converter_test.cpp b/searchsummary/src/tests/docsummary/annotation_converter/annotation_converter_test.cpp index 17a7457711d..0a05e078382 100644 --- a/searchsummary/src/tests/docsummary/annotation_converter/annotation_converter_test.cpp +++ b/searchsummary/src/tests/docsummary/annotation_converter/annotation_converter_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/annotation/annotation.h> #include <vespa/document/annotation/span.h> @@ -9,9 +9,9 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/juniper/juniper_separators.h> +#include <vespa/searchlib/util/linguisticsannotation.h> #include <vespa/searchsummary/docsummary/annotation_converter.h> #include <vespa/searchsummary/docsummary/i_juniper_converter.h> -#include <vespa/searchsummary/docsummary/linguisticsannotation.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt index 8a82b2efb94..36f9cd5535d 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_attribute_combiner_test_app TEST SOURCES attribute_combiner_test.cpp diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 005fed41838..ebb11e20f89 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/attribute/attributevector.h> diff --git a/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt index 09d56ba065d..c5b220474ba 100644 --- a/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attributedfw/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_attributedfw_test_app TEST SOURCES attributedfw_test.cpp diff --git a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp index 88b3423a199..71b5f5be0b9 100644 --- a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp +++ b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchsummary/docsummary/attributedfw.h> diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt index 3591a36c8de..6dc3168c78a 100644 --- a/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/document_id_dfw/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_document_id_dfw_test_app TEST SOURCES document_id_dfw_test.cpp diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp index 8dfdd1877f1..99fd5197540 100644 --- a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp +++ b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/documentid.h> #include <vespa/document/datatype/documenttype.h> diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt b/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt index 701adf59268..b4d6eabe44a 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_matched_elements_filter_test_app TEST SOURCES matched_elements_filter_test.cpp diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 7b83dbe2c4e..c5a309993f1 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/datatype/documenttype.h> #include <vespa/document/datatype/arraydatatype.h> diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 778a0b145ce..28af522089b 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/juniper/rpinterface.h> #include <vespa/searchlib/attribute/extendableattributes.h> diff --git a/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt b/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt index 12138948aca..842722cd05e 100644 --- a/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/query_term_filter_factory/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_query_term_filter_factory_test_app TEST SOURCES query_term_filter_factory_test.cpp diff --git a/searchsummary/src/tests/docsummary/query_term_filter_factory/query_term_filter_factory_test.cpp b/searchsummary/src/tests/docsummary/query_term_filter_factory/query_term_filter_factory_test.cpp index c966edb3958..55c8bccef27 100644 --- a/searchsummary/src/tests/docsummary/query_term_filter_factory/query_term_filter_factory_test.cpp +++ b/searchsummary/src/tests/docsummary/query_term_filter_factory/query_term_filter_factory_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchcommon/common/schema.h> #include <vespa/searchsummary/docsummary/i_query_term_filter.h> diff --git a/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt b/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt index 3b941aec244..6882408f669 100644 --- a/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/result_class/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_result_class_test_app TEST SOURCES result_class_test.cpp diff --git a/searchsummary/src/tests/docsummary/result_class/result_class_test.cpp b/searchsummary/src/tests/docsummary/result_class/result_class_test.cpp index f5790c20b80..2c84e92d21a 100644 --- a/searchsummary/src/tests/docsummary/result_class/result_class_test.cpp +++ b/searchsummary/src/tests/docsummary/result_class/result_class_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchsummary/docsummary/docsum_field_writer.h> #include <vespa/searchsummary/docsummary/resultclass.h> diff --git a/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt b/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt index 807a2fe734d..a435430bca2 100644 --- a/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/slime_filler/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_slime_filler_test_app TEST SOURCES slime_filler_test.cpp diff --git a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp index ea0f6016da1..c20f9570ef8 100644 --- a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/documentid.h> #include <vespa/document/datatype/documenttype.h> @@ -68,6 +68,7 @@ using search::docsummary::IStringFieldConverter; using search::docsummary::ResultConfig; using search::docsummary::SlimeFiller; using search::docsummary::SlimeFillerFilter; +using vespalib::Memory; using vespalib::SimpleBuffer; using vespalib::Slime; using vespalib::eval::SimpleValue; @@ -146,17 +147,27 @@ get_document_types_config() class MockStringFieldConverter : public IStringFieldConverter { std::vector<vespalib::string> _result; + bool _render_wset_as_array; + bool _insert; public: - MockStringFieldConverter() + MockStringFieldConverter(bool render_wset_as_array, bool insert) : IStringFieldConverter(), - _result() + _result(), + _render_wset_as_array(render_wset_as_array), + _insert(insert) { } ~MockStringFieldConverter() override = default; - void convert(const document::StringFieldValue& input, vespalib::slime::Inserter&) override { + void convert(const document::StringFieldValue& input, vespalib::slime::Inserter& inserter) override { _result.emplace_back(input.getValueRef()); + if (_insert) { + inserter.insertString(Memory(input.getValueRef())); + } } const std::vector<vespalib::string>& get_result() const noexcept { return _result; } + bool render_weighted_set_as_array() const override { + return _render_wset_as_array; + } }; } @@ -188,6 +199,7 @@ protected: void expect_insert_summary_field_with_filter(const vespalib::string& exp, const FieldValue& fv, const std::vector<uint32_t>& matching_elems); void expect_insert_summary_field_with_field_filter(const vespalib::string& exp, const FieldValue& fv, const SlimeFillerFilter* filter); void expect_insert_juniper_field(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv); + void expect_insert_summary_field_with_converter(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv, MockStringFieldConverter& converter); }; SlimeFillerTest::SlimeFillerTest() @@ -317,7 +329,7 @@ SlimeFillerTest::expect_insert_callback(const std::vector<vespalib::string>& exp { Slime slime; SlimeInserter inserter(slime); - MockStringFieldConverter converter; + MockStringFieldConverter converter(false, false); SlimeFiller filler(inserter, &converter, SlimeFillerFilter::all()); fv.accept(filler); auto act_null = slime_to_string(slime); @@ -351,7 +363,7 @@ SlimeFillerTest::expect_insert_summary_field_with_field_filter(const vespalib::s { Slime slime; SlimeInserter inserter(slime); - SlimeFiller::insert_summary_field_with_field_filter(fv, inserter, filter); + SlimeFiller::insert_summary_field_with_field_filter(fv, inserter, nullptr, filter); auto act = slime_to_string(slime); EXPECT_EQ(exp, act); } @@ -361,7 +373,7 @@ SlimeFillerTest::expect_insert_juniper_field(const std::vector<vespalib::string> { Slime slime; SlimeInserter inserter(slime); - MockStringFieldConverter converter; + MockStringFieldConverter converter(false, false); SlimeFiller::insert_juniper_field(fv, inserter, converter); auto act_slime = slime_to_string(slime); EXPECT_EQ(exp_slime, act_slime); @@ -369,6 +381,18 @@ SlimeFillerTest::expect_insert_juniper_field(const std::vector<vespalib::string> EXPECT_EQ(exp, act); } +void +SlimeFillerTest::expect_insert_summary_field_with_converter(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv, MockStringFieldConverter& converter) +{ + Slime slime; + SlimeInserter inserter(slime); + SlimeFiller::insert_summary_field(fv, inserter, &converter); + auto act_slime = slime_to_string(slime); + EXPECT_EQ(exp_slime, act_slime); + auto act = converter.get_result(); + EXPECT_EQ(exp, act); +} + TEST_F(SlimeFillerTest, insert_primitive_values) { { @@ -625,4 +649,16 @@ TEST_F(SlimeFillerTest, insert_juniper_field) expect_insert_juniper_field({}, "null", make_empty_array()); } +TEST_F(SlimeFillerTest, string_field_is_not_converted_for_weighted_set_rendering) +{ + MockStringFieldConverter cvt_as_wset(false, true); + expect_insert_summary_field_with_converter({}, R"([{"item":"foo","weight":2},{"item":"bar","weight":4},{"item":"baz","weight":6}])", make_weighted_set(), cvt_as_wset); +} + +TEST_F(SlimeFillerTest, weighted_set_can_be_rendered_as_array) +{ + MockStringFieldConverter cvt_as_array(true, true); + expect_insert_summary_field_with_converter({"foo","bar","baz"}, R"(["foo","bar","baz"])", make_weighted_set(), cvt_as_array); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt b/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt index 2f886a24a95..47ab5eafd21 100644 --- a/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/slime_filler_filter/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_slime_filler_filter_test_app TEST SOURCES slime_filler_filter_test.cpp diff --git a/searchsummary/src/tests/docsummary/slime_filler_filter/slime_filler_filter_test.cpp b/searchsummary/src/tests/docsummary/slime_filler_filter/slime_filler_filter_test.cpp index 9211981679b..37aeb09a1f1 100644 --- a/searchsummary/src/tests/docsummary/slime_filler_filter/slime_filler_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_filler_filter/slime_filler_filter_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchsummary/docsummary/slime_filler_filter.h> #include <vespa/vespalib/gtest/gtest.h> diff --git a/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt b/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt index 26456dae395..943e65a59bc 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(searchsummary_slime_summary_test_app TEST SOURCES slime_summary_test.cpp diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 7077f707247..7ab114ccd23 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -1,4 +1,4 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/documentid.h> #include <vespa/document/datatype/documenttype.h> diff --git a/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt new file mode 100644 index 00000000000..68885a74b1b --- /dev/null +++ b/searchsummary/src/tests/docsummary/tokens_converter/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_tokens_converter_test_app TEST + SOURCES + tokens_converter_test.cpp + DEPENDS + searchsummary + GTest::gtest +) + +vespa_add_test(NAME searchsummary_tokens_converter_test_app COMMAND searchsummary_tokens_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp new file mode 100644 index 00000000000..493cbe0ecba --- /dev/null +++ b/searchsummary/src/tests/docsummary/tokens_converter/tokens_converter_test.cpp @@ -0,0 +1,178 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/datatype/annotationtype.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/searchlib/util/linguisticsannotation.h> +#include <vespa/searchlib/util/token_extractor.h> +#include <vespa/searchsummary/docsummary/tokens_converter.h> +#include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> + +using document::Annotation; +using document::AnnotationType; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::Span; +using document::SpanList; +using document::SpanTree; +using document::StringFieldValue; +using search::docsummary::TokensConverter; +using search::linguistics::SPANTREE_NAME; +using search::linguistics::TokenExtractor; +using vespalib::SimpleBuffer; +using vespalib::Slime; +using vespalib::slime::JsonFormat; +using vespalib::slime::SlimeInserter; + +namespace { + +vespalib::string +slime_to_string(const Slime& slime) +{ + SimpleBuffer buf; + JsonFormat::encode(slime, buf, true); + return buf.get().make_string(); +} + +DocumenttypesConfig +get_document_types_config() +{ + using namespace document::config_builder; + DocumenttypesConfigBuilderHelper builder; + builder.document(42, "indexingdocument", + Struct("indexingdocument.header"), + Struct("indexingdocument.body")); + return builder.config(); +} + +} + +class TokensConverterTest : public testing::Test +{ +protected: + std::shared_ptr<const DocumentTypeRepo> _repo; + const DocumentType* _document_type; + document::FixedTypeRepo _fixed_repo; + vespalib::string _dummy_field_name; + TokenExtractor _token_extractor; + + TokensConverterTest(); + ~TokensConverterTest() override; + void set_span_tree(StringFieldValue& value, std::unique_ptr<SpanTree> tree); + StringFieldValue make_annotated_string(bool alt_tokens); + StringFieldValue make_annotated_chinese_string(); + vespalib::string make_exp_annotated_chinese_string_tokens(); + vespalib::string convert(const StringFieldValue& fv); +}; + +TokensConverterTest::TokensConverterTest() + : testing::Test(), + _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())), + _document_type(_repo->getDocumentType("indexingdocument")), + _fixed_repo(*_repo, *_document_type), + _dummy_field_name(), + _token_extractor(_dummy_field_name, 100) +{ +} + +TokensConverterTest::~TokensConverterTest() = default; + +void +TokensConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree) +{ + StringFieldValue::SpanTrees trees; + trees.push_back(std::move(tree)); + value.setSpanTrees(trees, _fixed_repo); +} + +StringFieldValue +TokensConverterTest::make_annotated_string(bool alt_tokens) +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + tree->annotate(span_list->add(std::make_unique<Span>(0, 3)), *AnnotationType::TERM); + if (alt_tokens) { + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), *AnnotationType::TERM); + } + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), + Annotation(*AnnotationType::TERM, std::make_unique<StringFieldValue>("baz"))); + StringFieldValue value("foo bar"); + set_span_tree(value, std::move(tree)); + return value; +} + +StringFieldValue +TokensConverterTest::make_annotated_chinese_string() +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + // These chinese characters each use 3 bytes in their UTF8 encoding. + tree->annotate(span_list->add(std::make_unique<Span>(0, 15)), *AnnotationType::TERM); + tree->annotate(span_list->add(std::make_unique<Span>(15, 9)), *AnnotationType::TERM); + StringFieldValue value("我就是那个大灰狼"); + set_span_tree(value, std::move(tree)); + return value; +} + +vespalib::string +TokensConverterTest::make_exp_annotated_chinese_string_tokens() +{ + return R"(["我就是那个","大灰狼"])"; +} + +vespalib::string +TokensConverterTest::convert(const StringFieldValue& fv) +{ + TokensConverter converter(_token_extractor); + Slime slime; + SlimeInserter inserter(slime); + converter.convert(fv, inserter); + return slime_to_string(slime); +} + +TEST_F(TokensConverterTest, convert_empty_string) +{ + vespalib::string exp(R"([])"); + StringFieldValue plain_string(""); + EXPECT_EQ(exp, convert(plain_string)); +} + +TEST_F(TokensConverterTest, convert_plain_string) +{ + vespalib::string exp(R"(["Foo Bar Baz"])"); + StringFieldValue plain_string("Foo Bar Baz"); + EXPECT_EQ(exp, convert(plain_string)); +} + +TEST_F(TokensConverterTest, convert_annotated_string) +{ + vespalib::string exp(R"(["foo","baz"])"); + auto annotated_string = make_annotated_string(false); + EXPECT_EQ(exp, convert(annotated_string)); +} + +TEST_F(TokensConverterTest, convert_annotated_string_with_alternatives) +{ + vespalib::string exp(R"(["foo",["bar","baz"]])"); + auto annotated_string = make_annotated_string(true); + EXPECT_EQ(exp, convert(annotated_string)); +} + +TEST_F(TokensConverterTest, convert_annotated_chinese_string) +{ + auto exp = make_exp_annotated_chinese_string_tokens(); + auto annotated_chinese_string = make_annotated_chinese_string(); + EXPECT_EQ(exp, convert(annotated_chinese_string)); +} + +GTEST_MAIN_RUN_ALL_TESTS() |