diff options
18 files changed, 250 insertions, 75 deletions
diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml index d2287506912..c2a011e0408 100644 --- a/maven-plugins/pom.xml +++ b/maven-plugins/pom.xml @@ -9,8 +9,7 @@ <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugins</artifactId> - <name>bundle-plugins</name> + <artifactId>maven-plugins</artifactId> <packaging>pom</packaging> <version>7-SNAPSHOT</version> <description>Parent artifact for Vespa maven plugins.</description> diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index edb91787214..c65257e7f6a 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/queryeval/begin_and_end_id.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/transport.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/stringfmt.h> @@ -213,4 +214,10 @@ DocsumContext::ParseLocation(search::docsummary::GetDocsumsState *state) state->_parsedLocation.reset(getLocation(_request.location, _attrMgr)); } +std::unique_ptr<MatchingElements> +DocsumContext::fill_matching_elements() +{ + return std::make_unique<MatchingElements>(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h index 45f3ca8e44f..30b5ef16cb1 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h @@ -48,6 +48,7 @@ public: void FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; void FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; void ParseLocation(search::docsummary::GetDocsumsState * state) override; + std::unique_ptr<search::MatchingElements> fill_matching_elements() override; }; } // namespace proton diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt index df323b9c982..cffdef25e5b 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/attribute_combiner/CMakeLists.txt @@ -1,8 +1,10 @@ # Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(GTest REQUIRED) vespa_add_executable(searchsummary_attribute_combiner_test_app TEST SOURCES attribute_combiner_test.cpp DEPENDS searchsummary + GTest::GTest ) vespa_add_test(NAME searchsummary_attribute_combiner_test_app COMMAND searchsummary_attribute_combiner_test_app) diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 60b1574a8d5..961acadeddf 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -8,12 +8,13 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> #include <vespa/searchsummary/docsummary/attribute_combiner_dfw.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("attribute_combiner_test"); @@ -23,6 +24,7 @@ using search::AttributeManager; using search::AttributeVector; using search::IntegerAttribute; using search::FloatingPointAttribute; +using search::MatchingElements; using search::StringAttribute; using search::attribute::BasicType; using search::attribute::CollectionType; @@ -122,7 +124,7 @@ AttributeManagerFixture::buildAttribute(const vespalib::string &name, for (const auto &docValues : values) { uint32_t docId = 0; EXPECT_TRUE(attr->addDoc(docId)); - EXPECT_NOT_EQUAL(0u, docId); + EXPECT_NE(0u, docId); for (const auto &value : docValues) { attr->append(docId, value, 1); } @@ -157,40 +159,65 @@ AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name, class DummyStateCallback : public GetDocsumsStateCallback { public: + MatchingElements _matching_elements; + + DummyStateCallback(); void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void ParseLocation(GetDocsumsState *) override { } + std::unique_ptr<MatchingElements> fill_matching_elements() override { return std::make_unique<MatchingElements>(_matching_elements); } ~DummyStateCallback() override { } }; +DummyStateCallback::DummyStateCallback() + : GetDocsumsStateCallback(), + _matching_elements() +{ + _matching_elements.add_matching_elements(1, "array", {1}); + _matching_elements.add_matching_elements(3, "array", {0}); + _matching_elements.add_matching_elements(4, "array", {1}); + _matching_elements.add_matching_elements(1, "smap", {1}); + _matching_elements.add_matching_elements(3, "smap", {0}); + _matching_elements.add_matching_elements(4, "smap", {1}); + _matching_elements.add_matching_elements(1, "map", {1}); + _matching_elements.add_matching_elements(3, "map", {0}); + _matching_elements.add_matching_elements(4, "map", {1}); +} -struct Fixture +struct AttributeCombinerTest : public ::testing::Test { AttributeManagerFixture attrs; std::unique_ptr<IDocsumFieldWriter> writer; DummyStateCallback stateCallback; GetDocsumsState state; - Fixture(const vespalib::string &fieldName); - ~Fixture(); + AttributeCombinerTest(); + ~AttributeCombinerTest(); + void set_field(const vespalib::string &field_name, bool filter_elements); void assertWritten(const vespalib::string &exp, uint32_t docId); }; -Fixture::Fixture(const vespalib::string &fieldName) +AttributeCombinerTest::AttributeCombinerTest() : attrs(), - writer(AttributeCombinerDFW::create(fieldName, attrs.mgr)), + writer(), stateCallback(), state(stateCallback) { - EXPECT_TRUE(writer->setFieldWriterStateIndex(0)); state._attrCtx = attrs.mgr.createContext(); - state._fieldWriterStates.resize(1); } -Fixture::~Fixture() = default; +AttributeCombinerTest::~AttributeCombinerTest() = default; + +void +AttributeCombinerTest::set_field(const vespalib::string &field_name, bool filter_elements) +{ + writer = AttributeCombinerDFW::create(field_name, attrs.mgr, filter_elements); + EXPECT_TRUE(writer->setFieldWriterStateIndex(0)); + state._fieldWriterStates.resize(1); +} void -Fixture::assertWritten(const vespalib::string &expectedJson, uint32_t docId) +AttributeCombinerTest::assertWritten(const vespalib::string &expectedJson, uint32_t docId) { vespalib::Slime target; vespalib::slime::SlimeInserter inserter(target); @@ -200,41 +227,76 @@ Fixture::assertWritten(const vespalib::string &expectedJson, uint32_t docId) search::SlimeOutputRawBufAdapter adapter(binary); vespalib::slime::BinaryFormat::encode(target, adapter); FieldBlock block(expectedJson); - if (!EXPECT_EQUAL(block.dataLen(), binary.GetUsedLen()) || - !EXPECT_EQUAL(0, memcmp(block.data(), binary.GetDrainPos(), block.dataLen()))) { + EXPECT_EQ(block.dataLen(), binary.GetUsedLen()); + EXPECT_EQ(0, memcmp(block.data(), binary.GetDrainPos(), block.dataLen())); + if (block.dataLen() != binary.GetUsedLen() || + memcmp(block.data(), binary.GetDrainPos(), block.dataLen()) != 0) { LOG(error, "Expected '%s'", expectedJson.c_str()); LOG(error, "Expected normalized '%s'", block.json.c_str()); LOG(error, "Got '%s'", json.c_str()); } } -TEST_F("require that attribute combiner dfw generates correct slime output for array of struct", Fixture("array")) +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_array_of_struct) +{ + set_field("array", false); + assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1); + assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2); + assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3); + assertWritten("[ { }, { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); + assertWritten("null", 5); +} + +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_struct) +{ + set_field("smap", false); + assertWritten("[ { key: \"k1.1\", value: { fval: 110.0, name: \"n1.1\", val: 10} }, { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); + assertWritten("[ { key: \"k2\", value: { fval: 120.0, name: \"n2\", val: 20} }, { value: { fval: 121.0, val: 21 } }]", 2); + assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} }, { key: \"k3.2\", value: { fval: 131.0, name: \"n3.2\"} } ]", 3); + assertWritten("[ { value: { } }, { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("null", 5); +} + +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_string) +{ + set_field("map", false); + assertWritten("[ { key: \"k1.1\", value: \"n1.1\" }, { key: \"k1.2\", value: \"n1.2\"}]", 1); + assertWritten("[ { key: \"k2\"}]", 2); + assertWritten("[ { key: \"k3.1\", value: \"n3.1\" }, { value: \"n3.2\"} ]", 3); + assertWritten("[ { }, { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("null", 5); +} + +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_array_of_struct) { - TEST_DO(f.assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1)); - TEST_DO(f.assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2)); - TEST_DO(f.assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3)); - TEST_DO(f.assertWritten("[ { }, { fval: 141.0, name: \"n4.2\", val: 41} ]", 4)); - TEST_DO(f.assertWritten("null", 5)); + set_field("array", true); + assertWritten("[ { name: \"n1.2\", val: 11}]", 1); + assertWritten("[ ]", 2); + assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30} ]", 3); + assertWritten("[ { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); + assertWritten("null", 5); } -TEST_F("require that attribute combiner dfw generates correct slime output for map of struct", Fixture("smap")) +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_struct) { - TEST_DO(f.assertWritten("[ { key: \"k1.1\", value: { fval: 110.0, name: \"n1.1\", val: 10} }, { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1)); - TEST_DO(f.assertWritten("[ { key: \"k2\", value: { fval: 120.0, name: \"n2\", val: 20} }, { value: { fval: 121.0, val: 21 } }]", 2)); - TEST_DO(f.assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} }, { key: \"k3.2\", value: { fval: 131.0, name: \"n3.2\"} } ]", 3)); - TEST_DO(f.assertWritten("[ { value: { } }, { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4)); - TEST_DO(f.assertWritten("null", 5)); + set_field("smap", true); + assertWritten("[ { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); + assertWritten("[ ]", 2); + assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} } ]", 3); + assertWritten("[ { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("null", 5); } -TEST_F("require that attribute combiner dfw generates correct slime output for map of string", Fixture("map")) +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_string) { - TEST_DO(f.assertWritten("[ { key: \"k1.1\", value: \"n1.1\" }, { key: \"k1.2\", value: \"n1.2\"}]", 1)); - TEST_DO(f.assertWritten("[ { key: \"k2\"}]", 2)); - TEST_DO(f.assertWritten("[ { key: \"k3.1\", value: \"n3.1\" }, { value: \"n3.2\"} ]", 3)); - TEST_DO(f.assertWritten("[ { }, { key: \"k4.2\", value: \"n4.2\" } ]", 4)); - TEST_DO(f.assertWritten("null", 5)); + set_field("map", true); + assertWritten("[ { key: \"k1.2\", value: \"n1.2\"}]", 1); + assertWritten("[ ]", 2); + assertWritten("[ { key: \"k3.1\", value: \"n3.1\" } ]", 3); + assertWritten("[ { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("null", 5); } } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 764ff4723cb..476891ca40f 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/docsumfieldwriter.h> #include <vespa/searchsummary/docsummary/positionsdfw.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> @@ -17,6 +18,7 @@ LOG_SETUP("positionsdfw_test"); using search::RawBuf; using search::IAttributeManager; +using search::MatchingElements; using search::SingleInt64ExtAttribute; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -104,6 +106,7 @@ struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { virtual void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} virtual void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} virtual void ParseLocation(GetDocsumsState *) override {} + std::unique_ptr<MatchingElements> fill_matching_elements() override { abort(); } }; template <typename AttrType> diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 5d6565ceadb..efeb066135f 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/docsumstate.h> @@ -9,6 +10,7 @@ using namespace vespalib::slime::convenience; using namespace search::docsummary; +using search::MatchingElements; namespace { @@ -77,6 +79,7 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void ParseLocation(GetDocsumsState *) override { } + std::unique_ptr<MatchingElements> fill_matching_elements() override { abort(); } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp index 6a643a74458..9d4d3829047 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -5,7 +5,9 @@ #include "attribute_field_writer.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <cassert> using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -18,20 +20,29 @@ namespace { class ArrayAttributeFieldWriterState : public DocsumFieldWriterState { std::vector<std::unique_ptr<AttributeFieldWriter>> _writers; + const vespalib::string& _field_name; + const MatchingElements* const _matching_elements; public: ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames, const std::vector<vespalib::string> &attributeNames, - IAttributeContext &context); + IAttributeContext &context, + const vespalib::string &field_name, + const MatchingElements* matching_elements); ~ArrayAttributeFieldWriterState() override; + void insert_element(uint32_t element_index, Cursor &array); void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; }; ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames, const std::vector<vespalib::string> &attributeNames, - IAttributeContext &context) + IAttributeContext &context, + const vespalib::string &field_name, + const MatchingElements *matching_elements) : DocsumFieldWriterState(), - _writers() + _writers(), + _field_name(field_name), + _matching_elements(matching_elements) { size_t fields = fieldNames.size(); _writers.reserve(fields); @@ -46,6 +57,15 @@ ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector ArrayAttributeFieldWriterState::~ArrayAttributeFieldWriterState() = default; void +ArrayAttributeFieldWriterState::insert_element(uint32_t element_index, Cursor &array) +{ + Cursor &obj = array.addObject(); + for (auto &writer : _writers) { + writer->print(element_index, obj); + } +} + +void ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Inserter &target) { uint32_t elems = 0; @@ -59,10 +79,19 @@ ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Ins return; } Cursor &arr = target.insertArray(); - for (uint32_t idx = 0; idx < elems; ++idx) { - Cursor &obj = arr.addObject(); - for (auto &writer : _writers) { - writer->print(idx, obj); + if (_matching_elements != nullptr) { + auto &elements = _matching_elements->get_matching_elements(docId, _field_name); + auto elements_iterator = elements.cbegin(); + for (uint32_t idx = 0; idx < elems && elements_iterator != elements.cend(); ++idx) { + assert(*elements_iterator >= idx); + if (*elements_iterator == idx) { + insert_element(idx, arr); + ++elements_iterator; + } + } + } else { + for (uint32_t idx = 0; idx < elems; ++idx) { + insert_element(idx, arr); } } } @@ -70,8 +99,9 @@ ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Ins } ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &fields) - : AttributeCombinerDFW(fieldName), + const std::vector<vespalib::string> &fields, + bool filter_elements) + : AttributeCombinerDFW(fieldName, filter_elements), _fields(fields), _attributeNames() { @@ -85,9 +115,9 @@ ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fie ArrayAttributeCombinerDFW::~ArrayAttributeCombinerDFW() = default; std::unique_ptr<DocsumFieldWriterState> -ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context) +ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context, const MatchingElements* matching_elements) { - return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context); + return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context, _fieldName, matching_elements); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h index c02d2bd5da6..15c3030c782 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -19,10 +19,11 @@ class ArrayAttributeCombinerDFW : public AttributeCombinerDFW std::vector<vespalib::string> _fields; std::vector<vespalib::string> _attributeNames; - std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) override; + std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: ArrayAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &fields); + const std::vector<vespalib::string> &fields, + bool filter_elements); ~ArrayAttributeCombinerDFW() override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index ef621dc5b4a..ed053ce0115 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -89,9 +89,10 @@ StructFields::~StructFields() = default; } -AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName) +AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements) : ISimpleDFW(), _stateIndex(0), + _filter_elements(filter_elements), _fieldName(fieldName) { } @@ -112,15 +113,15 @@ AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) } std::unique_ptr<IDocsumFieldWriter> -AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr) +AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr, bool filter_elements) { StructFields structFields(fieldName, attrMgr); if (structFields.getError()) { return std::unique_ptr<IDocsumFieldWriter>(); } else if (!structFields.getMapFields().empty()) { - return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields.getMapFields()); + return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields.getMapFields(), filter_elements); } - return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields.getArrayFields()); + return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields.getArrayFields(), filter_elements); } void @@ -128,7 +129,11 @@ AttributeCombinerDFW::insertField(uint32_t docid, GetDocsumsState *state, ResTyp { auto &fieldWriterState = state->_fieldWriterStates[_stateIndex]; if (!fieldWriterState) { - fieldWriterState = allocFieldWriterState(*state->_attrCtx); + const MatchingElements *matching_elements = nullptr; + if (_filter_elements) { + matching_elements = &state->get_matching_elements(); + } + fieldWriterState = allocFieldWriterState(*state->_attrCtx, matching_elements); } fieldWriterState->insertField(docid, target); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index 6c0991fdf92..ec806ba0ac6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -4,6 +4,7 @@ #include "docsumfieldwriter.h" +namespace search { class MatchingElements; } namespace search::attribute { class IAttributeContext; } namespace search::docsummary { @@ -19,15 +20,16 @@ class AttributeCombinerDFW : public ISimpleDFW { protected: uint32_t _stateIndex; + const bool _filter_elements; vespalib::string _fieldName; - AttributeCombinerDFW(const vespalib::string &fieldName); + AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements); protected: - virtual std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) = 0; + virtual std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) = 0; public: ~AttributeCombinerDFW() override; bool IsGenerated() const override; bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; - static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr); + static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr, bool filter_elements); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index b95591546b5..3a06346c7b4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -94,7 +94,13 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & } } else if (overrideName == "attributecombiner") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { - fieldWriter = AttributeCombinerDFW::create(fieldName, *getEnvironment()->getAttributeManager()); + fieldWriter = AttributeCombinerDFW::create(fieldName, *getEnvironment()->getAttributeManager(), false); + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "matchedattributeelementsfilter") { + string source_field = argument.empty() ? fieldName : argument; + if (getEnvironment() && getEnvironment()->getAttributeManager()) { + fieldWriter = AttributeCombinerDFW::create(source_field, *getEnvironment()->getAttributeManager(), true); rc = static_cast<bool>(fieldWriter); } } else { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index b0431b6e6ac..7a609f2c971 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -4,10 +4,10 @@ #include <vespa/juniper/rpinterface.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/common/matching_elements.h> #include "docsum_field_writer_state.h" -namespace search { -namespace docsummary { +namespace search::docsummary { GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) : _args(), @@ -25,7 +25,8 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _parsedLocation(), _summaryFeatures(NULL), _summaryFeaturesCached(false), - _rankFeatures(NULL) + _rankFeatures(NULL), + _matching_elements() { _dynteaser._docid = static_cast<uint32_t>(-1); _dynteaser._input = static_cast<uint32_t>(-1); @@ -48,5 +49,13 @@ GetDocsumsState::~GetDocsumsState() } } +const MatchingElements & +GetDocsumsState::get_matching_elements() +{ + if (!_matching_elements) { + _matching_elements = _callback.fill_matching_elements(); + } + return *_matching_elements; } + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index fa47d5244eb..9eec51e3459 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -13,6 +13,7 @@ namespace juniper { class Result; } +namespace search { class MatchingElements; } namespace search::common { class Location; } namespace search::attribute { class IAttributeContext; @@ -31,6 +32,7 @@ public: virtual void FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; virtual void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; virtual void ParseLocation(GetDocsumsState * state) = 0; + virtual std::unique_ptr<MatchingElements> fill_matching_elements() = 0; virtual ~GetDocsumsStateCallback(void) { } GetDocsumsStateCallback(const GetDocsumsStateCallback &) = delete; GetDocsumsStateCallback & operator = (const GetDocsumsStateCallback &) = delete; @@ -84,10 +86,14 @@ public: // used by RankFeaturesDFW FeatureSet::SP _rankFeatures; + // Used by AttributeCombinerDFW when filtering is enabled + std::unique_ptr<search::MatchingElements> _matching_elements; + GetDocsumsState(const GetDocsumsState &) = delete; GetDocsumsState& operator=(const GetDocsumsState &) = delete; GetDocsumsState(GetDocsumsStateCallback &callback); ~GetDocsumsState(); + const MatchingElements &get_matching_elements(); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp index e4e30afbc4d..e34012de7de 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp @@ -5,7 +5,9 @@ #include "attribute_field_writer.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <cassert> using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -22,23 +24,32 @@ class StructMapAttributeFieldWriterState : public DocsumFieldWriterState { std::unique_ptr<AttributeFieldWriter> _keyWriter; std::vector<std::unique_ptr<AttributeFieldWriter>> _valueWriters; + const vespalib::string& _field_name; + const MatchingElements* const _matching_elements; public: StructMapAttributeFieldWriterState(const vespalib::string &keyAttributeName, const std::vector<vespalib::string> &valueFieldNames, const std::vector<vespalib::string> &valueAttributeNames, - IAttributeContext &context); + IAttributeContext &context, + const vespalib::string &field_name, + const MatchingElements* matching_elements); ~StructMapAttributeFieldWriterState() override; + void insert_element(uint32_t element_index, Cursor &array); void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; }; StructMapAttributeFieldWriterState::StructMapAttributeFieldWriterState(const vespalib::string &keyAttributeName, const std::vector<vespalib::string> &valueFieldNames, const std::vector<vespalib::string> &valueAttributeNames, - IAttributeContext &context) + IAttributeContext &context, + const vespalib::string& field_name, + const MatchingElements *matching_elements) : DocsumFieldWriterState(), _keyWriter(), - _valueWriters() + _valueWriters(), + _field_name(field_name), + _matching_elements(matching_elements) { const IAttributeVector *attr = context.getAttribute(keyAttributeName); if (attr != nullptr) { @@ -57,6 +68,19 @@ StructMapAttributeFieldWriterState::StructMapAttributeFieldWriterState(const ves StructMapAttributeFieldWriterState::~StructMapAttributeFieldWriterState() = default; void +StructMapAttributeFieldWriterState::insert_element(uint32_t element_index, Cursor &array) +{ + Cursor &keyValueObj = array.addObject(); + if (_keyWriter) { + _keyWriter->print(element_index, keyValueObj); + } + Cursor &obj = keyValueObj.setObject(valueName); + for (auto &valueWriter : _valueWriters) { + valueWriter->print(element_index, obj); + } +} + +void StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Inserter &target) { uint32_t elems = 0; @@ -76,14 +100,19 @@ StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime: return; } Cursor &arr = target.insertArray(); - for (uint32_t idx = 0; idx < elems; ++idx) { - Cursor &keyValueObj = arr.addObject(); - if (_keyWriter) { - _keyWriter->print(idx, keyValueObj); + if (_matching_elements != nullptr) { + auto &elements = _matching_elements->get_matching_elements(docId, _field_name); + auto elements_iterator = elements.cbegin(); + for (uint32_t idx = 0; idx < elems && elements_iterator != elements.cend(); ++idx) { + assert(*elements_iterator >= idx); + if (*elements_iterator == idx) { + insert_element(idx, arr); + ++elements_iterator; + } } - Cursor &obj = keyValueObj.setObject(valueName); - for (auto &valueWriter : _valueWriters) { - valueWriter->print(idx, obj); + } else { + for (uint32_t idx = 0; idx < elems; ++idx) { + insert_element(idx, arr); } } } @@ -91,8 +120,9 @@ StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime: } StructMapAttributeCombinerDFW::StructMapAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &valueFields) - : AttributeCombinerDFW(fieldName), + const std::vector<vespalib::string> &valueFields, + bool filter_elements) + : AttributeCombinerDFW(fieldName, filter_elements), _keyAttributeName(), _valueFields(valueFields), _valueAttributeNames() @@ -108,9 +138,9 @@ StructMapAttributeCombinerDFW::StructMapAttributeCombinerDFW(const vespalib::str StructMapAttributeCombinerDFW::~StructMapAttributeCombinerDFW() = default; std::unique_ptr<DocsumFieldWriterState> -StructMapAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context) +StructMapAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context, const MatchingElements* matching_elements) { - return std::make_unique<StructMapAttributeFieldWriterState>(_keyAttributeName, _valueFields, _valueAttributeNames, context); + return std::make_unique<StructMapAttributeFieldWriterState>(_keyAttributeName, _valueFields, _valueAttributeNames, context, _fieldName, matching_elements); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h index 99ad007559b..375d55df678 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h @@ -20,10 +20,11 @@ class StructMapAttributeCombinerDFW : public AttributeCombinerDFW std::vector<vespalib::string> _valueFields; std::vector<vespalib::string> _valueAttributeNames; - std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) override; + std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: StructMapAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &valueFields); + const std::vector<vespalib::string> &valueFields, + bool filter_elements); ~StructMapAttributeCombinerDFW() override; }; diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp index f87c327e86a..8a480419aa5 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -2,12 +2,14 @@ #include "vsm-adapter.h" #include "docsumconfig.h" +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.vsm-adapter"); using search::docsummary::ResConfigEntry; using search::docsummary::KeywordExtractor; +using search::MatchingElements; using config::ConfigSnapshot; namespace vsm { @@ -45,6 +47,11 @@ void GetDocsumsStateCallback::FillDocumentLocations(GetDocsumsState *state, IDoc (void) env; } +std::unique_ptr<MatchingElements> +GetDocsumsStateCallback::fill_matching_elements() +{ + return std::make_unique<MatchingElements>(); +} GetDocsumsStateCallback::~GetDocsumsStateCallback() = default; diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h index 2bdd0248be9..2a5b1e1d47c 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h @@ -39,6 +39,7 @@ public: void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override; void ParseLocation(GetDocsumsState * state) override; virtual void FillDocumentLocations(GetDocsumsState * state, IDocsumEnvironment * env); + virtual std::unique_ptr<search::MatchingElements> fill_matching_elements() override; void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } void setRankFeatures(const search::FeatureSet::SP & rf) { _rankFeatures = rf; } ~GetDocsumsStateCallback(); |