diff options
16 files changed, 213 insertions, 49 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index edb91787214..f3559b54eb4 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/queryeval/begin_and_end_id.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/transport.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/stringfmt.h> @@ -146,7 +147,8 @@ DocsumContext::DocsumContext(const DocsumRequest & request, IDocsumWriter & docs _attrCtx(attrCtx), _attrMgr(attrMgr), _docsumState(*this), - _sessionMgr(sessionMgr) + _sessionMgr(sessionMgr), + _matching_elements() { initState(); } @@ -213,4 +215,11 @@ DocsumContext::ParseLocation(search::docsummary::GetDocsumsState *state) state->_parsedLocation.reset(getLocation(_request.location, _attrMgr)); } +const MatchingElements& +DocsumContext::fill_matching_elements() +{ + _matching_elements = std::make_unique<MatchingElements>(); + return *_matching_elements; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h index 45f3ca8e44f..ec03efdd650 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.h @@ -25,6 +25,7 @@ private: search::IAttributeManager & _attrMgr; search::docsummary::GetDocsumsState _docsumState; matching::SessionManager & _sessionMgr; + std::unique_ptr<search::MatchingElements> _matching_elements; void initState(); search::engine::DocsumReply::UP createReply(); @@ -48,6 +49,7 @@ public: void FillSummaryFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; void FillRankFeatures(search::docsummary::GetDocsumsState * state, search::docsummary::IDocsumEnvironment * env) override; void ParseLocation(search::docsummary::GetDocsumsState * state) override; + const search::MatchingElements& fill_matching_elements() override; }; } // namespace proton diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 01b458daf5d..240e7334292 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_field_writer_state.h> @@ -23,6 +24,7 @@ using search::AttributeManager; using search::AttributeVector; using search::IntegerAttribute; using search::FloatingPointAttribute; +using search::MatchingElements; using search::StringAttribute; using search::attribute::BasicType; using search::attribute::CollectionType; @@ -157,12 +159,30 @@ AttributeManagerFixture::buildIntegerAttribute(const vespalib::string &name, class DummyStateCallback : public GetDocsumsStateCallback { public: + MatchingElements _matching_elements; + + DummyStateCallback(); void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void ParseLocation(GetDocsumsState *) override { } + const MatchingElements& fill_matching_elements() override { return _matching_elements; } ~DummyStateCallback() override { } }; +DummyStateCallback::DummyStateCallback() + : GetDocsumsStateCallback(), + _matching_elements() +{ + _matching_elements.add_matching_elements(1, "array", {1}); + _matching_elements.add_matching_elements(3, "array", {0}); + _matching_elements.add_matching_elements(4, "array", {1}); + _matching_elements.add_matching_elements(1, "smap", {1}); + _matching_elements.add_matching_elements(3, "smap", {0}); + _matching_elements.add_matching_elements(4, "smap", {1}); + _matching_elements.add_matching_elements(1, "map", {1}); + _matching_elements.add_matching_elements(3, "map", {0}); + _matching_elements.add_matching_elements(4, "map", {1}); +} struct AttributeCombinerTest : public ::testing::Test { @@ -173,7 +193,7 @@ struct AttributeCombinerTest : public ::testing::Test AttributeCombinerTest(); ~AttributeCombinerTest(); - void set_field_name(const vespalib::string &field_name); + void set_field(const vespalib::string &field_name, bool filter_elements); void assertWritten(const vespalib::string &exp, uint32_t docId); }; @@ -189,9 +209,9 @@ AttributeCombinerTest::AttributeCombinerTest() AttributeCombinerTest::~AttributeCombinerTest() = default; void -AttributeCombinerTest::set_field_name(const vespalib::string &field_name) +AttributeCombinerTest::set_field(const vespalib::string &field_name, bool filter_elements) { - writer = AttributeCombinerDFW::create(field_name, attrs.mgr); + writer = AttributeCombinerDFW::create(field_name, attrs.mgr, filter_elements); EXPECT_TRUE(writer->setFieldWriterStateIndex(0)); state._fieldWriterStates.resize(1); } @@ -219,7 +239,7 @@ AttributeCombinerTest::assertWritten(const vespalib::string &expectedJson, uint3 TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_array_of_struct) { - set_field_name("array"); + set_field("array", false); assertWritten("[ { fval: 110.0, name: \"n1.1\", val: 10}, { name: \"n1.2\", val: 11}]", 1); assertWritten("[ { fval: 120.0, name: \"n2\", val: 20}, { fval: 121.0, val: 21 }]", 2); assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30}, { fval: 131.0, name: \"n3.2\"} ]", 3); @@ -229,7 +249,7 @@ TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_corr TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_struct) { - set_field_name("smap"); + set_field("smap", false); assertWritten("[ { key: \"k1.1\", value: { fval: 110.0, name: \"n1.1\", val: 10} }, { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); assertWritten("[ { key: \"k2\", value: { fval: 120.0, name: \"n2\", val: 20} }, { value: { fval: 121.0, val: 21 } }]", 2); assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} }, { key: \"k3.2\", value: { fval: 131.0, name: \"n3.2\"} } ]", 3); @@ -239,7 +259,7 @@ TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_corr TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_map_of_string) { - set_field_name("map"); + set_field("map", false); assertWritten("[ { key: \"k1.1\", value: \"n1.1\" }, { key: \"k1.2\", value: \"n1.2\"}]", 1); assertWritten("[ { key: \"k2\"}]", 2); assertWritten("[ { key: \"k3.1\", value: \"n3.1\" }, { value: \"n3.2\"} ]", 3); @@ -247,6 +267,36 @@ TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_corr assertWritten("null", 5); } +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_array_of_struct) +{ + set_field("array", true); + assertWritten("[ { name: \"n1.2\", val: 11}]", 1); + assertWritten("[ ]", 2); + assertWritten("[ { fval: 130.0, name: \"n3.1\", val: 30} ]", 3); + assertWritten("[ { fval: 141.0, name: \"n4.2\", val: 41} ]", 4); + assertWritten("null", 5); +} + +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_struct) +{ + set_field("smap", true); + assertWritten("[ { key: \"k1.2\", value: { name: \"n1.2\", val: 11} }]", 1); + assertWritten("[ ]", 2); + assertWritten("[ { key: \"k3.1\", value: { fval: 130.0, name: \"n3.1\", val: 30} } ]", 3); + assertWritten("[ { key: \"k4.2\", value: { fval: 141.0, name: \"n4.2\", val: 41} } ]", 4); + assertWritten("null", 5); +} + +TEST_F(AttributeCombinerTest, require_that_attribute_combiner_dfw_generates_correct_slime_output_for_filtered_map_of_string) +{ + set_field("map", true); + assertWritten("[ { key: \"k1.2\", value: \"n1.2\"}]", 1); + assertWritten("[ ]", 2); + assertWritten("[ { key: \"k3.1\", value: \"n3.1\" } ]", 3); + assertWritten("[ { key: \"k4.2\", value: \"n4.2\" } ]", 4); + assertWritten("null", 5); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index 764ff4723cb..86a2e9ae76f 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/docsumfieldwriter.h> #include <vespa/searchsummary/docsummary/positionsdfw.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> @@ -17,6 +18,7 @@ LOG_SETUP("positionsdfw_test"); using search::RawBuf; using search::IAttributeManager; +using search::MatchingElements; using search::SingleInt64ExtAttribute; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -104,6 +106,7 @@ struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { virtual void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} virtual void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override {} virtual void ParseLocation(GetDocsumsState *) override {} + const MatchingElements& fill_matching_elements() override { abort(); } }; template <typename AttrType> diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index 5d6565ceadb..bd753f24bf3 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/docsumstate.h> @@ -9,6 +10,7 @@ using namespace vespalib::slime::convenience; using namespace search::docsummary; +using search::MatchingElements; namespace { @@ -77,6 +79,7 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { void FillSummaryFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void FillRankFeatures(GetDocsumsState *, IDocsumEnvironment *) override { } void ParseLocation(GetDocsumsState *) override { } + const MatchingElements& fill_matching_elements() override { abort(); } }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp index 6a643a74458..9d4d3829047 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -5,7 +5,9 @@ #include "attribute_field_writer.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <cassert> using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -18,20 +20,29 @@ namespace { class ArrayAttributeFieldWriterState : public DocsumFieldWriterState { std::vector<std::unique_ptr<AttributeFieldWriter>> _writers; + const vespalib::string& _field_name; + const MatchingElements* const _matching_elements; public: ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames, const std::vector<vespalib::string> &attributeNames, - IAttributeContext &context); + IAttributeContext &context, + const vespalib::string &field_name, + const MatchingElements* matching_elements); ~ArrayAttributeFieldWriterState() override; + void insert_element(uint32_t element_index, Cursor &array); void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; }; ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector<vespalib::string> &fieldNames, const std::vector<vespalib::string> &attributeNames, - IAttributeContext &context) + IAttributeContext &context, + const vespalib::string &field_name, + const MatchingElements *matching_elements) : DocsumFieldWriterState(), - _writers() + _writers(), + _field_name(field_name), + _matching_elements(matching_elements) { size_t fields = fieldNames.size(); _writers.reserve(fields); @@ -46,6 +57,15 @@ ArrayAttributeFieldWriterState::ArrayAttributeFieldWriterState(const std::vector ArrayAttributeFieldWriterState::~ArrayAttributeFieldWriterState() = default; void +ArrayAttributeFieldWriterState::insert_element(uint32_t element_index, Cursor &array) +{ + Cursor &obj = array.addObject(); + for (auto &writer : _writers) { + writer->print(element_index, obj); + } +} + +void ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Inserter &target) { uint32_t elems = 0; @@ -59,10 +79,19 @@ ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Ins return; } Cursor &arr = target.insertArray(); - for (uint32_t idx = 0; idx < elems; ++idx) { - Cursor &obj = arr.addObject(); - for (auto &writer : _writers) { - writer->print(idx, obj); + if (_matching_elements != nullptr) { + auto &elements = _matching_elements->get_matching_elements(docId, _field_name); + auto elements_iterator = elements.cbegin(); + for (uint32_t idx = 0; idx < elems && elements_iterator != elements.cend(); ++idx) { + assert(*elements_iterator >= idx); + if (*elements_iterator == idx) { + insert_element(idx, arr); + ++elements_iterator; + } + } + } else { + for (uint32_t idx = 0; idx < elems; ++idx) { + insert_element(idx, arr); } } } @@ -70,8 +99,9 @@ ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Ins } ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &fields) - : AttributeCombinerDFW(fieldName), + const std::vector<vespalib::string> &fields, + bool filter_elements) + : AttributeCombinerDFW(fieldName, filter_elements), _fields(fields), _attributeNames() { @@ -85,9 +115,9 @@ ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fie ArrayAttributeCombinerDFW::~ArrayAttributeCombinerDFW() = default; std::unique_ptr<DocsumFieldWriterState> -ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context) +ArrayAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context, const MatchingElements* matching_elements) { - return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context); + return std::make_unique<ArrayAttributeFieldWriterState>(_fields, _attributeNames, context, _fieldName, matching_elements); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h index c02d2bd5da6..15c3030c782 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -19,10 +19,11 @@ class ArrayAttributeCombinerDFW : public AttributeCombinerDFW std::vector<vespalib::string> _fields; std::vector<vespalib::string> _attributeNames; - std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) override; + std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: ArrayAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &fields); + const std::vector<vespalib::string> &fields, + bool filter_elements); ~ArrayAttributeCombinerDFW() override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index ef621dc5b4a..ed053ce0115 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -89,9 +89,10 @@ StructFields::~StructFields() = default; } -AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName) +AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements) : ISimpleDFW(), _stateIndex(0), + _filter_elements(filter_elements), _fieldName(fieldName) { } @@ -112,15 +113,15 @@ AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) } std::unique_ptr<IDocsumFieldWriter> -AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr) +AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr, bool filter_elements) { StructFields structFields(fieldName, attrMgr); if (structFields.getError()) { return std::unique_ptr<IDocsumFieldWriter>(); } else if (!structFields.getMapFields().empty()) { - return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields.getMapFields()); + return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields.getMapFields(), filter_elements); } - return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields.getArrayFields()); + return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields.getArrayFields(), filter_elements); } void @@ -128,7 +129,11 @@ AttributeCombinerDFW::insertField(uint32_t docid, GetDocsumsState *state, ResTyp { auto &fieldWriterState = state->_fieldWriterStates[_stateIndex]; if (!fieldWriterState) { - fieldWriterState = allocFieldWriterState(*state->_attrCtx); + const MatchingElements *matching_elements = nullptr; + if (_filter_elements) { + matching_elements = &state->get_matching_elements(); + } + fieldWriterState = allocFieldWriterState(*state->_attrCtx, matching_elements); } fieldWriterState->insertField(docid, target); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index 6c0991fdf92..ec806ba0ac6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -4,6 +4,7 @@ #include "docsumfieldwriter.h" +namespace search { class MatchingElements; } namespace search::attribute { class IAttributeContext; } namespace search::docsummary { @@ -19,15 +20,16 @@ class AttributeCombinerDFW : public ISimpleDFW { protected: uint32_t _stateIndex; + const bool _filter_elements; vespalib::string _fieldName; - AttributeCombinerDFW(const vespalib::string &fieldName); + AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements); protected: - virtual std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) = 0; + virtual std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) = 0; public: ~AttributeCombinerDFW() override; bool IsGenerated() const override; bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; - static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr); + static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr, bool filter_elements); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index b95591546b5..f6377563d90 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -94,7 +94,7 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & } } else if (overrideName == "attributecombiner") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { - fieldWriter = AttributeCombinerDFW::create(fieldName, *getEnvironment()->getAttributeManager()); + fieldWriter = AttributeCombinerDFW::create(fieldName, *getEnvironment()->getAttributeManager(), false); rc = static_cast<bool>(fieldWriter); } } else { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index b0431b6e6ac..d78c908fa6a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -4,10 +4,10 @@ #include <vespa/juniper/rpinterface.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/location.h> +#include <vespa/searchlib/common/matching_elements.h> #include "docsum_field_writer_state.h" -namespace search { -namespace docsummary { +namespace search::docsummary { GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) : _args(), @@ -25,7 +25,8 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _parsedLocation(), _summaryFeatures(NULL), _summaryFeaturesCached(false), - _rankFeatures(NULL) + _rankFeatures(NULL), + _matching_elements(nullptr) { _dynteaser._docid = static_cast<uint32_t>(-1); _dynteaser._input = static_cast<uint32_t>(-1); @@ -48,5 +49,13 @@ GetDocsumsState::~GetDocsumsState() } } +const MatchingElements & +GetDocsumsState::get_matching_elements() +{ + if (_matching_elements == nullptr) { + _matching_elements = &_callback.fill_matching_elements(); + } + return *_matching_elements; } + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index fa47d5244eb..394913d3ad9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -13,6 +13,7 @@ namespace juniper { class Result; } +namespace search { class MatchingElements; } namespace search::common { class Location; } namespace search::attribute { class IAttributeContext; @@ -31,6 +32,7 @@ public: virtual void FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; virtual void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) = 0; virtual void ParseLocation(GetDocsumsState * state) = 0; + virtual const MatchingElements& fill_matching_elements() = 0; virtual ~GetDocsumsStateCallback(void) { } GetDocsumsStateCallback(const GetDocsumsStateCallback &) = delete; GetDocsumsStateCallback & operator = (const GetDocsumsStateCallback &) = delete; @@ -84,10 +86,14 @@ public: // used by RankFeaturesDFW FeatureSet::SP _rankFeatures; + // Used by AttributeCombinerDFW when filtering is enabled + const search::MatchingElements* _matching_elements; + GetDocsumsState(const GetDocsumsState &) = delete; GetDocsumsState& operator=(const GetDocsumsState &) = delete; GetDocsumsState(GetDocsumsStateCallback &callback); ~GetDocsumsState(); + const MatchingElements &get_matching_elements(); }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp index e4e30afbc4d..e34012de7de 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp @@ -5,7 +5,9 @@ #include "attribute_field_writer.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/vespalib/data/slime/cursor.h> +#include <cassert> using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; @@ -22,23 +24,32 @@ class StructMapAttributeFieldWriterState : public DocsumFieldWriterState { std::unique_ptr<AttributeFieldWriter> _keyWriter; std::vector<std::unique_ptr<AttributeFieldWriter>> _valueWriters; + const vespalib::string& _field_name; + const MatchingElements* const _matching_elements; public: StructMapAttributeFieldWriterState(const vespalib::string &keyAttributeName, const std::vector<vespalib::string> &valueFieldNames, const std::vector<vespalib::string> &valueAttributeNames, - IAttributeContext &context); + IAttributeContext &context, + const vespalib::string &field_name, + const MatchingElements* matching_elements); ~StructMapAttributeFieldWriterState() override; + void insert_element(uint32_t element_index, Cursor &array); void insertField(uint32_t docId, vespalib::slime::Inserter &target) override; }; StructMapAttributeFieldWriterState::StructMapAttributeFieldWriterState(const vespalib::string &keyAttributeName, const std::vector<vespalib::string> &valueFieldNames, const std::vector<vespalib::string> &valueAttributeNames, - IAttributeContext &context) + IAttributeContext &context, + const vespalib::string& field_name, + const MatchingElements *matching_elements) : DocsumFieldWriterState(), _keyWriter(), - _valueWriters() + _valueWriters(), + _field_name(field_name), + _matching_elements(matching_elements) { const IAttributeVector *attr = context.getAttribute(keyAttributeName); if (attr != nullptr) { @@ -57,6 +68,19 @@ StructMapAttributeFieldWriterState::StructMapAttributeFieldWriterState(const ves StructMapAttributeFieldWriterState::~StructMapAttributeFieldWriterState() = default; void +StructMapAttributeFieldWriterState::insert_element(uint32_t element_index, Cursor &array) +{ + Cursor &keyValueObj = array.addObject(); + if (_keyWriter) { + _keyWriter->print(element_index, keyValueObj); + } + Cursor &obj = keyValueObj.setObject(valueName); + for (auto &valueWriter : _valueWriters) { + valueWriter->print(element_index, obj); + } +} + +void StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Inserter &target) { uint32_t elems = 0; @@ -76,14 +100,19 @@ StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime: return; } Cursor &arr = target.insertArray(); - for (uint32_t idx = 0; idx < elems; ++idx) { - Cursor &keyValueObj = arr.addObject(); - if (_keyWriter) { - _keyWriter->print(idx, keyValueObj); + if (_matching_elements != nullptr) { + auto &elements = _matching_elements->get_matching_elements(docId, _field_name); + auto elements_iterator = elements.cbegin(); + for (uint32_t idx = 0; idx < elems && elements_iterator != elements.cend(); ++idx) { + assert(*elements_iterator >= idx); + if (*elements_iterator == idx) { + insert_element(idx, arr); + ++elements_iterator; + } } - Cursor &obj = keyValueObj.setObject(valueName); - for (auto &valueWriter : _valueWriters) { - valueWriter->print(idx, obj); + } else { + for (uint32_t idx = 0; idx < elems; ++idx) { + insert_element(idx, arr); } } } @@ -91,8 +120,9 @@ StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime: } StructMapAttributeCombinerDFW::StructMapAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &valueFields) - : AttributeCombinerDFW(fieldName), + const std::vector<vespalib::string> &valueFields, + bool filter_elements) + : AttributeCombinerDFW(fieldName, filter_elements), _keyAttributeName(), _valueFields(valueFields), _valueAttributeNames() @@ -108,9 +138,9 @@ StructMapAttributeCombinerDFW::StructMapAttributeCombinerDFW(const vespalib::str StructMapAttributeCombinerDFW::~StructMapAttributeCombinerDFW() = default; std::unique_ptr<DocsumFieldWriterState> -StructMapAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context) +StructMapAttributeCombinerDFW::allocFieldWriterState(IAttributeContext &context, const MatchingElements* matching_elements) { - return std::make_unique<StructMapAttributeFieldWriterState>(_keyAttributeName, _valueFields, _valueAttributeNames, context); + return std::make_unique<StructMapAttributeFieldWriterState>(_keyAttributeName, _valueFields, _valueAttributeNames, context, _fieldName, matching_elements); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h index 99ad007559b..375d55df678 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h @@ -20,10 +20,11 @@ class StructMapAttributeCombinerDFW : public AttributeCombinerDFW std::vector<vespalib::string> _valueFields; std::vector<vespalib::string> _valueAttributeNames; - std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context) override; + std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: StructMapAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &valueFields); + const std::vector<vespalib::string> &valueFields, + bool filter_elements); ~StructMapAttributeCombinerDFW() override; }; diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp index f87c327e86a..6f9e58f8718 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -2,19 +2,22 @@ #include "vsm-adapter.h" #include "docsumconfig.h" +#include <vespa/searchlib/common/matching_elements.h> #include <vespa/log/log.h> LOG_SETUP(".vsm.vsm-adapter"); using search::docsummary::ResConfigEntry; using search::docsummary::KeywordExtractor; +using search::MatchingElements; using config::ConfigSnapshot; namespace vsm { GetDocsumsStateCallback::GetDocsumsStateCallback() : _summaryFeatures(), - _rankFeatures() + _rankFeatures(), + _matching_elements() { } void GetDocsumsStateCallback::FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env) @@ -45,6 +48,14 @@ void GetDocsumsStateCallback::FillDocumentLocations(GetDocsumsState *state, IDoc (void) env; } +const MatchingElements& +GetDocsumsStateCallback::fill_matching_elements() +{ + if (!_matching_elements) { + _matching_elements = std::make_unique<MatchingElements>(); + } + return *_matching_elements; +} GetDocsumsStateCallback::~GetDocsumsStateCallback() = default; diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h index 2bdd0248be9..aaa73f4ba24 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h @@ -32,6 +32,7 @@ class GetDocsumsStateCallback : public search::docsummary::GetDocsumsStateCallba private: search::FeatureSet::SP _summaryFeatures; search::FeatureSet::SP _rankFeatures; + std::unique_ptr<search::MatchingElements> _matching_elements; public: GetDocsumsStateCallback(); @@ -39,6 +40,7 @@ public: void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override; void ParseLocation(GetDocsumsState * state) override; virtual void FillDocumentLocations(GetDocsumsState * state, IDocsumEnvironment * env); + virtual const search::MatchingElements& fill_matching_elements() override; void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; } void setRankFeatures(const search::FeatureSet::SP & rf) { _rankFeatures = rf; } ~GetDocsumsStateCallback(); |