diff options
14 files changed, 309 insertions, 159 deletions
diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 5d122fa1042..c856c5c805e 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -217,7 +217,7 @@ AttributeCombinerTest::set_field(const vespalib::string &field_name, bool filter if (filter_elements) { _struct_field_mapper = std::make_shared<search::StructFieldMapper>(); } - writer = AttributeCombinerDFW::create(field_name, attrs.mgr, filter_elements, _struct_field_mapper); + writer = AttributeCombinerDFW::create(field_name, *state._attrCtx, filter_elements, _struct_field_mapper); EXPECT_TRUE(writer->setFieldWriterStateIndex(0)); state._fieldWriterStates.resize(1); } diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 40d0285b1ec..bbcab709e42 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -3,7 +3,11 @@ #include <vespa/document/datatype/datatype.h> #include <vespa/document/datatype/structdatatype.h> #include <vespa/document/document.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchlib/common/struct_field_mapper.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/idocsumenvironment.h> @@ -19,8 +23,15 @@ #include <vespa/log/log.h> LOG_SETUP("matched_elements_filter_test"); +using search::AttributeFactory; +using search::AttributeVector; using search::MatchingElements; using search::StructFieldMapper; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; +using search::attribute::IAttributeContext; +using search::attribute::IAttributeVector; using vespalib::Slime; using namespace document; @@ -38,12 +49,6 @@ struct SlimeValue { size_t used = JsonFormat::decode(json_input, slime); EXPECT_GT(used, 0); } - SlimeValue(const Slime& slime_with_raw_field) - : slime() - { - size_t used = BinaryFormat::decode(slime_with_raw_field.get().asString(), slime); - EXPECT_GT(used, 0); - } }; StructDataType::UP @@ -93,8 +98,10 @@ public: auto* result_class = _config.AddResultClass("test", class_id); EXPECT_TRUE(result_class->AddConfigEntry("array", ResType::RES_JSONSTRING)); EXPECT_TRUE(result_class->AddConfigEntry("map", ResType::RES_JSONSTRING)); + EXPECT_TRUE(result_class->AddConfigEntry("map2", ResType::RES_JSONSTRING)); _config.CreateEnumMaps(); } + ~DocsumStore() {} const ResultConfig& get_config() const { return _config; } const ResultClass* get_class() const { return _config.LookupResultClass(class_id); } search::docsummary::DocsumStoreValue getMappedDocsum() { @@ -113,6 +120,11 @@ public: map_value.put(StringFieldValue("c"), *make_elem_value("c", 7)); write_field_value(map_value); } + { + MapFieldValue map2_value(_map_type); + map2_value.put(StringFieldValue("dummy"), *make_elem_value("dummy", 2)); + write_field_value(map2_value); + } const char* buf; uint32_t buf_len; assert(_packer.GetDocsumBlob(&buf, &buf_len)); @@ -120,6 +132,29 @@ public: } }; +class AttributeContext : public IAttributeContext { +private: + AttributeVector::SP _map_value_name; + AttributeVector::SP _map2_key; + AttributeVector::SP _array_weight; +public: + AttributeContext() + : _map_value_name(AttributeFactory::createAttribute("map.value.name", Config(BasicType::STRING, CollectionType::ARRAY))), + _map2_key(AttributeFactory::createAttribute("map2.key", Config(BasicType::STRING, CollectionType::ARRAY))), + _array_weight(AttributeFactory::createAttribute("array.weight", Config(BasicType::INT32, CollectionType::ARRAY))) + {} + ~AttributeContext() {} + const IAttributeVector* getAttribute(const string&) const override { abort(); } + const IAttributeVector* getAttributeStableEnum(const string&) const override { abort(); } + void getAttributeList(std::vector<const IAttributeVector*>& list) const override { + list.push_back(_map_value_name.get()); + list.push_back(_map2_key.get()); + list.push_back(_array_weight.get()); + } + void releaseEnumGuards() override { abort(); } + void asyncForAttribute(const vespalib::string&, std::unique_ptr<search::attribute::IAttributeFunctor>) const override { abort(); } +}; + class StateCallback : public GetDocsumsStateCallback { private: std::string _field_name; @@ -144,34 +179,43 @@ public: class MatchedElementsFilterTest : public ::testing::Test { private: - DocsumStore _store; - - SlimeValue run_filter_field_writer(const std::string& input_field_name, const ElementVector& matching_elements) { - int input_field_enum = _store.get_config().GetFieldNameEnum().Lookup(input_field_name.c_str()); - EXPECT_GE(input_field_enum, 0); - MatchedElementsFilterDFW filter(input_field_name, input_field_enum); + DocsumStore _doc_store; + AttributeContext _attr_ctx; + std::shared_ptr<StructFieldMapper> _mapper; - GeneralResult result(_store.get_class()); - result.inplaceUnpack(_store.getMappedDocsum()); + Slime run_filter_field_writer(const std::string& input_field_name, const ElementVector& matching_elements) { + auto writer = make_field_writer(input_field_name); + GeneralResult result(_doc_store.get_class()); + result.inplaceUnpack(_doc_store.getMappedDocsum()); StateCallback callback(input_field_name, matching_elements); GetDocsumsState state(callback); Slime slime; SlimeInserter inserter(slime); - filter.insertField(doc_id, &result, &state, ResType::RES_JSONSTRING, inserter); - return SlimeValue(slime); + writer->insertField(doc_id, &result, &state, ResType::RES_JSONSTRING, inserter); + return slime; } public: MatchedElementsFilterTest() - : _store() + : _doc_store(), + _attr_ctx(), + _mapper(std::make_shared<StructFieldMapper>()) { } + ~MatchedElementsFilterTest() {} + std::unique_ptr<IDocsumFieldWriter> make_field_writer(const std::string& input_field_name) { + int input_field_enum = _doc_store.get_config().GetFieldNameEnum().Lookup(input_field_name.c_str()); + EXPECT_GE(input_field_enum, 0); + return MatchedElementsFilterDFW::create(input_field_name, input_field_enum, + _attr_ctx, _mapper); + } void expect_filtered(const std::string& input_field_name, const ElementVector& matching_elements, const std::string& exp_slime_as_json) { - SlimeValue act = run_filter_field_writer(input_field_name, matching_elements); + Slime act = run_filter_field_writer(input_field_name, matching_elements); SlimeValue exp(exp_slime_as_json); - EXPECT_EQ(exp.slime, act.slime); + EXPECT_EQ(exp.slime, act); } + const StructFieldMapper& mapper() const { return *_mapper; } }; TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value) @@ -185,6 +229,14 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value) "{'name':'c','weight':7}]"); } +TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_array_field_value) +{ + auto writer = make_field_writer("array"); + EXPECT_TRUE(mapper().is_struct_field("array")); + EXPECT_EQ("", mapper().get_struct_field("array.name")); + EXPECT_EQ("array", mapper().get_struct_field("array.weight")); +} + TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value) { expect_filtered("map", {}, "[]"); @@ -196,10 +248,28 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value) "{'key':'c','value':{'name':'c','weight':7}}]"); } +TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_map_field_value) +{ + { + auto writer = make_field_writer("map"); + EXPECT_TRUE(mapper().is_struct_field("map")); + EXPECT_EQ("", mapper().get_struct_field("map.key")); + EXPECT_EQ("map", mapper().get_struct_field("map.value.name")); + EXPECT_EQ("", mapper().get_struct_field("map.value.weight")); + } + { + auto writer = make_field_writer("map2"); + EXPECT_TRUE(mapper().is_struct_field("map2")); + EXPECT_EQ("map2", mapper().get_struct_field("map2.key")); + EXPECT_EQ("", mapper().get_struct_field("map2.value.name")); + EXPECT_EQ("", mapper().get_struct_field("map2.value.weight")); + } +} + TEST_F(MatchedElementsFilterTest, field_writer_is_not_generated_as_it_depends_on_data_from_document_store) { - MatchedElementsFilterDFW filter("foo", 0); - EXPECT_FALSE(filter.IsGenerated()); + auto writer = make_field_writer("array"); + EXPECT_FALSE(writer->IsGenerated()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index fb6a399e71c..6d846a47d93 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -22,6 +22,7 @@ vespa_add_library(searchsummary_docsummary OBJECT resultconfig.cpp resultpacker.cpp searchdatatype.cpp + struct_fields_resolver.cpp struct_map_attribute_combiner_dfw.cpp summaryfeaturesdfw.cpp summaryfieldconverter.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp index 88be8aac8c5..a1f3ce4d392 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.cpp @@ -1,8 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "array_attribute_combiner_dfw.h" -#include "docsum_field_writer_state.h" #include "attribute_field_writer.h" +#include "docsum_field_writer_state.h" +#include "struct_fields_resolver.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchlib/common/matching_elements.h> @@ -100,22 +101,15 @@ ArrayAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime::Ins } ArrayAttributeCombinerDFW::ArrayAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &fields, + const StructFieldsResolver& fields_resolver, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper) : AttributeCombinerDFW(fieldName, filter_elements, std::move(struct_field_mapper)), - _fields(fields), - _attributeNames() + _fields(fields_resolver.get_array_fields()), + _attributeNames(fields_resolver.get_array_attributes()) { - _attributeNames.reserve(_fields.size()); - vespalib::string prefix = fieldName + "."; - for (const auto &field : _fields) { - _attributeNames.emplace_back(prefix + field); - } if (filter_elements && _struct_field_mapper && !_struct_field_mapper->is_struct_field(fieldName)) { - for (const auto &sub_field : _attributeNames) { - _struct_field_mapper->add_mapping(fieldName, sub_field); - } + fields_resolver.apply_to(*_struct_field_mapper); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h index 473bdbbc4e7..c3e686965cf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/array_attribute_combiner_dfw.h @@ -9,10 +9,12 @@ namespace search::attribute { class IAttributeContext; } namespace search::docsummary { class DocsumFieldWriterState; +class StructFieldsResolver; -/* - * This class reads values from multiple struct field attributes and - * inserts them as an array of struct. +/** + * This class reads values from multiple struct field attributes and inserts them as an array of struct. + * + * Used to write both array of struct fields and map of primitives fields. */ class ArrayAttributeCombinerDFW : public AttributeCombinerDFW { @@ -22,7 +24,7 @@ class ArrayAttributeCombinerDFW : public AttributeCombinerDFW std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: ArrayAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &fields, + const StructFieldsResolver& fields_resolver, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper); ~ArrayAttributeCombinerDFW() override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index 69296509a85..8eb77c0ed9c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -1,95 +1,21 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "attribute_combiner_dfw.h" #include "array_attribute_combiner_dfw.h" -#include "struct_map_attribute_combiner_dfw.h" +#include "attribute_combiner_dfw.h" #include "docsum_field_writer_state.h" #include "docsumstate.h" -#include <vespa/searchlib/attribute/attributeguard.h> -#include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/iattributemanager.h> +#include "struct_fields_resolver.h" +#include "struct_map_attribute_combiner_dfw.h" #include <vespa/searchlib/common/struct_field_mapper.h> #include <algorithm> #include <vespa/log/log.h> LOG_SETUP(".searchsummary.docsummary.attribute_combiner_dfw"); -using search::AttributeGuard; -using search::AttributeVector; -using search::attribute::CollectionType; +using search::attribute::IAttributeContext; namespace search::docsummary { -namespace { - -class StructFields -{ - std::vector<vespalib::string> _mapFields; - std::vector<vespalib::string> _arrayFields; - bool _hasMapKey; - bool _error; - -public: - StructFields(const vespalib::string &fieldName, const IAttributeManager &attrMgr); - ~StructFields(); - const std::vector<vespalib::string> &getMapFields() const { return _mapFields; } - const std::vector<vespalib::string> &getArrayFields() const { return _arrayFields; } - bool hasMapKey() const { return _hasMapKey; } - bool getError() const { return _error; } -}; - - -StructFields::StructFields(const vespalib::string &fieldName, const IAttributeManager &attrMgr) - : _mapFields(), - _arrayFields(), - _hasMapKey(false), - _error(false) -{ - std::vector<const search::attribute::IAttributeVector *> attrs; - auto attrCtx = attrMgr.createContext(); - attrCtx->getAttributeList(attrs); - vespalib::string prefix = fieldName + "."; - vespalib::string keyName = prefix + "key"; - vespalib::string valuePrefix = prefix + "value."; - for (const auto attr : attrs) { - vespalib::string name = attr->getName(); - if (name.substr(0, prefix.size()) != prefix) { - continue; - } - auto collType = attr->getCollectionType(); - if (collType != CollectionType::Type::ARRAY) { - LOG(warning, "Attribute %s is not an array attribute", name.c_str()); - _error = true; - break; - } - if (name.substr(0, valuePrefix.size()) == valuePrefix) { - _mapFields.emplace_back(name.substr(valuePrefix.size())); - } else { - _arrayFields.emplace_back(name.substr(prefix.size())); - if (name == keyName) { - _hasMapKey = true; - } - } - } - if (!_error) { - std::sort(_arrayFields.begin(), _arrayFields.end()); - std::sort(_mapFields.begin(), _mapFields.end()); - if (!_mapFields.empty()) { - if (!_hasMapKey) { - LOG(warning, "Missing key attribute '%s', have value attributes for map", keyName.c_str()); - _error = true; - } else if (_arrayFields.size() != 1u) { - LOG(warning, "Could not determine if field '%s' is array or map of struct", fieldName.c_str()); - _error = true; - } - } - } -} - -StructFields::~StructFields() = default; - -} - AttributeCombinerDFW::AttributeCombinerDFW(const vespalib::string &fieldName, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper) : ISimpleDFW(), _stateIndex(0), @@ -115,15 +41,15 @@ AttributeCombinerDFW::setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) } std::unique_ptr<IDocsumFieldWriter> -AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeManager &attrMgr, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper) +AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeContext &attrCtx, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper) { - StructFields structFields(fieldName, attrMgr); - if (structFields.getError()) { + StructFieldsResolver structFields(fieldName, attrCtx, true); + if (structFields.has_error()) { return std::unique_ptr<IDocsumFieldWriter>(); - } else if (!structFields.getMapFields().empty()) { - return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields.getMapFields(), filter_elements, std::move(struct_field_mapper)); + } else if (structFields.is_map_of_struct()) { + return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(struct_field_mapper)); } - return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields.getArrayFields(), filter_elements, std::move(struct_field_mapper)); + return std::make_unique<ArrayAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(struct_field_mapper)); } void diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index 3b7ab911956..a8ab5f2f8f5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -33,7 +33,7 @@ public: ~AttributeCombinerDFW() override; bool IsGenerated() const override; bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; - static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, IAttributeManager &attrMgr, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper); + static std::unique_ptr<IDocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper); void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp index 59a1a67a890..9235ce2b181 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumconfig.cpp @@ -1,14 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "attribute_combiner_dfw.h" #include "docsumconfig.h" #include "docsumwriter.h" +#include "geoposdfw.h" #include "idocsumenvironment.h" +#include "juniperdfw.h" +#include "matched_elements_filter_dfw.h" +#include "positionsdfw.h" #include "rankfeaturesdfw.h" #include "textextractordfw.h" -#include "geoposdfw.h" -#include "positionsdfw.h" -#include "juniperdfw.h" -#include "attribute_combiner_dfw.h" #include <vespa/searchlib/common/struct_field_mapper.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> @@ -95,13 +96,23 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & } } else if (overrideName == "attributecombiner") { if (getEnvironment() && getEnvironment()->getAttributeManager()) { - fieldWriter = AttributeCombinerDFW::create(fieldName, *getEnvironment()->getAttributeManager(), false, std::shared_ptr<StructFieldMapper>()); + auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); + fieldWriter = AttributeCombinerDFW::create(fieldName, *attr_ctx, false, std::shared_ptr<StructFieldMapper>()); rc = static_cast<bool>(fieldWriter); } } else if (overrideName == "matchedattributeelementsfilter") { string source_field = argument.empty() ? fieldName : argument; if (getEnvironment() && getEnvironment()->getAttributeManager()) { - fieldWriter = AttributeCombinerDFW::create(source_field, *getEnvironment()->getAttributeManager(), true, struct_field_mapper); + auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); + fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, struct_field_mapper); + rc = static_cast<bool>(fieldWriter); + } + } else if (overrideName == "matchedelementsfilter") { + string source_field = argument.empty() ? fieldName : argument; + if (getEnvironment() && getEnvironment()->getAttributeManager()) { + auto attr_ctx = getEnvironment()->getAttributeManager()->createContext(); + fieldWriter = MatchedElementsFilterDFW::create(source_field, resultConfig.GetFieldNameEnum().Lookup(source_field.c_str()), + *attr_ctx, struct_field_mapper); rc = static_cast<bool>(fieldWriter); } } else { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index 1b7533b53e3..7c270d7184d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -2,6 +2,8 @@ #include "docsumstate.h" #include "matched_elements_filter_dfw.h" +#include "struct_fields_resolver.h" +#include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/common/struct_field_mapper.h> #include <vespa/vespalib/data/slime/binary_format.h> @@ -19,12 +21,25 @@ using vespalib::slime::inject; namespace search::docsummary { -MatchedElementsFilterDFW::MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum) +MatchedElementsFilterDFW::MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum, + std::shared_ptr<StructFieldMapper> struct_field_mapper) : _input_field_name(input_field_name), _input_field_enum(input_field_enum), - _struct_field_mapper(std::make_shared<StructFieldMapper>()) + _struct_field_mapper(std::move(struct_field_mapper)) { - // TODO: Take struct field mapper in constructor and populate based on available attribute vectors. +} + +std::unique_ptr<IDocsumFieldWriter> +MatchedElementsFilterDFW::create(const std::string& input_field_name, uint32_t input_field_enum, + search::attribute::IAttributeContext& attr_ctx, + std::shared_ptr<StructFieldMapper> struct_field_mapper) +{ + StructFieldsResolver resolver(input_field_name, attr_ctx, false); + if (resolver.has_error()) { + return std::unique_ptr<IDocsumFieldWriter>(); + } + resolver.apply_to(*struct_field_mapper); + return std::make_unique<MatchedElementsFilterDFW>(input_field_name, input_field_enum, std::move(struct_field_mapper)); } MatchedElementsFilterDFW::~MatchedElementsFilterDFW() = default; @@ -56,14 +71,6 @@ filter_matching_elements_in_input_field(const Slime& input_field, const std::vec } } -void -encode_output_field(const Slime& output_field, Inserter& target) -{ - vespalib::SmartBuffer buf(4096); - BinaryFormat::encode(output_field, buf); - target.insertString(buf.obtain()); -} - } void @@ -80,7 +87,7 @@ MatchedElementsFilterDFW::insertField(uint32_t docid, GeneralResult* result, Get Slime output_field; filter_matching_elements_in_input_field(input_field, state->get_matching_elements(*_struct_field_mapper).get_matching_elements(docid, _input_field_name), output_field); - encode_output_field(output_field, target); + inject(output_field.get(), target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h index b96d3595b0a..6962accc91d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -4,6 +4,8 @@ #include "docsumfieldwriter.h" +namespace search::attribute { class IAttributeContext; } + namespace search::docsummary { /** @@ -17,7 +19,11 @@ private: std::shared_ptr<StructFieldMapper> _struct_field_mapper; public: - MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum); + MatchedElementsFilterDFW(const std::string& input_field_name, uint32_t input_field_enum, + std::shared_ptr<StructFieldMapper> struct_field_mapper); + static std::unique_ptr<IDocsumFieldWriter> create(const std::string& input_field_name, uint32_t input_field_enum, + search::attribute::IAttributeContext& attr_ctx, + std::shared_ptr<StructFieldMapper> struct_field_mapper); ~MatchedElementsFilterDFW(); bool IsGenerated() const override { return false; } void insertField(uint32_t docid, GeneralResult* result, GetDocsumsState *state, diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp new file mode 100644 index 00000000000..c29f0324ce2 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.cpp @@ -0,0 +1,94 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "struct_fields_resolver.h" +#include <vespa/searchcommon/attribute/iattributecontext.h> +#include <vespa/searchlib/common/struct_field_mapper.h> +#include <algorithm> + +#include <vespa/log/log.h> +LOG_SETUP(".searchsummary.docsummary.struct_fields_resolver"); + +using search::attribute::CollectionType; +using search::attribute::IAttributeContext; + +namespace search::docsummary { + +StructFieldsResolver::StructFieldsResolver(const vespalib::string& field_name, const IAttributeContext& attr_ctx, + bool require_all_struct_fields_as_attribute) + : _field_name(field_name), + _map_key_attribute(), + _map_value_fields(), + _map_value_attributes(), + _array_fields(), + _array_attributes(), + _has_map_key(false), + _error(false) +{ + std::vector<const search::attribute::IAttributeVector *> attrs; + attr_ctx.getAttributeList(attrs); + vespalib::string prefix = field_name + "."; + _map_key_attribute = prefix + "key"; + vespalib::string value_prefix = prefix + "value."; + for (const auto attr : attrs) { + vespalib::string name = attr->getName(); + if (name.substr(0, prefix.size()) != prefix) { + continue; + } + if (attr->getCollectionType() != CollectionType::Type::ARRAY) { + LOG(warning, "Attribute '%s' is not an array attribute", name.c_str()); + _error = true; + break; + } + if (name.substr(0, value_prefix.size()) == value_prefix) { + _map_value_fields.emplace_back(name.substr(value_prefix.size())); + } else { + _array_fields.emplace_back(name.substr(prefix.size())); + if (name == _map_key_attribute) { + _has_map_key = true; + } + } + } + if (!_error) { + std::sort(_map_value_fields.begin(), _map_value_fields.end()); + for (const auto& field : _map_value_fields) { + _map_value_attributes.emplace_back(value_prefix + field); + } + + std::sort(_array_fields.begin(), _array_fields.end()); + for (const auto& field : _array_fields) { + _array_attributes.emplace_back(prefix + field); + } + + if (require_all_struct_fields_as_attribute && !_map_value_fields.empty()) { + if (!_has_map_key) { + LOG(warning, "Missing key attribute '%s', have value attributes for map", _map_key_attribute.c_str()); + _error = true; + } else if (_array_fields.size() != 1u) { + LOG(warning, "Could not determine if field '%s' is array or map of struct", field_name.c_str()); + _error = true; + } + } + } +} + +StructFieldsResolver::~StructFieldsResolver() = default; + +void +StructFieldsResolver::apply_to(StructFieldMapper& mapper) const +{ + if (is_map_of_struct()) { + if (_has_map_key) { + mapper.add_mapping(_field_name, _map_key_attribute); + } + for (const auto& sub_field : _map_value_attributes) { + mapper.add_mapping(_field_name, sub_field); + } + } else { + for (const auto& sub_field : _array_attributes) { + mapper.add_mapping(_field_name, sub_field); + } + } +} + +} + diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h new file mode 100644 index 00000000000..66d9fd69db4 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_fields_resolver.h @@ -0,0 +1,46 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vector> + +namespace search { +namespace attribute { class IAttributeContext; } +class StructFieldMapper; +} + +namespace search::docsummary { + +/** + * Class used to resolve which struct sub fields a complex field consists of, + * based on which attribute vectors are present. + */ +class StructFieldsResolver { +private: + using StringVector = std::vector<vespalib::string>; + vespalib::string _field_name; + vespalib::string _map_key_attribute; + StringVector _map_value_fields; + StringVector _map_value_attributes; + StringVector _array_fields; + StringVector _array_attributes; + bool _has_map_key; + bool _error; + +public: + StructFieldsResolver(const vespalib::string& field_name, const search::attribute::IAttributeContext& attr_ctx, + bool require_all_struct_fields_as_attributes); + ~StructFieldsResolver(); + bool is_map_of_struct() const { return !_map_value_fields.empty(); } + const vespalib::string& get_map_key_attribute() const { return _map_key_attribute; } + const StringVector& get_map_value_fields() const { return _map_value_fields; } + const StringVector& get_map_value_attributes() const { return _map_value_attributes; } + const StringVector& get_array_fields() const { return _array_fields; } + const StringVector& get_array_attributes() const { return _array_attributes; } + bool has_error() const { return _error; } + void apply_to(StructFieldMapper& mapper) const; +}; + +} + diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp index 1b43acc7231..d5922ddf46b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.cpp @@ -1,8 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "struct_map_attribute_combiner_dfw.h" -#include "docsum_field_writer_state.h" #include "attribute_field_writer.h" +#include "docsum_field_writer_state.h" +#include "struct_fields_resolver.h" +#include "struct_map_attribute_combiner_dfw.h" #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchlib/common/matching_elements.h> @@ -121,25 +122,16 @@ StructMapAttributeFieldWriterState::insertField(uint32_t docId, vespalib::slime: } StructMapAttributeCombinerDFW::StructMapAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &valueFields, + const StructFieldsResolver& fields_resolver, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper) : AttributeCombinerDFW(fieldName, filter_elements, std::move(struct_field_mapper)), - _keyAttributeName(), - _valueFields(valueFields), - _valueAttributeNames() + _keyAttributeName(fields_resolver.get_map_key_attribute()), + _valueFields(fields_resolver.get_map_value_fields()), + _valueAttributeNames(fields_resolver.get_map_value_attributes()) { - _keyAttributeName = fieldName + ".key"; - _valueAttributeNames.reserve(_valueFields.size()); - vespalib::string prefix = fieldName + ".value."; - for (const auto &field : _valueFields) { - _valueAttributeNames.emplace_back(prefix + field); - } if (filter_elements && _struct_field_mapper && !_struct_field_mapper->is_struct_field(fieldName)) { - _struct_field_mapper->add_mapping(fieldName, _keyAttributeName); - for (const auto &sub_field : _valueAttributeNames) { - _struct_field_mapper->add_mapping(fieldName, sub_field); - } + fields_resolver.apply_to(*_struct_field_mapper); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h index 1f92172d68b..a28e487fb1c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/struct_map_attribute_combiner_dfw.h @@ -9,6 +9,7 @@ namespace search::attribute { class IAttributeContext; } namespace search::docsummary { class DocsumFieldWriterState; +class StructFieldsResolver; /* * This class reads values from multiple struct field attributes and @@ -23,7 +24,7 @@ class StructMapAttributeCombinerDFW : public AttributeCombinerDFW std::unique_ptr<DocsumFieldWriterState> allocFieldWriterState(search::attribute::IAttributeContext &context, const MatchingElements* matching_elements) override; public: StructMapAttributeCombinerDFW(const vespalib::string &fieldName, - const std::vector<vespalib::string> &valueFields, + const StructFieldsResolver& fields_resolver, bool filter_elements, std::shared_ptr<StructFieldMapper> struct_field_mapper); ~StructMapAttributeCombinerDFW() override; |