Enable map/array element filtering in document summary for streaming search.

5 files changed, 89 insertions, 5 deletions

diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp
index f3d9710c93c..7402a45fa4a 100644
--- a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp
+++ b/vsm/src/vespa/vsm/vsm/docsumconfig.cpp
@@ -2,12 +2,41 @@
 
 #include <vespa/vsm/vsm/docsumconfig.h>
 #include <vespa/searchsummary/docsummary/docsumfieldwriter.h>
+#include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h>
+#include <vespa/searchlib/common/struct_field_mapper.h>
+#include <vespa/vsm/config/config-vsmfields.h>
+#include <vespa/vsm/config/config-vsmsummary.h>
 
+using search::StructFieldMapper;
 using search::docsummary::IDocsumFieldWriter;
 using search::docsummary::EmptyDFW;
+using search::docsummary::MatchedElementsFilterDFW;
+using search::docsummary::ResultConfig;
+using vespa::config::search::vsm::VsmfieldsConfig;
+using vespa::config::search::vsm::VsmsummaryConfig;
 
 namespace vsm {
 
+namespace {
+
+void populate_mapper(StructFieldMapper& mapper, VsmfieldsConfig& fields_config, const vespalib::string& field_name)
+{
+    vespalib::string prefix = field_name + ".";
+    for (const auto& spec : fields_config.fieldspec) {
+        if (spec.name.substr(0, prefix.size()) == prefix) {
+            mapper.add_mapping(field_name, spec.name);
+        }
+    }
+}
+
+}
+
+DynamicDocsumConfig::DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config)
+    : Parent(env, writer),
+      _vsm_fields_config(std::move(vsm_fields_config))
+{
+}
+
 IDocsumFieldWriter::UP
 DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string & overrideName, const string & argument, bool & rc, std::shared_ptr<search::StructFieldMapper> struct_field_mapper)
 {
@@ -26,6 +55,14 @@ DynamicDocsumConfig::createFieldWriter(const string & fieldName, const string &
             (overrideName == "attributecombiner") ||
             (overrideName == "geopos")) {
         rc = true;
+    } else if ((overrideName == "matchedattributeelementsfilter") ||
+               (overrideName == "matchedelementsfilter")) {
+        string source_field = argument.empty() ? fieldName : argument;
+        const ResultConfig& resultConfig = getResultConfig();
+        int source_field_enum = resultConfig.GetFieldNameEnum().Lookup(source_field.c_str());
+        populate_mapper(*struct_field_mapper, *_vsm_fields_config, source_field);
+        fieldWriter = MatchedElementsFilterDFW::create(source_field, source_field_enum, struct_field_mapper);
+        rc = static_cast<bool>(fieldWriter);
     } else {
         fieldWriter = search::docsummary::DynamicDocsumConfig::createFieldWriter(fieldName, overrideName, argument, rc, struct_field_mapper);
     }
diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.h b/vsm/src/vespa/vsm/vsm/docsumconfig.h
index 351e2523526..17128798ef2 100644
--- a/vsm/src/vespa/vsm/vsm/docsumconfig.h
+++ b/vsm/src/vespa/vsm/vsm/docsumconfig.h
@@ -4,13 +4,21 @@
 
 #include <vespa/searchsummary/docsummary/docsumconfig.h>
 
+namespace vespa::config::search::vsm {
+namespace internal { class InternalVsmfieldsType; }
+typedef const internal::InternalVsmfieldsType VsmfieldsConfig;
+}
 namespace vsm {
 
 class DynamicDocsumConfig : public search::docsummary::DynamicDocsumConfig
 {
 public:
     using Parent = search::docsummary::DynamicDocsumConfig;
-    using Parent::Parent;
+    using VsmfieldsConfig = vespa::config::search::vsm::VsmfieldsConfig;
+private:
+    std::shared_ptr<VsmfieldsConfig> _vsm_fields_config;
+public:
+    DynamicDocsumConfig(search::docsummary::IDocsumEnvironment* env, search::docsummary::DynamicDocsumWriter* writer, std::shared_ptr<VsmfieldsConfig> vsm_fields_config);
 private:
     std::unique_ptr<search::docsummary::IDocsumFieldWriter>
         createFieldWriter(const string & fieldName, const string & overrideName,
diff --git a/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h b/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h
new file mode 100644
index 00000000000..a30dcbf2a5b
--- /dev/null
+++ b/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h
@@ -0,0 +1,24 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+
+namespace search {
+class MatchingElements;
+class StructFieldMapper;
+}
+
+namespace vsm {
+
+/*
+ * Interface class for filling matching elements structure for
+ * streaming search.
+ */
+class IMatchingElementsFiller {
+public:
+    virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::StructFieldMapper& struct_field_mapper) = 0;
+    virtual ~IMatchingElementsFiller() = default;
+};
+    
+}
diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp
index d161130687e..dd7b36015e4 100644
--- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp
+++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp
@@ -2,6 +2,7 @@
 
 #include "vsm-adapter.h"
 #include "docsumconfig.h"
+#include "i_matching_elements_filler.h"
 #include <vespa/searchlib/common/matching_elements.h>
 
 #include <vespa/log/log.h>
@@ -16,7 +17,8 @@ namespace vsm {
 
 GetDocsumsStateCallback::GetDocsumsStateCallback() :
     _summaryFeatures(),
-    _rankFeatures()
+    _rankFeatures(),
+    _matching_elements_filler()
 { }
 
 void GetDocsumsStateCallback::FillSummaryFeatures(GetDocsumsState * state, IDocsumEnvironment * env)
@@ -48,11 +50,20 @@ void GetDocsumsStateCallback::FillDocumentLocations(GetDocsumsState *state, IDoc
 }
 
 std::unique_ptr<MatchingElements>
-GetDocsumsStateCallback::fill_matching_elements(const search::StructFieldMapper &)
+GetDocsumsStateCallback::fill_matching_elements(const search::StructFieldMapper& struct_field_mapper)
 {
+    if (_matching_elements_filler) {
+        return _matching_elements_filler->fill_matching_elements(struct_field_mapper);
+    }
     return std::make_unique<MatchingElements>();
 }
 
+void
+GetDocsumsStateCallback::set_matching_elements_filler(std::unique_ptr<IMatchingElementsFiller> matching_elements_filler)
+{
+    _matching_elements_filler = std::move(matching_elements_filler);
+}
+
 GetDocsumsStateCallback::~GetDocsumsStateCallback() = default;
 
 DocsumTools::FieldSpec::FieldSpec() :
@@ -156,7 +167,7 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot)
     docsumTools->setJuniper(std::move(juniper));
 
     // configure dynamic docsum writer
-    DynamicDocsumConfig dynDocsumConfig(docsumTools.get(), docsumTools->getDocsumWriter());
+    DynamicDocsumConfig dynDocsumConfig(docsumTools.get(), docsumTools->getDocsumWriter(), _fieldsCfg.get());
     dynDocsumConfig.configure(*summaryMap.get());
 
     // configure new docsum tools
diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/vsm/src/vespa/vsm/vsm/vsm-adapter.h
index 6f67f9eb90c..96d12e23db6 100644
--- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h
+++ b/vsm/src/vespa/vsm/vsm/vsm-adapter.h
@@ -27,11 +27,14 @@ using vespa::config::search::summary::JuniperrcConfig;
 
 namespace vsm {
 
+class IMatchingElementsFiller;
+
 class GetDocsumsStateCallback : public search::docsummary::GetDocsumsStateCallback
 {
 private:
     search::FeatureSet::SP _summaryFeatures;
     search::FeatureSet::SP _rankFeatures;
+    std::unique_ptr<IMatchingElementsFiller> _matching_elements_filler;
 
 public:
     GetDocsumsStateCallback();
@@ -39,9 +42,10 @@ public:
     void FillRankFeatures(GetDocsumsState * state, IDocsumEnvironment * env) override;
     void ParseLocation(GetDocsumsState * state) override;
     virtual void FillDocumentLocations(GetDocsumsState * state, IDocsumEnvironment * env);
-    virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::StructFieldMapper &) override;
+    virtual std::unique_ptr<search::MatchingElements> fill_matching_elements(const search::StructFieldMapper& struct_field_mapper) override;
     void setSummaryFeatures(const search::FeatureSet::SP & sf) { _summaryFeatures = sf; }
     void setRankFeatures(const search::FeatureSet::SP & rf) { _rankFeatures = rf; }
+    void set_matching_elements_filler(std::unique_ptr<IMatchingElementsFiller> matching_elements_filler);
     ~GetDocsumsStateCallback();
 };