1 files changed, 28 insertions, 0 deletions
diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp
index 180200e2eaf..95e0f961d13 100644
--- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp
+++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp
@@ -1,6 +1,8 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "docsum_field_writer_factory.h"
+#include "fieldsearchspec.h"
+#include "tokens_dfw.h"
 #include <vespa/searchlib/common/matching_elements_fields.h>
 #include <vespa/searchsummary/docsummary/copy_dfw.h>
 #include <vespa/searchsummary/docsummary/docsum_field_writer.h>
@@ -8,8 +10,10 @@
 #include <vespa/searchsummary/docsummary/empty_dfw.h>
 #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h>
 #include <vespa/vsm/config/config-vsmfields.h>
+#include <algorithm>
 
 using search::MatchingElementsFields;
+using search::Normalizing;
 using search::docsummary::CopyDFW;
 using search::docsummary::DocsumFieldWriter;
 using search::docsummary::EmptyDFW;
@@ -35,6 +39,23 @@ void populate_fields(MatchingElementsFields& fields, VsmfieldsConfig& fields_con
     }
 }
 
+bool is_exact_match(vespalib::stringref arg1) {
+    return ((arg1 == "exact") || (arg1 == "word"));
+}
+
+std::unique_ptr<DocsumFieldWriter>
+make_tokens_dfw(const vespalib::string& source, VsmfieldsConfig& fields_config)
+{
+    bool exact_match = false;
+    Normalizing normalize_mode = Normalizing::LOWERCASE;
+    auto it = std::find_if(fields_config.fieldspec.begin(), fields_config.fieldspec.end(), [&source](auto& fs) { return source == fs.name; });
+    if (it != fields_config.fieldspec.end()) {
+        exact_match = is_exact_match(it->arg1);
+        normalize_mode = FieldSearchSpecMap::convert_normalize_mode(it->normalize);
+    }
+    return std::make_unique<TokensDFW>(source, exact_match, normalize_mode);
+}
+
 }
 
 DocsumFieldWriterFactory::DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env, const IQueryTermFilterFactory& query_term_filter_factory, const vespa::config::search::vsm::VsmfieldsConfig& vsm_fields_config)
@@ -68,6 +89,13 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie
         vespalib::string source_field = source.empty() ? field_name : source;
         populate_fields(*matching_elems_fields, _vsm_fields_config, source_field);
         fieldWriter = MatchedElementsFilterDFW::create(source_field, matching_elems_fields);
+    } else if ((command == command::tokens) ||
+               (command == command::attribute_tokens)) {
+        if (!source.empty()) {
+            fieldWriter = make_tokens_dfw(source, _vsm_fields_config);
+        } else {
+            throw_missing_source(command);
+        }
     } else {
         return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source, matching_elems_fields);
     }