From aea9c1ee84d1e17c2a6dc11e01f2981d724c3822 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 27 Mar 2024 14:52:53 +0100 Subject: Add streaming mode version of tokens document field writer. --- streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h (limited to 'streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h') diff --git a/streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h b/streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h new file mode 100644 index 00000000000..23fd869d005 --- /dev/null +++ b/streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h @@ -0,0 +1,28 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include + +namespace vsm { + +/* + * Class for writing annotated string field values from document as + * arrays containing the tokens. + */ +class TokensDFW : public search::docsummary::DocsumFieldWriter +{ +private: + vespalib::string _input_field_name; + bool _exact_match; + search::Normalizing _normalize_mode; + +public: + explicit TokensDFW(const vespalib::string& input_field_name, bool exact_match, search::Normalizing normalize_mode); + ~TokensDFW() override; + bool isGenerated() const override; + void insertField(uint32_t docid, const search::docsummary::IDocsumStoreDocument* doc, search::docsummary::GetDocsumsState& state, vespalib::slime::Inserter& target) const override; +}; + +} -- cgit v1.2.3 From 65d47f5dc645c2f746cb31a40891af99bfdeba64 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 27 Mar 2024 15:35:02 +0100 Subject: Update class comment. --- streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h') diff --git a/streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h b/streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h index 23fd869d005..4199630a94d 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h +++ b/streamingvisitors/src/vespa/vsm/vsm/tokens_dfw.h @@ -8,8 +8,9 @@ namespace vsm { /* - * Class for writing annotated string field values from document as - * arrays containing the tokens. + * Class for writing string field values from document as + * arrays containing the tokens. Tokenization is performed + * on the fly using the exact_match and normalize_mode settings. */ class TokensDFW : public search::docsummary::DocsumFieldWriter { -- cgit v1.2.3