From aea9c1ee84d1e17c2a6dc11e01f2981d724c3822 Mon Sep 17 00:00:00 2001
From: Tor Egge <Tor.Egge@online.no>
Date: Wed, 27 Mar 2024 14:52:53 +0100
Subject: Add streaming mode version of tokens document field writer.

---
 .../src/vespa/vsm/vsm/tokens_converter.h           | 27 ++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 streamingvisitors/src/vespa/vsm/vsm/tokens_converter.h

(limited to 'streamingvisitors/src/vespa/vsm/vsm/tokens_converter.h')

diff --git a/streamingvisitors/src/vespa/vsm/vsm/tokens_converter.h b/streamingvisitors/src/vespa/vsm/vsm/tokens_converter.h
new file mode 100644
index 00000000000..689b0b95b02
--- /dev/null
+++ b/streamingvisitors/src/vespa/vsm/vsm/tokens_converter.h
@@ -0,0 +1,27 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/searchsummary/docsummary/i_string_field_converter.h>
+#include <vespa/searchlib/query/query_normalization.h>
+
+namespace vsm {
+
+/*
+ * Class converting a string field value into an array
+ * containing the tokens.
+ */
+class TokensConverter : public search::docsummary::IStringFieldConverter
+{
+    vespalib::stringref _text;
+    bool                _exact_match;
+    search::Normalizing _normalize_mode;
+
+public:
+    TokensConverter(bool exact_match, search::Normalizing normalize_mode);
+    ~TokensConverter() override;
+    void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override;
+    bool render_weighted_set_as_array() const override;
+};
+
+}
-- 
cgit v1.2.3