blob: 1798abac203259edcc77a38d31d9dc4e5a105f83 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
#include "i_string_field_converter.h"
namespace search::linguistics { class TokenExtractor; }
namespace search::docsummary {
/*
* Class converting a string field value with annotations into an array
* containing the tokens. Multiple tokens at same position are
* placed in a nested array.
*/
class TokensConverter : public IStringFieldConverter
{
const linguistics::TokenExtractor& _token_extractor;
vespalib::stringref _text;
template <typename ForwardIt>
void handle_alternative_index_terms(ForwardIt it, ForwardIt last, vespalib::slime::Inserter& inserter);
void handle_index_term(vespalib::stringref word, vespalib::slime::Inserter& inserter);
void handle_indexing_terms(const document::StringFieldValue& value, vespalib::slime::Inserter& inserter);
public:
TokensConverter(const linguistics::TokenExtractor& token_extractor);
~TokensConverter() override;
void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override;
bool render_weighted_set_as_array() const override;
};
}
|
