1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#include "snippetmodifier.h"
#include <vespa/document/fieldvalue/stringfieldvalue.h>
#include <vespa/juniper/juniper_separators.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/log/log.h>
LOG_SETUP(".vsm.snippetmodifier");
using namespace document;
using search::streaming::QueryTerm;
using search::streaming::QueryTermList;
using FieldQueryTermMap = vespalib::hash_map<vsm::FieldIdT, QueryTermList>;
namespace {
void
addIfNotPresent(FieldQueryTermMap & map, vsm::FieldIdT fId, QueryTerm * qt)
{
auto itr = map.find(fId);
if (itr != map.end()) {
QueryTermList & qtl = itr->second;
if (std::find(qtl.begin(), qtl.end(), qt) == qtl.end()) {
qtl.push_back(qt);
}
} else {
map[fId].push_back(qt);
}
}
}
namespace vsm {
void
SnippetModifier::considerSeparator()
{
if (_useSep) {
_valueBuf->put(_recordSep);
}
}
void
SnippetModifier::onPrimitive(uint32_t, const Content & c)
{
considerSeparator();
_searcher->onValue(c.getValue());
_valueBuf->put(_searcher->getModifiedBuf().getBuffer(), _searcher->getModifiedBuf().getPos());
_useSep = true;
}
void
SnippetModifier::reset()
{
_valueBuf->reset();
_useSep = false;
}
SnippetModifier::SnippetModifier(const UTF8SubstringSnippetModifier::SP & searcher) :
_searcher(searcher),
_valueBuf(new CharBuffer(32)),
_recordSep(juniper::separators::record_separator),
_useSep(false),
_empty()
{
}
SnippetModifier::SnippetModifier(const UTF8SubstringSnippetModifier::SP & searcher, const CharBuffer::SP & valueBuf) :
_searcher(searcher),
_valueBuf(valueBuf),
_recordSep(juniper::separators::record_separator),
_useSep(false),
_empty()
{
}
SnippetModifier::~SnippetModifier() {}
FieldValue::UP
SnippetModifier::modify(const FieldValue & fv, const document::FieldPath & path)
{
reset();
fv.iterateNested(path, *this);
return FieldValue::UP(new StringFieldValue(vespalib::string(_valueBuf->getBuffer(), _valueBuf->getPos())));
}
SnippetModifierManager::SnippetModifierManager() :
_modifiers(),
_searchBuf(new SearcherBuf(64)),
_searchModifyBuf(new CharBuffer(64)),
_searchOffsetBuf(new std::vector<size_t>(64)),
_modifierBuf(new CharBuffer(128))
{
}
SnippetModifierManager::~SnippetModifierManager() {}
void
SnippetModifierManager::setup(const QueryTermList& queryTerms,
const FieldSearchSpecMapT& specMap,
const IndexFieldMapT& indexMap,
const vsm::FieldPathMapT& field_paths,
search::fef::IQueryEnvironment& query_env)
{
FieldQueryTermMap fqtm;
// setup modifiers
for (auto qt : queryTerms) {
auto itr = indexMap.find(qt->index());
if (itr != indexMap.end()) {
for (auto fId : itr->second) {
const FieldSearchSpec & spec = specMap.find(fId)->second;
if (spec.searcher().substring() || qt->isSubstring()) { // we need a modifier for this field id
addIfNotPresent(fqtm, fId, qt);
if (_modifiers.getModifier(fId) == nullptr) {
LOG(debug, "Create snippet modifier for field id '%u'", fId);
UTF8SubstringSnippetModifier::SP searcher
(new UTF8SubstringSnippetModifier(fId, _searchModifyBuf, _searchOffsetBuf));
_modifiers.map()[fId] = std::make_unique<SnippetModifier>(searcher, _modifierBuf);
}
}
}
}
}
// prepare modifiers
for (auto & entry : _modifiers.map()) {
FieldIdT fId = entry.first;
SnippetModifier & smod = static_cast<SnippetModifier &>(*entry.second);
smod.getSearcher()->prepare(fqtm[fId], _searchBuf, field_paths, query_env);
}
}
}
|
