blob: dbbc321d05752aa7297cd737c896ee19c481c9bd (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.prelude.query.parser;
import com.yahoo.prelude.query.IntItem;
import com.yahoo.prelude.query.Item;
import com.yahoo.prelude.query.WeakAndItem;
import com.yahoo.search.query.parser.ParserEnvironment;
import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER;
import static com.yahoo.prelude.query.parser.Token.Kind.WORD;
/**
* A parser which turns contiguous searchable character into tokens and filters out other characters.
* The resulting tokens are collected into a WeakAnd item.
*
* @author bratseth
*/
public final class TokenizeParser extends AbstractParser {
public TokenizeParser(ParserEnvironment environment) {
super(environment);
}
@Override
protected Item parseItems() {
WeakAndItem weakAnd = new WeakAndItem();
Token token;
while (null != (token = tokens.next())) {
Item termItem = toTerm(token);
if (termItem != null)
weakAnd.addItem(termItem);
}
return weakAnd;
}
/** Returns the item representing this token if it is searchable, and null otherwise */
private Item toTerm(Token token) {
if (token.kind == WORD)
return segment("", token, false);
else if (token.kind == NUMBER)
return new IntItem(token.image);
else
return null;
}
}
|