blob: ac29777795a4cdd7761c3197b25cfef7878427ba (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
/**
* An enumeration of token types.
*
* @author Mathias Mølster Lidal
*/
public enum TokenType {
UNKNOWN(0),
SPACE(1),
PUNCTUATION(2),
SYMBOL(3),
ALPHABETIC(4),
NUMERIC(5),
INDEXABLE_SYMBOL(6),
MARKER(255);
private final int value;
TokenType(int value) {
this.value = value;
}
/** Returns an int code for this type */
public int getValue() { return value; }
/**
* Marker for whether this type of token can be indexed for search.
* Note that a Token can be excluded from an index, even though the token type marks
* it as indexable.
*
* @see com.yahoo.language.process.Token#isIndexable()
* @return whether this type of token can be indexed
*/
public boolean isIndexable() {
return switch (this) {
case ALPHABETIC, NUMERIC, INDEXABLE_SYMBOL -> true;
default -> false;
};
}
/** Translates this from the int code representation returned from {@link #getValue} */
public static TokenType valueOf(int value) {
for (TokenType type : values()) {
if (value == type.value) return type;
}
return UNKNOWN;
}
}
|
