blob: e772ce9f0a3e8c74552463f4cc0139d46c8f82a2 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.language.process;
import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Immutable named lists of "special tokens" - strings which should override the normal tokenizer semantics
* and be tokenized into a single token.
*
* @author bratseth
*/
public class SpecialTokenRegistry {
/**
* The current special token lists, indexed on name.
* These lists are unmodifiable and used directly by clients of this
*/
private final Map<String, SpecialTokens> specialTokenMap;
/** Creates an empty special token registry */
public SpecialTokenRegistry() {
this(List.of());
}
/** Create a special token registry from a configuration object. */
public SpecialTokenRegistry(SpecialtokensConfig config) {
this(specialTokensFrom(config));
}
public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) {
specialTokenMap = specialTokensList.stream().collect(Collectors.toUnmodifiableMap(t -> t.name(), t -> t));
}
private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) {
List<SpecialTokens> specialTokensList = new ArrayList<>();
for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
Tokenlist tokenListConfig = i.next();
List<SpecialTokens.Token> tokenList = new ArrayList<>();
for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) {
Tokens tokenConfig = j.next();
tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace()));
}
specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList));
}
return specialTokensList;
}
/**
* Returns the list of special tokens for a given name.
*
* @param name the name of the special tokens to return
* null, the empty string or the string "default" returns
* the default ones
* @return a read-only list of SpecialToken instances, an empty list if this name
* has no special tokens
*/
public SpecialTokens getSpecialTokens(String name) {
if (name == null || name.trim().equals(""))
name = "default";
return specialTokenMap.getOrDefault(name, SpecialTokens.empty());
}
}
|
