diff options
author | Jon Bratseth <bratseth@vespa.ai> | 2023-05-19 11:24:07 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@vespa.ai> | 2023-05-19 11:24:07 +0200 |
commit | d9e2eff020c6ea308142c264a8b63a23f0cce32e (patch) | |
tree | c0063d4fb497ca2f61983f049d735a85c050b51f /linguistics | |
parent | ff426d5cf4d02a259719770a360cce664087503d (diff) |
Use dollar and hour base units
Diffstat (limited to 'linguistics')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index b791c843357..2728249333e 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -57,7 +57,7 @@ public class SimpleTokenizer implements Tokenizer { } /** Tokenize the input, and apply the given transform to each token string. */ - public Iterable<Token> tokenize(String input, Function<String, String> tokenProocessor) { + public Iterable<Token> tokenize(String input, Function<String, String> tokenProcessor) { if (input.isEmpty()) return List.of(); List<Token> tokens = new ArrayList<>(); @@ -71,7 +71,7 @@ public class SimpleTokenizer implements Tokenizer { String original = input.substring(prev, next); tokens.add(new SimpleToken(original).setOffset(prev) .setType(tokenType) - .setTokenString(tokenProocessor.apply(original))); + .setTokenString(tokenProcessor.apply(original))); prev = next; prevType = nextType; tokenType = prevType; |