summaryrefslogtreecommitdiffstats
path: root/linguistics
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@vespa.ai>2023-05-19 11:24:07 +0200
committerJon Bratseth <bratseth@vespa.ai>2023-05-19 11:24:07 +0200
commitd9e2eff020c6ea308142c264a8b63a23f0cce32e (patch)
treec0063d4fb497ca2f61983f049d735a85c050b51f /linguistics
parentff426d5cf4d02a259719770a360cce664087503d (diff)
Use dollar and hour base units
Diffstat (limited to 'linguistics')
-rw-r--r--linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java4
1 files changed, 2 insertions, 2 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
index b791c843357..2728249333e 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
@@ -57,7 +57,7 @@ public class SimpleTokenizer implements Tokenizer {
}
/** Tokenize the input, and apply the given transform to each token string. */
- public Iterable<Token> tokenize(String input, Function<String, String> tokenProocessor) {
+ public Iterable<Token> tokenize(String input, Function<String, String> tokenProcessor) {
if (input.isEmpty()) return List.of();
List<Token> tokens = new ArrayList<>();
@@ -71,7 +71,7 @@ public class SimpleTokenizer implements Tokenizer {
String original = input.substring(prev, next);
tokens.add(new SimpleToken(original).setOffset(prev)
.setType(tokenType)
- .setTokenString(tokenProocessor.apply(original)));
+ .setTokenString(tokenProcessor.apply(original)));
prev = next;
prevType = nextType;
tokenType = prevType;