diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/simple')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java | 6 | ||||
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java | 4 |
2 files changed, 4 insertions, 6 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 61d446cd8d0..53b8ad7ad70 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -130,14 +130,10 @@ public class SimpleDetector implements Detector { } public String guessEncoding(byte[] input) { - return guessEncoding(input, 0, input.length); - } - - public String guessEncoding(byte[] input, int offset, int length) { boolean isUtf8 = true; boolean hasHighs = false; scan: - for (int i = offset; i < offset + length; i++) { + for (int i = 0; i < input.length; i++) { final int l = isLeadingFor(input[i]); if (l < 0 || i + l >= input.length) { hasHighs = true; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index b10beb8c9af..3ca46dcc4f1 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -2,7 +2,8 @@ package com.yahoo.language.simple; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; +import com.yahoo.collections.Tuple2; +import com.yahoo.component.Version; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.CharacterClasses; @@ -15,6 +16,7 @@ import com.yahoo.language.process.Stemmer; import com.yahoo.language.process.StemmerImpl; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import java.util.List; |