summaryrefslogtreecommitdiffstats
path: root/linguistics/src/main/java/com/yahoo/language/simple
diff options
context:
space:
mode:
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/simple')
-rw-r--r--linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java6
-rw-r--r--linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java4
2 files changed, 4 insertions, 6 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
index 61d446cd8d0..53b8ad7ad70 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
@@ -130,14 +130,10 @@ public class SimpleDetector implements Detector {
}
public String guessEncoding(byte[] input) {
- return guessEncoding(input, 0, input.length);
- }
-
- public String guessEncoding(byte[] input, int offset, int length) {
boolean isUtf8 = true;
boolean hasHighs = false;
scan:
- for (int i = offset; i < offset + length; i++) {
+ for (int i = 0; i < input.length; i++) {
final int l = isLeadingFor(input[i]);
if (l < 0 || i + l >= input.length) {
hasHighs = true;
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
index b10beb8c9af..3ca46dcc4f1 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
@@ -2,7 +2,8 @@
package com.yahoo.language.simple;
import com.google.inject.Inject;
-import com.yahoo.component.AbstractComponent;
+import com.yahoo.collections.Tuple2;
+import com.yahoo.component.Version;
import com.yahoo.language.Linguistics;
import com.yahoo.language.detect.Detector;
import com.yahoo.language.process.CharacterClasses;
@@ -15,6 +16,7 @@ import com.yahoo.language.process.Stemmer;
import com.yahoo.language.process.StemmerImpl;
import com.yahoo.language.process.Tokenizer;
import com.yahoo.language.process.Transformer;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
import java.util.List;