diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java | 37 |
1 files changed, 10 insertions, 27 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java index 809e9b8d133..6cc68c7ac14 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java @@ -15,48 +15,35 @@ import java.util.Objects; public class SimpleToken implements Token { private final List<Token> components = new ArrayList<>(); - private final String original; + private final String orig; private TokenType type = TokenType.UNKNOWN; private TokenScript script = TokenScript.UNKNOWN; private String tokenString; - private List<String> stems = null; // Any additional stems after tokenString private boolean specialToken = false; private long offset = 0; - public SimpleToken(String original) { - this(original, (String)null); + public SimpleToken(String orig) { + this(orig, null); } - public SimpleToken(String original, String tokenString) { - this.original = original; + public SimpleToken(String orig, String tokenString) { + this.orig = orig; this.tokenString = tokenString; } - /** Exposed as fromStems */ - private SimpleToken(String original, List<String> stems) { - this.type = TokenType.ALPHABETIC; // Only type which may have stems - this.original = original; - this.tokenString = stems.get(0); - this.stems = List.copyOf(stems.subList(1, stems.size())); - } - @Override public String getOrig() { - return original; + return orig; } @Override public int getNumStems() { - return (tokenString != null ? 1 : 0) + (stems != null ? stems.size() : 0); + return tokenString != null ? 1 : 0; } @Override public String getStem(int i) { - if (i == 0) - return tokenString; - if (stems != null && i-1 < stems.size()) - return stems.get(i-1); - return tokenString; // TODO Vespa 9: throw new IllegalArgumentException() instead + return tokenString; } @Override @@ -144,12 +131,12 @@ public class SimpleToken implements Token { @Override public int hashCode() { - return original.hashCode(); + return orig.hashCode(); } @Override public String toString() { - return "token '" + original + "'"; + return "token '" + orig + "'"; } public String toDetailString() { @@ -184,8 +171,4 @@ public class SimpleToken implements Token { return getType().isIndexable() && (getOrig().length() > 0); } - public static SimpleToken fromStems(String original, List<String> stems) { - return new SimpleToken(original, stems); - } - } |