aboutsummaryrefslogtreecommitdiffstats
path: root/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java
diff options
context:
space:
mode:
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java')
-rw-r--r--linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java37
1 files changed, 10 insertions, 27 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java
index 809e9b8d133..6cc68c7ac14 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java
@@ -15,48 +15,35 @@ import java.util.Objects;
public class SimpleToken implements Token {
private final List<Token> components = new ArrayList<>();
- private final String original;
+ private final String orig;
private TokenType type = TokenType.UNKNOWN;
private TokenScript script = TokenScript.UNKNOWN;
private String tokenString;
- private List<String> stems = null; // Any additional stems after tokenString
private boolean specialToken = false;
private long offset = 0;
- public SimpleToken(String original) {
- this(original, (String)null);
+ public SimpleToken(String orig) {
+ this(orig, null);
}
- public SimpleToken(String original, String tokenString) {
- this.original = original;
+ public SimpleToken(String orig, String tokenString) {
+ this.orig = orig;
this.tokenString = tokenString;
}
- /** Exposed as fromStems */
- private SimpleToken(String original, List<String> stems) {
- this.type = TokenType.ALPHABETIC; // Only type which may have stems
- this.original = original;
- this.tokenString = stems.get(0);
- this.stems = List.copyOf(stems.subList(1, stems.size()));
- }
-
@Override
public String getOrig() {
- return original;
+ return orig;
}
@Override
public int getNumStems() {
- return (tokenString != null ? 1 : 0) + (stems != null ? stems.size() : 0);
+ return tokenString != null ? 1 : 0;
}
@Override
public String getStem(int i) {
- if (i == 0)
- return tokenString;
- if (stems != null && i-1 < stems.size())
- return stems.get(i-1);
- return tokenString; // TODO Vespa 9: throw new IllegalArgumentException() instead
+ return tokenString;
}
@Override
@@ -144,12 +131,12 @@ public class SimpleToken implements Token {
@Override
public int hashCode() {
- return original.hashCode();
+ return orig.hashCode();
}
@Override
public String toString() {
- return "token '" + original + "'";
+ return "token '" + orig + "'";
}
public String toDetailString() {
@@ -184,8 +171,4 @@ public class SimpleToken implements Token {
return getType().isIndexable() && (getOrig().length() > 0);
}
- public static SimpleToken fromStems(String original, List<String> stems) {
- return new SimpleToken(original, stems);
- }
-
}