diff options
Diffstat (limited to 'linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java')
-rw-r--r-- | linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java b/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java index a5e665b9444..9301e73aa5d 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java +++ b/linguistics/src/main/java/com/yahoo/language/process/SegmenterImpl.java @@ -33,7 +33,10 @@ public class SegmenterImpl implements Segmenter { int len; if (token.isSpecialToken() || (len = token.getNumComponents()) == 0) { if (token.isIndexable()) { - out.add(token.getOrig()); + String orig = token.getOrig(); + if (! orig.isEmpty()) { + out.add(orig); + } } } else { for (int i = 0; i < len; ++i) { |