// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; import com.yahoo.language.Language; import com.yahoo.language.LinguisticsCase; import com.yahoo.language.process.*; import com.yahoo.language.simple.kstem.KStemmer; import java.util.ArrayList; import java.util.Collections; import java.util.List; /** *
A tokenizer which splits on whitespace, normalizes and transforms using the given implementations * and stems using the kstem algorithm.
* *This is not multithread safe.
* * @author Mathias Mølster Lidal * @author bratseth */ public class SimpleTokenizer implements Tokenizer { private final static int SPACE_CODE = 32; private final Normalizer normalizer; private final Transformer transformer; private final KStemmer stemmer = new KStemmer(); public SimpleTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); } public SimpleTokenizer(Normalizer normalizer) { this(normalizer, new SimpleTransformer()); } public SimpleTokenizer(Normalizer normalizer, Transformer transformer) { this.normalizer = normalizer; this.transformer = transformer; } @Override public Iterable