Revert "Merge pull request #17746 from vespa-engine/revert-17738-revert-17737-revert-17736-bratseth/special-tokens"

This reverts commit 491856b396d003885e159345fe3f533f0fa35933, reversing
changes made to 3720186303f4aef1d185525eaf61092097a64ec9.

1 files changed, 6 insertions, 1 deletions

diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
index e1a04b2985d..4ffe2a866d8 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
@@ -11,10 +11,14 @@ import com.yahoo.language.process.GramSplitter;
 import com.yahoo.language.process.Normalizer;
 import com.yahoo.language.process.Segmenter;
 import com.yahoo.language.process.SegmenterImpl;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.language.process.Stemmer;
 import com.yahoo.language.process.StemmerImpl;
 import com.yahoo.language.process.Tokenizer;
 import com.yahoo.language.process.Transformer;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+
+import java.util.List;
 
 /**
  * Factory of simple linguistic processor implementations.
@@ -31,6 +35,7 @@ public class SimpleLinguistics implements Linguistics {
     private final Detector detector;
     private final CharacterClasses characterClasses;
     private final GramSplitter gramSplitter;
+    private final SpecialTokenRegistry specialTokenRegistry = new SpecialTokenRegistry(List.of());
 
     @Inject
     public SimpleLinguistics() {
@@ -45,7 +50,7 @@ public class SimpleLinguistics implements Linguistics {
     public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); }
 
     @Override
-    public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer); }
+    public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); }
 
     @Override
     public Normalizer getNormalizer() { return normalizer; }