Merge pull request #17760 from vespa-engine/bratseth/special-tokens-take-3

special-tokens-take-3

24 files changed, 434 insertions, 438 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
index 78d9dd473b3..3bc07db9507 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.model;
 
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeResources;
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java
index ee246b5e485..2b2b17c76c3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java
@@ -23,8 +23,9 @@ public class DocprocChain extends Chain<DocumentProcessor> {
     }
 
     /**
-     * The field name schema map that applies to this whole chain
-     * @return doctype,from → to
+     * The field name schema map that applies to this whole chain.
+     *
+     * @return doctype, from → to
      */
     public Map<Pair<String,String>,String> fieldNameSchemaMap() {
         return fieldNameSchemaMap;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java
index 2c1d979e2c4..8fe6b51f2b4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java
@@ -2,14 +2,12 @@
 package com.yahoo.vespa.model.search;
 
 import com.yahoo.component.ComponentId;
-import com.yahoo.component.ComponentSpecification;
 import com.yahoo.component.chain.Phase;
 import com.yahoo.component.chain.model.ChainSpecification;
 import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
 import com.yahoo.vespa.model.container.docproc.DocprocChain;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index b5933936adf..74ed9d33f04 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -5574,8 +5574,8 @@
       "public com.yahoo.search.query.parser.ParserEnvironment setIndexFacts(com.yahoo.prelude.IndexFacts)",
       "public com.yahoo.language.Linguistics getLinguistics()",
       "public com.yahoo.search.query.parser.ParserEnvironment setLinguistics(com.yahoo.language.Linguistics)",
-      "public com.yahoo.prelude.query.parser.SpecialTokens getSpecialTokens()",
-      "public com.yahoo.search.query.parser.ParserEnvironment setSpecialTokens(com.yahoo.prelude.query.parser.SpecialTokens)",
+      "public com.yahoo.language.process.SpecialTokens getSpecialTokens()",
+      "public com.yahoo.search.query.parser.ParserEnvironment setSpecialTokens(com.yahoo.language.process.SpecialTokens)",
       "public static com.yahoo.search.query.parser.ParserEnvironment fromExecutionContext(com.yahoo.search.searchchain.Execution$Context)",
       "public static com.yahoo.search.query.parser.ParserEnvironment fromParserEnvironment(com.yahoo.search.query.parser.ParserEnvironment)"
     ],
@@ -7765,7 +7765,7 @@
       "final"
     ],
     "methods": [
-      "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.prelude.query.parser.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)",
+      "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)",
       "public static com.yahoo.search.searchchain.Execution$Context createContextStub()",
       "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts)",
       "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts)",
@@ -7779,8 +7779,8 @@
       "public void setIndexFacts(com.yahoo.prelude.IndexFacts)",
       "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()",
       "public com.yahoo.search.rendering.RendererRegistry rendererRegistry()",
-      "public com.yahoo.prelude.query.parser.SpecialTokenRegistry getTokenRegistry()",
-      "public void setTokenRegistry(com.yahoo.prelude.query.parser.SpecialTokenRegistry)",
+      "public com.yahoo.language.process.SpecialTokenRegistry getTokenRegistry()",
+      "public void setTokenRegistry(com.yahoo.language.process.SpecialTokenRegistry)",
       "public void setDetailedDiagnostics(boolean)",
       "public boolean getDetailedDiagnostics()",
       "public boolean getBreakdown()",
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
index 902be7e15dd..732466748eb 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
@@ -19,7 +19,6 @@ import java.util.*;
  * @author bratseth
  * @author Steinar Knutsen
  */
-@SuppressWarnings("deprecation")
 public abstract class AbstractParser implements CustomParser {
 
     /** The current submodes of this parser */
@@ -48,7 +47,7 @@ public abstract class AbstractParser implements CustomParser {
      * of these may be active at the same time. SubModes are activated or
      * deactivated by specifying special indexes in the query.
      */
-    final class Submodes {
+    static final class Submodes {
 
         /**
          * Url mode allows "_" and "-" as word characters. Default is false
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
deleted file mode 100644
index be2d9f9f68b..00000000000
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.query.parser;
-
-import com.yahoo.config.subscription.ConfigGetter;
-import com.yahoo.config.subscription.ConfigSubscriber;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
-
-import java.util.*;
-import java.util.logging.Logger;
-
-
-/**
- * A <i>registry</i> which is responsible for knowing the current
- * set of special tokens. The default registry returns empty token lists
- * for all names. Usage of this registry is multithread safe.
- *
- * @author bratseth
- */
-public class SpecialTokenRegistry {
-
-    /** The log of this */
-    private static final Logger log = Logger.getLogger(SpecialTokenRegistry.class.getName());
-
-    private static final SpecialTokens nullSpecialTokens = new SpecialTokens();
-
-    /**
-     * The current authorative special token lists, indexed on name.
-     * These lists are unmodifiable and used directly by clients of this
-     */
-    private Map<String,SpecialTokens> specialTokenMap = new HashMap<>();
-
-    private boolean frozen = false;
-
-    /**
-     * Creates an empty special token registry which
-     * does not subscribe to any configuration
-     */
-    public SpecialTokenRegistry() {}
-
-    /**
-     * Create a special token registry which subscribes to the specialtokens
-     * configuration. Only used for testing.
-     */
-    public SpecialTokenRegistry(String configId) {
-        try {
-            build(new ConfigGetter<>(SpecialtokensConfig.class).getConfig(configId));
-        } catch (Exception e) {
-            log.config(
-                    "No special tokens are configured (" + e.getMessage() + ")");
-        }
-    }
-
-    /**
-     * Create a special token registry from a configuration object. This is the production code path.
-     */
-    public SpecialTokenRegistry(SpecialtokensConfig config) {
-        if (config != null) {
-            build(config);
-        }
-        freeze();
-    }
-
-    private void freeze() {
-        frozen = true;
-    }
-
-    private void build(SpecialtokensConfig config) {
-        List<SpecialTokens> list = new ArrayList<>();
-        for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
-            Tokenlist tokenList = i.next();
-            SpecialTokens tokens = new SpecialTokens(tokenList.name());
-
-            for (Iterator<Tokens> j = tokenList.tokens().iterator(); j.hasNext();) {
-                Tokens token = j.next();
-                tokens.addSpecialToken(token.token(), token.replace());
-            }
-            tokens.freeze();
-            list.add(tokens);
-        }
-        addSpecialTokens(list);
-    }
-
-    /**
-     * Adds a SpecialTokens instance to the registry. That is, add the
-     * tokens contained for the name of the SpecialTokens instance
-     * given.
-     *
-     * @param specialTokens the SpecialTokens object to add
-     */
-    public void addSpecialTokens(SpecialTokens specialTokens) {
-        ensureNotFrozen();
-        List<SpecialTokens> list = new ArrayList<>();
-        list.add(specialTokens);
-        addSpecialTokens(list);
-
-    }
-
-    private void ensureNotFrozen() {
-        if (frozen) {
-            throw new IllegalStateException("Tried to modify a frozen SpecialTokenRegistry instance.");
-        }
-    }
-
-    private void addSpecialTokens(List<SpecialTokens> list) {
-        HashMap<String,SpecialTokens> tokens = new HashMap<>(specialTokenMap);
-        for(SpecialTokens t: list) {
-            tokens.put(t.getName(),t);
-        }
-        specialTokenMap = tokens;
-    }
-
-
-    /**
-     * Returns the currently authorative list of special tokens for
-     * a given name.
-     *
-     * @param  name the name of the special tokens to return
-     *         null, the empth string or the string "default" returns
-     *         the default ones
-     * @return a read-only list of SpecialToken instances, an empty list if this name
-     *         has no special tokens
-     */
-    public SpecialTokens getSpecialTokens(String name) {
-        if (name == null || name.trim().equals("")) {
-            name = "default";
-        }
-        SpecialTokens specialTokens = specialTokenMap.get(name);
-
-        if (specialTokens == null) {
-            return nullSpecialTokens;
-        }
-        return specialTokens;
-    }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
deleted file mode 100644
index f45ecefefa6..00000000000
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
+++ /dev/null
@@ -1,167 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.query.parser;
-
-import java.util.logging.Level;
-import com.yahoo.prelude.query.Substring;
-
-import java.util.*;
-import java.util.logging.Logger;
-
-import static com.yahoo.language.LinguisticsCase.toLowerCase;
-
-/**
- * A list of special tokens - string that should be treated as word
- * no matter what they contain. Special tokens are case insensitive.
- *
- * @author bratseth
- */
-public class SpecialTokens {
-
-    private static final Logger log = Logger.getLogger(SpecialTokens.class.getName());
-
-    private final String name;
-
-    private final List<SpecialToken> specialTokens = new ArrayList<>();
-
-    private boolean frozen = false;
-
-    private int currentMaximumLength = 0;
-
-    /** Creates a null list of special tokens */
-    public SpecialTokens() {
-        this.name = "(null)";
-    }
-
-    public SpecialTokens(String name) {
-        this.name = name;
-    }
-
-    /** Returns the name of this special tokens list */
-    public String getName() {
-        return name;
-    }
-
-    /**
-     * Adds a special token to this
-     *
-     * @param token the special token string to add
-     * @param replace the token to replace instances of the special token with, or null to keep the token
-     */
-    public void addSpecialToken(String token, String replace) {
-        ensureNotFrozen();
-        if (!caseIndependentLength(token)) {
-            return;
-        }
-        // TODO are special tokens correctly unicode normalized in reagards to query parsing?
-        final SpecialToken specialTokenToAdd = new SpecialToken(token, replace);
-        currentMaximumLength = Math.max(currentMaximumLength, specialTokenToAdd.token.length());
-        specialTokens.add(specialTokenToAdd);
-        Collections.sort(specialTokens);
-    }
-
-    private boolean caseIndependentLength(String token) {
-        // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
-        String asLow = toLowerCase(token);
-        // TODO put along with the global toLowerCase
-        String asHigh = token.toUpperCase(Locale.ENGLISH);
-        if (asLow.length() != token.length() || asHigh.length() != token.length()) {
-            log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token."
-                    + " Please report this message in a bug to the Vespa team.");
-            return false;
-        } else {
-            return true;
-        }
-    }
-
-    /**
-     * Returns the special token starting at the start of the given string, or null if no
-     * special token starts at this string
-     *
-     * @param string the string to search for a special token at the start position
-     * @param substring true to allow the special token to be followed by a character which does not
-     *        mark the end of a token
-     */
-    public SpecialToken tokenize(String string, boolean substring) {
-        // XXX detonator pattern token.length may be != the length of the
-        // matching data in string, ref caseIndependentLength(String)
-        final String input = toLowerCase(string.substring(0, Math.min(string.length(), currentMaximumLength)));
-        for (Iterator<SpecialToken> i = specialTokens.iterator(); i.hasNext();) {
-            SpecialTokens.SpecialToken special = i.next();
-
-            if (input.startsWith(special.token())) {
-                if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string))
-                    return special;
-            }
-        }
-        return null;
-    }
-
-    private boolean tokenEndsAt(int position,String string) {
-        return !Character.isLetterOrDigit(string.charAt(position));
-    }
-
-    /** Returns the number of special tokens in this */
-    public int size() {
-        return specialTokens.size();
-    }
-
-    private void ensureNotFrozen() {
-        if (frozen) {
-            throw new IllegalStateException("Tried to modify a frozen SpecialTokens instance.");
-        }
-    }
-
-    public void freeze() {
-        frozen = true;
-    }
-
-    /** An immutable special token */
-    public final static class SpecialToken implements Comparable<SpecialToken> {
-
-        private String token;
-
-        private String replace;
-
-        public SpecialToken(String token, String replace) {
-            this.token = toLowerCase(token);
-            if (replace == null || replace.trim().equals("")) {
-                this.replace = this.token;
-            } else {
-                this.replace = toLowerCase(replace);
-            }
-        }
-
-        /** Returns the special token */
-        public String token() {
-            return token;
-        }
-
-        /** Returns the right replace value, never null or an empty string */
-        public String replace() {
-            return replace;
-        }
-
-        @Override
-        public int compareTo(SpecialToken other) {
-            if (this.token().length() < other.token().length()) return 1;
-            if (this.token().length() == other.token().length()) return 0;
-            return -1;
-        }
-
-        @Override
-        public boolean equals(Object other) {
-            if (other == this) return true;
-            if ( ! (other instanceof SpecialToken)) return false;
-            return Objects.equals(this.token, ((SpecialToken)other).token);
-        }
-
-        @Override
-        public int hashCode() { return token.hashCode(); }
-
-        public Token toToken(int start, String rawSource) {
-            return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe?
-        }
-
-    }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
index 2dc2254df68..b71bd57539f 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
@@ -3,6 +3,7 @@ package com.yahoo.prelude.query.parser;
 
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.CharacterClasses;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.query.Substring;
@@ -200,7 +201,7 @@ public final class Tokenizer {
         }
         StringBuilder tmp = new StringBuilder();
         for (int i = 0; i < tokencnt; i++) {
-            Token useToken = tokens.get(backtrack+i);
+            Token useToken = tokens.get(backtrack + i);
             tmp.append(useToken.image);
         }
         String indexName = tmp.toString();
@@ -216,20 +217,20 @@ public final class Tokenizer {
     }
 
     private int consumeSpecialToken(int start) {
-        SpecialTokens.SpecialToken specialToken=getSpecialToken(start);
-        if (specialToken==null) return start;
-        tokens.add(specialToken.toToken(start,source));
-        return start + specialToken.token().length();
+        SpecialTokens.Token token = getSpecialToken(start);
+        if (token == null) return start;
+        tokens.add(toToken(token, start, source));
+        return start + token.token().length();
     }
 
-    private SpecialTokens.SpecialToken getSpecialToken(int start) {
+    private SpecialTokens.Token getSpecialToken(int start) {
         if (specialTokens == null) return null;
         return specialTokens.tokenize(source.substring(start), substringSpecialTokens);
     }
 
     private int consumeExact(int start,Index index) {
         if (index.getExactTerminator() == null) return consumeHeuristicExact(start);
-        return consumeToTerminator(start,index.getExactTerminator());
+        return consumeToTerminator(start, index.getExactTerminator());
     }
 
     private boolean looksLikeExactEnd(int end) {
@@ -467,7 +468,7 @@ public final class Tokenizer {
     /** Consumes a word or number <i>and/or possibly</i> a special token starting within this word or number */
     private int consumeWordOrNumber(int start, Index currentIndex) {
         int tokenEnd = start;
-        SpecialTokens.SpecialToken substringSpecialToken = null;
+        SpecialTokens.Token substringToken = null;
         boolean digitsOnly = true;
         // int underscores = 0;
         // boolean underscoresOnly = true;
@@ -475,8 +476,8 @@ public final class Tokenizer {
 
         while (tokenEnd < source.length()) {
             if (substringSpecialTokens) {
-                substringSpecialToken = getSpecialToken(tokenEnd);
-                if (substringSpecialToken != null) break;
+                substringToken = getSpecialToken(tokenEnd);
+                if (substringToken != null) break;
             }
 
             int c = source.codePointAt(tokenEnd);
@@ -524,11 +525,11 @@ public final class Tokenizer {
             }
         }
 
-        if (substringSpecialToken == null)
+        if (substringToken == null)
             return --tokenEnd;
         // TODO: test the logic around tokenEnd with friends
-        addToken(substringSpecialToken.toToken(tokenEnd, source));
-        return --tokenEnd + substringSpecialToken.token().length();
+        addToken(toToken(substringToken, tokenEnd, source));
+        return --tokenEnd + substringToken.token().length();
     }
 
     private void addToken(Token.Kind kind, String word, int start, int end) {
@@ -539,4 +540,11 @@ public final class Tokenizer {
         tokens.add(token);
     }
 
+    public Token toToken(SpecialTokens.Token specialToken, int start, String rawSource) {
+        return new Token(Token.Kind.WORD,
+                         specialToken.replacement(),
+                         true,
+                         new Substring(start, start + specialToken.token().length(), rawSource)); // XXX: Unsafe?
+    }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
index 94b9bf6ce65..df96d314455 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
@@ -4,7 +4,7 @@ package com.yahoo.search.query.parser;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.search.Searcher;
 import com.yahoo.search.searchchain.Execution;
 
@@ -18,7 +18,7 @@ public final class ParserEnvironment {
 
     private IndexFacts indexFacts = new IndexFacts();
     private Linguistics linguistics = new SimpleLinguistics();
-    private SpecialTokens specialTokens = new SpecialTokens();
+    private SpecialTokens specialTokens = SpecialTokens.empty();
 
     public IndexFacts getIndexFacts() {
         return indexFacts;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
index 84fe88d0292..0574fc660c3 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
@@ -6,7 +6,7 @@ import com.yahoo.language.Linguistics;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.Ping;
 import com.yahoo.prelude.Pong;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.processing.Processor;
 import com.yahoo.processing.Request;
 import com.yahoo.processing.Response;
@@ -17,8 +17,6 @@ import com.yahoo.search.cluster.PingableSearcher;
 import com.yahoo.search.rendering.RendererRegistry;
 import com.yahoo.search.statistics.TimeTracker;
 
-import java.util.logging.Logger;
-
 /**
  * <p>An execution of a search chain. This keeps track of the call state for an execution (in the calling thread)
  * of the searchers of a search chain.</p>
@@ -111,7 +109,7 @@ public class Execution extends com.yahoo.processing.execution.Execution {
         public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts,
                        SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics)
         {
-            owner=null;
+            owner = null;
             // The next time something is added here, compose into wrapper objects. Many arguments...
 
             // Four methods need to be updated when adding something:
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
index 31b6d06f78e..a813229c984 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
@@ -13,7 +13,7 @@ import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.processing.rendering.Renderer;
 import com.yahoo.search.Searcher;
 import com.yahoo.search.config.IndexInfoConfig;
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
index 6afea895f3a..cef8ae1751c 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
@@ -18,16 +18,14 @@ import com.yahoo.prelude.query.PhraseSegmentItem;
 import com.yahoo.prelude.query.PrefixItem;
 import com.yahoo.prelude.query.RankItem;
 import com.yahoo.prelude.query.SubstringItem;
-import com.yahoo.prelude.query.SubstringItem;
 import com.yahoo.prelude.query.SuffixItem;
 import com.yahoo.prelude.query.TaggableItem;
 import com.yahoo.prelude.query.WordItem;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.query.parser.TestLinguistics;
 import com.yahoo.search.Query;
 import org.junit.Test;
 
-import java.util.Collections;
 import java.util.Iterator;
 
 import static org.junit.Assert.assertEquals;
@@ -1639,7 +1637,7 @@ public class ParseTestCase {
 
     @Test
     public void testNonSpecialTokenParsing() {
-        ParsingTester customTester = new ParsingTester(new SpecialTokens("default"));
+        ParsingTester customTester = new ParsingTester(SpecialTokens.empty());
         customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY);
     }
 
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
index 17155fff5de..fd7e4cbe0e6 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
@@ -11,8 +11,8 @@ import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.NullItem;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.search.Query;
 import com.yahoo.search.config.IndexInfoConfig;
 import com.yahoo.search.query.parser.Parsable;
@@ -20,6 +20,9 @@ import com.yahoo.search.query.parser.Parser;
 import com.yahoo.search.query.parser.ParserEnvironment;
 import com.yahoo.search.query.parser.ParserFactory;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -32,7 +35,7 @@ import static org.junit.Assert.assertTrue;
 public class ParsingTester {
 
     private static final Linguistics linguistics = new SimpleLinguistics();
-    private IndexFacts indexFacts;
+    private final IndexFacts indexFacts;
     private SpecialTokenRegistry tokenRegistry;
 
     public ParsingTester() {
@@ -49,11 +52,10 @@ public class ParsingTester {
 
     public ParsingTester(IndexFacts indexFacts, SpecialTokens specialTokens) {
         indexFacts.freeze();
-        specialTokens.freeze();
 
         this.indexFacts = indexFacts;
         tokenRegistry = new SpecialTokenRegistry();
-        tokenRegistry.addSpecialTokens(specialTokens);
+        tokenRegistry = new SpecialTokenRegistry(List.of(specialTokens));
     }
 
     /**
@@ -72,13 +74,13 @@ public class ParsingTester {
      * This can be used to add new tokens and passing the resulting special tokens to the constructor of this.
      */
     public static SpecialTokens createSpecialTokens() {
-        SpecialTokens tokens = new SpecialTokens("default");
-        tokens.addSpecialToken("c++", null);
-        tokens.addSpecialToken(".net", "dotnet");
-        tokens.addSpecialToken("tcp/ip", null);
-        tokens.addSpecialToken("c#", null);
-        tokens.addSpecialToken("special-token-fs","firstsecond");
-        return tokens;
+        List<SpecialTokens.Token> tokens = new ArrayList<>();
+        tokens.add(new SpecialTokens.Token("c++"));
+        tokens.add(new SpecialTokens.Token(".net", "dotnet"));
+        tokens.add(new SpecialTokens.Token("tcp/ip"));
+        tokens.add(new SpecialTokens.Token("c#"));
+        tokens.add(new SpecialTokens.Token("special-token-fs","firstsecond"));
+        return new SpecialTokens("default", tokens);
     }
 
     /**
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
index aa2e9dbcf75..e10fbd71c72 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
@@ -6,12 +6,13 @@ import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.query.parser.Token;
 import com.yahoo.prelude.query.parser.Tokenizer;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -39,13 +40,11 @@ import static org.junit.Assert.assertTrue;
  */
 public class TokenizerTestCase {
 
-    private SpecialTokenRegistry defaultRegistry = new SpecialTokenRegistry("file:src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg");
-
     @Test
     public void testPlainTokenization() {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(createSpecialTokens());
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
         List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1 & &a");
 
         assertEquals(new Token(WORD, "drive"), tokens.get(0));
@@ -87,7 +86,7 @@ public class TokenizerTestCase {
     public void testOneSpecialToken() {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(createSpecialTokens());
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
         List<?> tokens = tokenizer.tokenize("c++ lovers, please apply");
 
         assertEquals(new Token(WORD, "c++"), tokens.get(0));
@@ -97,7 +96,7 @@ public class TokenizerTestCase {
     public void testSpecialTokenCombination() {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(createSpecialTokens());
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
         List<?> tokens = tokenizer.tokenize("c#, c++ or .net know, not tcp/ip");
 
         assertEquals(new Token(WORD, "c#"), tokens.get(0));
@@ -123,10 +122,9 @@ public class TokenizerTestCase {
      */
     @Test
     public void testSpecialTokenCJK() {
-        assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size());
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
         tokenizer.setSubstringSpecialTokens(true);
-        tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default"));
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing"));
 
         List<?> tokens = tokenizer.tokenize("fooc#bar,c++with spacebarknowknowknow,knowknownot know");
         assertEquals(new Token(WORD, "foo"), tokens.get(0));
@@ -151,7 +149,7 @@ public class TokenizerTestCase {
     public void testSpecialTokenCaseInsensitive() {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(createSpecialTokens());
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
         List<?> tokens = tokenizer.tokenize("The AS/400 is great");
 
         assertEquals(new Token(WORD, "The"), tokens.get(0));
@@ -167,7 +165,7 @@ public class TokenizerTestCase {
     public void testSpecialTokenNonMatch() {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(createSpecialTokens());
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
         List<?> tokens = tokenizer.tokenize("c++ c+ aS/400 i/o .net i/ooo ap.net");
 
         assertEquals(new Token(WORD, "c++"), tokens.get(0));
@@ -190,18 +188,9 @@ public class TokenizerTestCase {
 
     @Test
     public void testSpecialTokenConfigurationDefault() {
-        String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg";
-
-        SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile);
-        assertEquals("Special tokens configured", 6,
-                r.getSpecialTokens("default").size());
-        assertEquals("Special tokens configured", 4,
-                r.getSpecialTokens("other").size());
-
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(
-                r.getSpecialTokens("default"));
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
         List<?> tokens = tokenizer.tokenize(
                 "with space, c++ or .... know, not b.s.d.");
 
@@ -224,18 +213,9 @@ public class TokenizerTestCase {
 
     @Test
     public void testSpecialTokenConfigurationOther() {
-        String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg";
-
-        SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile);
-        assertEquals("Special tokens configured", 6,
-                r.getSpecialTokens("default").size());
-        assertEquals("Special tokens configured", 4,
-                r.getSpecialTokens("other").size());
-
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(
-                r.getSpecialTokens("other"));
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("other"));
         List<?> tokens = tokenizer.tokenize(
                 "with space,!!!*** [huh] or ------ " + "know, &&&%%% b.s.d.");
 
@@ -267,26 +247,9 @@ public class TokenizerTestCase {
     }
 
     @Test
-    public void testSpecialTokenConfigurationMissing() {
-        String tokenFile = "file:source/bogus/specialtokens.cfg";
-
-        SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile);
-
-        Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
-
-        tokenizer.setSpecialTokens(r.getSpecialTokens("other"));
-        List<?> tokens = tokenizer.tokenize("c++");
-
-        assertEquals(new Token(WORD, "c"), tokens.get(0));
-        assertEquals(new Token(PLUS, "+"), tokens.get(1));
-        assertEquals(new Token(PLUS, "+"), tokens.get(2));
-    }
-
-    @Test
     public void testTokenReplacing() {
-        assertEquals("Special tokens configured", 6, defaultRegistry.getSpecialTokens("default").size());
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
-        tokenizer.setSpecialTokens(defaultRegistry.getSpecialTokens("default"));
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing"));
 
         List<?> tokens = tokenizer.tokenize("with space, c++ or .... know, not b.s.d.");
         assertEquals(new Token(WORD, "with-space"), tokens.get(0));
@@ -745,7 +708,7 @@ public class TokenizerTestCase {
     public void testSingleQuoteAsWordCharacter() {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
-        tokenizer.setSpecialTokens(createSpecialTokens());
+        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
         List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en nalle:a'a ugcapi_1 'a' 'a a'");
 
         assertEquals(new Token(WORD, "drive"), tokens.get(0));
@@ -781,17 +744,38 @@ public class TokenizerTestCase {
         assertEquals(new Token(WORD, "a'"), tokens.get(30));
     }
 
-    private SpecialTokens createSpecialTokens() {
-        SpecialTokens tokens = new SpecialTokens("default");
-
-        tokens.addSpecialToken("c+", null);
-        tokens.addSpecialToken("c++", null);
-        tokens.addSpecialToken(".net", null);
-        tokens.addSpecialToken("tcp/ip", null);
-        tokens.addSpecialToken("i/o", null);
-        tokens.addSpecialToken("c#", null);
-        tokens.addSpecialToken("AS/400", null);
-        return tokens;
+    private SpecialTokenRegistry createSpecialTokens() {
+        List<SpecialTokens.Token> tokens = new ArrayList<>();
+        tokens.add(new SpecialTokens.Token("c+"));
+        tokens.add(new SpecialTokens.Token("c++"));
+        tokens.add(new SpecialTokens.Token(".net"));
+        tokens.add(new SpecialTokens.Token("tcp/ip"));
+        tokens.add(new SpecialTokens.Token("i/o"));
+        tokens.add(new SpecialTokens.Token("c#"));
+        tokens.add(new SpecialTokens.Token("AS/400"));
+        tokens.add(new SpecialTokens.Token("...."));
+        tokens.add(new SpecialTokens.Token("b.s.d."));
+        tokens.add(new SpecialTokens.Token("with space"));
+        tokens.add(new SpecialTokens.Token("dvd\\xB1r"));
+        SpecialTokens defaultTokens = new SpecialTokens("default", tokens);
+
+        tokens = new ArrayList<>();
+        tokens.add(new SpecialTokens.Token("[huh]"));
+        tokens.add(new SpecialTokens.Token("&&&%%%"));
+        tokens.add(new SpecialTokens.Token("------"));
+        tokens.add(new SpecialTokens.Token("!!!***"));
+        SpecialTokens otherTokens = new SpecialTokens("other", tokens);
+
+        tokens = new ArrayList<>();
+        tokens.add(new SpecialTokens.Token("...."));
+        tokens.add(new SpecialTokens.Token("c++", "cpp"));
+        tokens.add(new SpecialTokens.Token("b.s.d."));
+        tokens.add(new SpecialTokens.Token("with space", "with-space"));
+        tokens.add(new SpecialTokens.Token("c#"));
+        tokens.add(new SpecialTokens.Token("know", "knuwww"));
+        SpecialTokens replacingTokens = new SpecialTokens("replacing", tokens);
+
+        return new SpecialTokenRegistry(List.of(defaultTokens, otherTokens, replacingTokens));
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg
deleted file mode 100644
index 6a189de0164..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/replacingtokens.cfg
+++ /dev/null
@@ -1,12 +0,0 @@
-tokenlist[1]
-tokenlist[0].name default
-tokenlist[0].tokens[6]
-tokenlist[0].tokens[0].token ....
-tokenlist[0].tokens[1].token c++
-tokenlist[0].tokens[1].replace cpp
-tokenlist[0].tokens[2].token b.s.d.
-tokenlist[0].tokens[3].token with space
-tokenlist[0].tokens[3].replace with-space
-tokenlist[0].tokens[4].token c#
-tokenlist[0].tokens[5].token know
-tokenlist[0].tokens[5].replace knuwww
diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java
index 5508c2a73a7..08146bbe069 100644
--- a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java
@@ -8,7 +8,7 @@ import org.junit.Test;
 import com.yahoo.prelude.query.AndItem;
 import com.yahoo.prelude.query.CompositeItem;
 import com.yahoo.prelude.query.Item;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.search.Query;
 import com.yahoo.search.searchchain.Execution;
 import com.yahoo.search.searchchain.Execution.Context;
diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json
index 58b838d7332..b77b03664d4 100644
--- a/linguistics/abi-spec.json
+++ b/linguistics/abi-spec.json
@@ -427,6 +427,57 @@
     ],
     "fields": []
   },
+  "com.yahoo.language.process.SpecialTokenRegistry": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>()",
+      "public void <init>(com.yahoo.vespa.configdefinition.SpecialtokensConfig)",
+      "public void <init>(java.util.List)",
+      "public com.yahoo.language.process.SpecialTokens getSpecialTokens(java.lang.String)"
+    ],
+    "fields": []
+  },
+  "com.yahoo.language.process.SpecialTokens$Token": {
+    "superClass": "java.lang.Object",
+    "interfaces": [
+      "java.lang.Comparable"
+    ],
+    "attributes": [
+      "public",
+      "final"
+    ],
+    "methods": [
+      "public void <init>(java.lang.String)",
+      "public void <init>(java.lang.String, java.lang.String)",
+      "public java.lang.String token()",
+      "public java.lang.String replacement()",
+      "public int compareTo(com.yahoo.language.process.SpecialTokens$Token)",
+      "public boolean equals(java.lang.Object)",
+      "public int hashCode()",
+      "public java.lang.String toString()",
+      "public bridge synthetic int compareTo(java.lang.Object)"
+    ],
+    "fields": []
+  },
+  "com.yahoo.language.process.SpecialTokens": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>(java.lang.String, java.util.List)",
+      "public java.lang.String name()",
+      "public java.util.Map asMap()",
+      "public com.yahoo.language.process.SpecialTokens$Token tokenize(java.lang.String, boolean)",
+      "public static com.yahoo.language.process.SpecialTokens empty()"
+    ],
+    "fields": []
+  },
   "com.yahoo.language.process.StemList": {
     "superClass": "java.util.AbstractList",
     "interfaces": [],
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
index e1185cb2457..73518876c3f 100644
--- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
@@ -4,6 +4,7 @@ package com.yahoo.language.opennlp;
 import com.yahoo.language.Language;
 import com.yahoo.language.LinguisticsCase;
 import com.yahoo.language.process.Normalizer;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.language.process.StemMode;
 import com.yahoo.language.process.Token;
 import com.yahoo.language.process.TokenType;
@@ -32,15 +33,21 @@ public class OpenNlpTokenizer implements Tokenizer {
     private final Normalizer normalizer;
     private final Transformer transformer;
     private final SimpleTokenizer simpleTokenizer;
+    private final SpecialTokenRegistry specialTokenRegistry;
 
     public OpenNlpTokenizer() {
         this(new SimpleNormalizer(), new SimpleTransformer());
     }
 
     public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) {
+        this(normalizer, transformer, new SpecialTokenRegistry(List.of()));
+    }
+
+    public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) {
         this.normalizer = normalizer;
         this.transformer = transformer;
-        simpleTokenizer = new SimpleTokenizer(normalizer, transformer);
+        this.specialTokenRegistry = specialTokenRegistry;
+        this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry);
     }
 
     @Override
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java
new file mode 100644
index 00000000000..b6335d67967
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java
@@ -0,0 +1,72 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.process;
+
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Immutable named lists of "special tokens" - strings which should override the normal tokenizer semantics
+ * and be tokenized into a single token.
+ *
+ * @author bratseth
+ */
+public class SpecialTokenRegistry {
+
+    /**
+     * The current special token lists, indexed on name.
+     * These lists are unmodifiable and used directly by clients of this
+     */
+    private final Map<String, SpecialTokens> specialTokenMap;
+
+    /** Creates an empty special token registry */
+    public SpecialTokenRegistry() {
+        this(List.of());
+    }
+
+    /** Create a special token registry from a configuration object. */
+    public SpecialTokenRegistry(SpecialtokensConfig config) {
+        this(specialTokensFrom(config));
+    }
+
+    public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) {
+        specialTokenMap = specialTokensList.stream().collect(Collectors.toUnmodifiableMap(t -> t.name(), t -> t));
+    }
+
+    private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) {
+        List<SpecialTokens> specialTokensList = new ArrayList<>();
+        for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
+            Tokenlist tokenListConfig = i.next();
+
+            List<SpecialTokens.Token> tokenList = new ArrayList<>();
+            for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) {
+                Tokens tokenConfig = j.next();
+                tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace()));
+            }
+            specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList));
+        }
+        return specialTokensList;
+    }
+
+    /**
+     * Returns the list of special tokens for a given name.
+     *
+     * @param  name the name of the special tokens to return
+     *         null, the empty string or the string "default" returns
+     *         the default ones
+     * @return a read-only list of SpecialToken instances, an empty list if this name
+     *         has no special tokens
+     */
+    public SpecialTokens getSpecialTokens(String name) {
+        if (name == null || name.trim().equals(""))
+            name = "default";
+        return specialTokenMap.getOrDefault(name, SpecialTokens.empty());
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java
new file mode 100644
index 00000000000..465d9b754b3
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java
@@ -0,0 +1,141 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.process;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * An immutable list of special tokens - strings which should override the normal tokenizer semantics
+ * and be tokenized into a single token. Special tokens are case insensitive.
+ *
+ * @author bratseth
+ */
+public class SpecialTokens {
+
+    private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of());
+
+    private final String name;
+    private final int maximumLength;
+    private final List<Token> tokens;
+    private final Map<String, String> tokenMap;
+
+    public SpecialTokens(String name,  List<Token> tokens) {
+        tokens.stream().peek(token -> token.validate());
+        List<Token> mutableTokens = new ArrayList<>(tokens);
+        Collections.sort(mutableTokens);
+        this.name = name;
+        this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0);
+        this.tokens = List.copyOf(mutableTokens);
+        this.tokenMap = tokens.stream().collect(Collectors.toUnmodifiableMap(t -> t.token(), t -> t.replacement()));
+    }
+
+    /** Returns the name of this special tokens list */
+    public String name() {
+        return name;
+    }
+
+    /**
+     * Returns the tokens of this as an immutable map from token to replacement.
+     * Tokens which do not have a replacement token maps to themselves.
+     */
+    public Map<String, String> asMap() { return tokenMap; }
+
+    /**
+     * Returns the special token starting at the start of the given string, or null if no
+     * special token starts at this string
+     *
+     * @param string the string to search for a special token at the start position
+     * @param substring true to allow the special token to be followed by a character which does not
+     *        mark the end of a token
+     */
+    public Token tokenize(String string, boolean substring) {
+        // XXX detonator pattern token.length may be != the length of the
+        // matching data in string, ref caseIndependentLength(String)
+        String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength)));
+        for (Iterator<Token> i = tokens.iterator(); i.hasNext();) {
+            Token special = i.next();
+
+            if (input.startsWith(special.token())) {
+                if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string))
+                    return special;
+            }
+        }
+        return null;
+    }
+
+    private boolean tokenEndsAt(int position, String string) {
+        return !Character.isLetterOrDigit(string.charAt(position));
+    }
+
+    public static SpecialTokens empty() { return empty; }
+
+    /** An immutable special token */
+    public final static class Token implements Comparable<Token> {
+
+        private final String token;
+        private final String replacement;
+
+        /** Creates a special token */
+        public Token(String token) {
+            this(token, null);
+        }
+
+        /** Creates a special token which will be represented by the given replacement token */
+        public Token(String token, String replacement) {
+            this.token = toLowerCase(token);
+            if (replacement == null || replacement.trim().equals(""))
+                this.replacement = this.token;
+            else
+                this.replacement = toLowerCase(replacement);
+        }
+
+        /** Returns the special token */
+        public String token() { return token; }
+
+        /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */
+        public String replacement() { return replacement; }
+
+        @Override
+        public int compareTo(Token other) {
+            if (this.token().length() < other.token().length()) return 1;
+            if (this.token().length() == other.token().length()) return 0;
+            return -1;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (other == this) return true;
+            if ( ! (other instanceof Token)) return false;
+            return Objects.equals(this.token, ((Token)other).token);
+        }
+
+        @Override
+        public int hashCode() { return token.hashCode(); }
+
+        @Override
+        public String toString() {
+            return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'");
+        }
+
+        private void validate() {
+            // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
+            String asLow = toLowerCase(token);
+            // TODO: Put along with the global toLowerCase
+            String asHigh = token.toUpperCase(Locale.ENGLISH);
+            if (asLow.length() != token.length() || asHigh.length() != token.length()) {
+                throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " +
+                                                   "Please report this to the Vespa team.");
+            }
+        }
+
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java
index 57a5b6edb68..ad154d1b003 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java
@@ -4,7 +4,7 @@ package com.yahoo.language.process;
 /**
  * An enumeration of token types.
  *
- * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a>
+ * @author Mathias Mølster Lidal
  */
 public enum TokenType {
 
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
index e1a04b2985d..4ffe2a866d8 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
@@ -11,10 +11,14 @@ import com.yahoo.language.process.GramSplitter;
 import com.yahoo.language.process.Normalizer;
 import com.yahoo.language.process.Segmenter;
 import com.yahoo.language.process.SegmenterImpl;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.language.process.Stemmer;
 import com.yahoo.language.process.StemmerImpl;
 import com.yahoo.language.process.Tokenizer;
 import com.yahoo.language.process.Transformer;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+
+import java.util.List;
 
 /**
  * Factory of simple linguistic processor implementations.
@@ -31,6 +35,7 @@ public class SimpleLinguistics implements Linguistics {
     private final Detector detector;
     private final CharacterClasses characterClasses;
     private final GramSplitter gramSplitter;
+    private final SpecialTokenRegistry specialTokenRegistry = new SpecialTokenRegistry(List.of());
 
     @Inject
     public SimpleLinguistics() {
@@ -45,7 +50,7 @@ public class SimpleLinguistics implements Linguistics {
     public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); }
 
     @Override
-    public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer); }
+    public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); }
 
     @Override
     public Normalizer getNormalizer() { return normalizer; }
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
index 7df432f496d..740307c0cca 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
@@ -23,11 +23,13 @@ import java.util.logging.Level;
  */
 public class SimpleTokenizer implements Tokenizer {
 
+    private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName());
     private final static int SPACE_CODE = 32;
+
     private final Normalizer normalizer;
     private final Transformer transformer;
     private final KStemmer stemmer = new KStemmer();
-    private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName());
+    private final SpecialTokenRegistry specialTokenRegistry;
 
     public SimpleTokenizer() {
         this(new SimpleNormalizer(), new SimpleTransformer());
@@ -38,8 +40,13 @@ public class SimpleTokenizer implements Tokenizer {
     }
 
     public SimpleTokenizer(Normalizer normalizer, Transformer transformer) {
+        this(normalizer, transformer, new SpecialTokenRegistry(List.of()));
+    }
+
+    public SimpleTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) {
         this.normalizer = normalizer;
         this.transformer = transformer;
+        this.specialTokenRegistry = specialTokenRegistry;
     }
 
     @Override
@@ -56,8 +63,8 @@ public class SimpleTokenizer implements Tokenizer {
                 String original = input.substring(prev, next);
                 String token = processToken(original, language, stemMode, removeAccents);
                 tokens.add(new SimpleToken(original).setOffset(prev)
-                                                .setType(prevType)
-                                                .setTokenString(token));
+                                                    .setType(prevType)
+                                                    .setTokenString(token));
                 prev = next;
                 prevType = nextType;
             }
@@ -67,20 +74,20 @@ public class SimpleTokenizer implements Tokenizer {
     }
 
     private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) {
-        final String original = token;
-        log.log(Level.FINEST, () -> "processToken '"+original+"'");
+        String original = token;
+        log.log(Level.FINEST, () -> "processToken '" + original + "'");
         token = normalizer.normalize(token);
         token = LinguisticsCase.toLowerCase(token);
         if (removeAccents)
             token = transformer.accentDrop(token, language);
         if (stemMode != StemMode.NONE) {
-            final String oldToken = token;
+            String oldToken = token;
             token = stemmer.stem(token);
-            final String newToken = token;
-            log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'");
+            String newToken = token;
+            log.log(Level.FINEST, () -> "stem '" + oldToken+"' to '" + newToken+"'");
         }
-        final String result = token;
-        log.log(Level.FINEST, () -> "processed token is: "+result);
+        String result = token;
+        log.log(Level.FINEST, () -> "processed token is: " + result);
         return result;
     }
 
diff --git a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java
new file mode 100644
index 00000000000..47c3ba7933c
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java
@@ -0,0 +1,40 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.process;
+
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class SpecialTokensTestCase {
+
+    @Test
+    public void testSpecialTokensConfig() {
+        var builder = new SpecialtokensConfig.Builder();
+        var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder();
+        tokenBuilder.name("default");
+
+        var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
+        tokenListBuilder1.token("c++");
+        tokenListBuilder1.replace("cpp");
+        tokenBuilder.tokens(tokenListBuilder1);
+
+        var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
+        tokenListBuilder2.token("...");
+        tokenBuilder.tokens(tokenListBuilder2);
+
+        builder.tokenlist(tokenBuilder);
+
+        var registry = new SpecialTokenRegistry(builder.build());
+
+        var defaultTokens = registry.getSpecialTokens("default");
+        assertEquals("default", defaultTokens.name());
+        assertEquals(2, defaultTokens.asMap().size());
+        assertEquals("cpp", defaultTokens.asMap().get("c++"));
+        assertEquals("...", defaultTokens.asMap().get("..."));
+    }
+
+}