Move specialtokens to linguistics

11 files changed, 15 insertions, 267 deletions

diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index b5933936adf..74ed9d33f04 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -5574,8 +5574,8 @@
       "public com.yahoo.search.query.parser.ParserEnvironment setIndexFacts(com.yahoo.prelude.IndexFacts)",
       "public com.yahoo.language.Linguistics getLinguistics()",
       "public com.yahoo.search.query.parser.ParserEnvironment setLinguistics(com.yahoo.language.Linguistics)",
-      "public com.yahoo.prelude.query.parser.SpecialTokens getSpecialTokens()",
-      "public com.yahoo.search.query.parser.ParserEnvironment setSpecialTokens(com.yahoo.prelude.query.parser.SpecialTokens)",
+      "public com.yahoo.language.process.SpecialTokens getSpecialTokens()",
+      "public com.yahoo.search.query.parser.ParserEnvironment setSpecialTokens(com.yahoo.language.process.SpecialTokens)",
       "public static com.yahoo.search.query.parser.ParserEnvironment fromExecutionContext(com.yahoo.search.searchchain.Execution$Context)",
       "public static com.yahoo.search.query.parser.ParserEnvironment fromParserEnvironment(com.yahoo.search.query.parser.ParserEnvironment)"
     ],
@@ -7765,7 +7765,7 @@
       "final"
     ],
     "methods": [
-      "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.prelude.query.parser.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)",
+      "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)",
       "public static com.yahoo.search.searchchain.Execution$Context createContextStub()",
       "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts)",
       "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts)",
@@ -7779,8 +7779,8 @@
       "public void setIndexFacts(com.yahoo.prelude.IndexFacts)",
       "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()",
       "public com.yahoo.search.rendering.RendererRegistry rendererRegistry()",
-      "public com.yahoo.prelude.query.parser.SpecialTokenRegistry getTokenRegistry()",
-      "public void setTokenRegistry(com.yahoo.prelude.query.parser.SpecialTokenRegistry)",
+      "public com.yahoo.language.process.SpecialTokenRegistry getTokenRegistry()",
+      "public void setTokenRegistry(com.yahoo.language.process.SpecialTokenRegistry)",
       "public void setDetailedDiagnostics(boolean)",
       "public boolean getDetailedDiagnostics()",
       "public boolean getBreakdown()",
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
deleted file mode 100644
index 9c735a031d7..00000000000
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.query.parser;
-
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * A registry which is responsible for knowing the current
- * set of special tokens.Usage of this registry is multithread safe.
- *
- * @author bratseth
- */
-public class SpecialTokenRegistry {
-
-    /**
-     * The current special token lists, indexed on name.
-     * These lists are unmodifiable and used directly by clients of this
-     */
-    private Map<String, SpecialTokens> specialTokenMap;
-
-    private boolean frozen = false;
-
-    /** Creates an empty special token registry */
-    public SpecialTokenRegistry() {
-        this(List.of());
-    }
-
-    /** Create a special token registry from a configuration object. */
-    public SpecialTokenRegistry(SpecialtokensConfig config) {
-        this(specialTokensFrom(config));
-    }
-
-    public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) {
-        specialTokenMap = specialTokensList.stream().collect(Collectors.toMap(t -> t.name(), t -> t));
-        freeze();
-    }
-
-    private void freeze() {
-        frozen = true;
-    }
-
-    private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) {
-        List<SpecialTokens> specialTokensList = new ArrayList<>();
-        for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
-            Tokenlist tokenListConfig = i.next();
-
-            List<SpecialTokens.Token> tokenList = new ArrayList<>();
-            for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) {
-                Tokens tokenConfig = j.next();
-                tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace()));
-            }
-            specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList));
-        }
-        return specialTokensList;
-    }
-
-    private void ensureNotFrozen() {
-        if (frozen) {
-            throw new IllegalStateException("Tried to modify a frozen SpecialTokenRegistry instance.");
-        }
-    }
-
-    /**
-     * Returns the list of special tokens for a given name.
-     *
-     * @param  name the name of the special tokens to return
-     *         null, the empty string or the string "default" returns
-     *         the default ones
-     * @return a read-only list of SpecialToken instances, an empty list if this name
-     *         has no special tokens
-     */
-    public SpecialTokens getSpecialTokens(String name) {
-        if (name == null || name.trim().equals(""))
-            name = "default";
-        return specialTokenMap.getOrDefault(name, SpecialTokens.empty());
-    }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
deleted file mode 100644
index 4b29b50f095..00000000000
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
+++ /dev/null
@@ -1,136 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.query.parser;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
-
-import com.yahoo.prelude.query.Substring;
-
-import static com.yahoo.language.LinguisticsCase.toLowerCase;
-
-/**
- * A list of special tokens - string that should be treated as word
- * no matter what they contain. Special tokens are case insensitive.
- *
- * @author bratseth
- */
-public class SpecialTokens {
-
-    private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of());
-
-    private final String name;
-    private final List<Token> tokens;
-    private final int maximumLength;
-
-    public SpecialTokens(String name,  List<Token> tokens) {
-        tokens.stream().peek(token -> token.validate());
-        List<Token> mutableTokens = new ArrayList<>(tokens);
-        Collections.sort(mutableTokens);
-        this.tokens = List.copyOf(mutableTokens);
-        this.name = name;
-        this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0);
-    }
-
-    /** Returns the name of this special tokens list */
-    public String name() {
-        return name;
-    }
-
-    /** Returns a sorted immutable list of the special tokens in this */
-    public List<Token> tokens() { return tokens; }
-
-    /**
-     * Returns the special token starting at the start of the given string, or null if no
-     * special token starts at this string
-     *
-     * @param string the string to search for a special token at the start position
-     * @param substring true to allow the special token to be followed by a character which does not
-     *        mark the end of a token
-     */
-    public Token tokenize(String string, boolean substring) {
-        // XXX detonator pattern token.length may be != the length of the
-        // matching data in string, ref caseIndependentLength(String)
-        String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength)));
-        for (Iterator<Token> i = tokens.iterator(); i.hasNext();) {
-            Token special = i.next();
-
-            if (input.startsWith(special.token())) {
-                if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string))
-                    return special;
-            }
-        }
-        return null;
-    }
-
-    private boolean tokenEndsAt(int position,String string) {
-        return !Character.isLetterOrDigit(string.charAt(position));
-    }
-
-    public static SpecialTokens empty() { return empty; }
-
-    /** An immutable special token */
-    public final static class Token implements Comparable<Token> {
-
-        private final String token;
-        private final String replacement;
-
-        /** Creates a special token */
-        public Token(String token) {
-            this(token, null);
-        }
-
-        /** Creates a special token which will be represented by the given replacement token */
-        public Token(String token, String replacement) {
-            this.token = toLowerCase(token);
-            if (replacement == null || replacement.trim().equals(""))
-                this.replacement = this.token;
-            else
-                this.replacement = toLowerCase(replacement);
-        }
-
-        /** Returns the special token */
-        public String token() { return token; }
-
-        /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */
-        public String replacement() { return replacement; }
-
-        @Override
-        public int compareTo(Token other) {
-            if (this.token().length() < other.token().length()) return 1;
-            if (this.token().length() == other.token().length()) return 0;
-            return -1;
-        }
-
-        @Override
-        public boolean equals(Object other) {
-            if (other == this) return true;
-            if ( ! (other instanceof Token)) return false;
-            return Objects.equals(this.token, ((Token)other).token);
-        }
-
-        @Override
-        public int hashCode() { return token.hashCode(); }
-
-        @Override
-        public String toString() {
-            return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'");
-        }
-
-        private void validate() {
-            // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
-            String asLow = toLowerCase(token);
-            // TODO: Put along with the global toLowerCase
-            String asHigh = token.toUpperCase(Locale.ENGLISH);
-            if (asLow.length() != token.length() || asHigh.length() != token.length()) {
-                throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " +
-                                                   "Please report this to the Vespa team.");
-            }
-        }
-
-    }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
index f0656efa59a..b71bd57539f 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
@@ -3,6 +3,7 @@ package com.yahoo.prelude.query.parser;
 
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.CharacterClasses;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.query.Substring;
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
index 1b35296082e..df96d314455 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
@@ -4,7 +4,7 @@ package com.yahoo.search.query.parser;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.search.Searcher;
 import com.yahoo.search.searchchain.Execution;
 
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
index 7adfccc2ed1..1954e6e657e 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
@@ -6,7 +6,7 @@ import com.yahoo.language.Linguistics;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.Ping;
 import com.yahoo.prelude.Pong;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.processing.Processor;
 import com.yahoo.processing.Request;
 import com.yahoo.processing.Response;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
index 31b6d06f78e..a813229c984 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
@@ -13,7 +13,7 @@ import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.processing.rendering.Renderer;
 import com.yahoo.search.Searcher;
 import com.yahoo.search.config.IndexInfoConfig;
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
index eb86af993d7..cef8ae1751c 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
@@ -18,16 +18,14 @@ import com.yahoo.prelude.query.PhraseSegmentItem;
 import com.yahoo.prelude.query.PrefixItem;
 import com.yahoo.prelude.query.RankItem;
 import com.yahoo.prelude.query.SubstringItem;
-import com.yahoo.prelude.query.SubstringItem;
 import com.yahoo.prelude.query.SuffixItem;
 import com.yahoo.prelude.query.TaggableItem;
 import com.yahoo.prelude.query.WordItem;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.query.parser.TestLinguistics;
 import com.yahoo.search.Query;
 import org.junit.Test;
 
-import java.util.Collections;
 import java.util.Iterator;
 
 import static org.junit.Assert.assertEquals;
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
index a17d791f906..fd7e4cbe0e6 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
@@ -11,8 +11,8 @@ import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.NullItem;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.search.Query;
 import com.yahoo.search.config.IndexInfoConfig;
 import com.yahoo.search.query.parser.Parsable;
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
index 5634cf67212..e10fbd71c72 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
@@ -6,11 +6,10 @@ import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.query.parser.Token;
 import com.yahoo.prelude.query.parser.Tokenizer;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -745,34 +744,6 @@ public class TokenizerTestCase {
         assertEquals(new Token(WORD, "a'"), tokens.get(30));
     }
 
-    @Test
-    public void testSpecialTokensConfig() {
-        var builder = new SpecialtokensConfig.Builder();
-        var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder();
-        tokenBuilder.name("default");
-
-        var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
-        tokenListBuilder1.token("c++");
-        tokenListBuilder1.replace("cpp");
-        tokenBuilder.tokens(tokenListBuilder1);
-
-        var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
-        tokenListBuilder2.token("...");
-        tokenBuilder.tokens(tokenListBuilder2);
-
-        builder.tokenlist(tokenBuilder);
-
-        var registry = new SpecialTokenRegistry(builder.build());
-
-        var defaultTokens = registry.getSpecialTokens("default");
-        assertEquals("default", defaultTokens.name());
-        assertEquals(2, defaultTokens.tokens().size());
-        assertEquals("c++", defaultTokens.tokens().get(0).token());
-        assertEquals("cpp", defaultTokens.tokens().get(0).replacement());
-        assertEquals("...", defaultTokens.tokens().get(1).token());
-        assertEquals("...", defaultTokens.tokens().get(1).replacement());
-    }
-
     private SpecialTokenRegistry createSpecialTokens() {
         List<SpecialTokens.Token> tokens = new ArrayList<>();
         tokens.add(new SpecialTokens.Token("c+"));
diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java
index 5508c2a73a7..08146bbe069 100644
--- a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java
@@ -8,7 +8,7 @@ import org.junit.Test;
 import com.yahoo.prelude.query.AndItem;
 import com.yahoo.prelude.query.CompositeItem;
 import com.yahoo.prelude.query.Item;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.search.Query;
 import com.yahoo.search.searchchain.Execution;
 import com.yahoo.search.searchchain.Execution.Context;