Move specialtokens to linguistics

3 files changed, 5 insertions, 36 deletions

diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
index eb86af993d7..cef8ae1751c 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
@@ -18,16 +18,14 @@ import com.yahoo.prelude.query.PhraseSegmentItem;
 import com.yahoo.prelude.query.PrefixItem;
 import com.yahoo.prelude.query.RankItem;
 import com.yahoo.prelude.query.SubstringItem;
-import com.yahoo.prelude.query.SubstringItem;
 import com.yahoo.prelude.query.SuffixItem;
 import com.yahoo.prelude.query.TaggableItem;
 import com.yahoo.prelude.query.WordItem;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.query.parser.TestLinguistics;
 import com.yahoo.search.Query;
 import org.junit.Test;
 
-import java.util.Collections;
 import java.util.Iterator;
 
 import static org.junit.Assert.assertEquals;
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
index a17d791f906..fd7e4cbe0e6 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java
@@ -11,8 +11,8 @@ import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.NullItem;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.search.Query;
 import com.yahoo.search.config.IndexInfoConfig;
 import com.yahoo.search.query.parser.Parsable;
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
index 5634cf67212..e10fbd71c72 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
@@ -6,11 +6,10 @@ import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
-import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
-import com.yahoo.prelude.query.parser.SpecialTokens;
+import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.query.parser.Token;
 import com.yahoo.prelude.query.parser.Tokenizer;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -745,34 +744,6 @@ public class TokenizerTestCase {
         assertEquals(new Token(WORD, "a'"), tokens.get(30));
     }
 
-    @Test
-    public void testSpecialTokensConfig() {
-        var builder = new SpecialtokensConfig.Builder();
-        var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder();
-        tokenBuilder.name("default");
-
-        var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
-        tokenListBuilder1.token("c++");
-        tokenListBuilder1.replace("cpp");
-        tokenBuilder.tokens(tokenListBuilder1);
-
-        var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
-        tokenListBuilder2.token("...");
-        tokenBuilder.tokens(tokenListBuilder2);
-
-        builder.tokenlist(tokenBuilder);
-
-        var registry = new SpecialTokenRegistry(builder.build());
-
-        var defaultTokens = registry.getSpecialTokens("default");
-        assertEquals("default", defaultTokens.name());
-        assertEquals(2, defaultTokens.tokens().size());
-        assertEquals("c++", defaultTokens.tokens().get(0).token());
-        assertEquals("cpp", defaultTokens.tokens().get(0).replacement());
-        assertEquals("...", defaultTokens.tokens().get(1).token());
-        assertEquals("...", defaultTokens.tokens().get(1).replacement());
-    }
-
     private SpecialTokenRegistry createSpecialTokens() {
         List<SpecialTokens.Token> tokens = new ArrayList<>();
         tokens.add(new SpecialTokens.Token("c+"));