diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-05-04 16:17:07 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-05-04 16:17:07 +0200 |
commit | b399aa85883146aa3ba1396769d8e82c88877674 (patch) | |
tree | 5628548eb45d7ef6aed4561360dc51563cfd380e /container-search/src/test/java/com/yahoo | |
parent | 20d71c1dd96cd74803504f22df3f100b63e9d838 (diff) |
Move specialtokens to linguistics
Diffstat (limited to 'container-search/src/test/java/com/yahoo')
4 files changed, 6 insertions, 37 deletions
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java index eb86af993d7..cef8ae1751c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java @@ -18,16 +18,14 @@ import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.SubstringItem; -import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; import com.yahoo.prelude.query.TaggableItem; import com.yahoo.prelude.query.WordItem; -import com.yahoo.prelude.query.parser.SpecialTokens; +import com.yahoo.language.process.SpecialTokens; import com.yahoo.prelude.query.parser.TestLinguistics; import com.yahoo.search.Query; import org.junit.Test; -import java.util.Collections; import java.util.Iterator; import static org.junit.Assert.assertEquals; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java index a17d791f906..fd7e4cbe0e6 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParsingTester.java @@ -11,8 +11,8 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.parser.SpecialTokenRegistry; -import com.yahoo.prelude.query.parser.SpecialTokens; +import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.language.process.SpecialTokens; import com.yahoo.search.Query; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.query.parser.Parsable; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java index 5634cf67212..e10fbd71c72 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -6,11 +6,10 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; -import com.yahoo.prelude.query.parser.SpecialTokenRegistry; -import com.yahoo.prelude.query.parser.SpecialTokens; +import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.language.process.SpecialTokens; import com.yahoo.prelude.query.parser.Token; import com.yahoo.prelude.query.parser.Tokenizer; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import org.junit.Test; import java.util.ArrayList; @@ -745,34 +744,6 @@ public class TokenizerTestCase { assertEquals(new Token(WORD, "a'"), tokens.get(30)); } - @Test - public void testSpecialTokensConfig() { - var builder = new SpecialtokensConfig.Builder(); - var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder(); - tokenBuilder.name("default"); - - var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); - tokenListBuilder1.token("c++"); - tokenListBuilder1.replace("cpp"); - tokenBuilder.tokens(tokenListBuilder1); - - var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); - tokenListBuilder2.token("..."); - tokenBuilder.tokens(tokenListBuilder2); - - builder.tokenlist(tokenBuilder); - - var registry = new SpecialTokenRegistry(builder.build()); - - var defaultTokens = registry.getSpecialTokens("default"); - assertEquals("default", defaultTokens.name()); - assertEquals(2, defaultTokens.tokens().size()); - assertEquals("c++", defaultTokens.tokens().get(0).token()); - assertEquals("cpp", defaultTokens.tokens().get(0).replacement()); - assertEquals("...", defaultTokens.tokens().get(1).token()); - assertEquals("...", defaultTokens.tokens().get(1).replacement()); - } - private SpecialTokenRegistry createSpecialTokens() { List<SpecialTokens.Token> tokens = new ArrayList<>(); tokens.add(new SpecialTokens.Token("c+")); diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java index 5508c2a73a7..08146bbe069 100644 --- a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java @@ -8,7 +8,7 @@ import org.junit.Test; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.search.Query; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.Execution.Context; |