diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-05-05 21:18:50 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-05-05 21:18:50 +0200 |
commit | 346c5a523fec4ec7cecb373e480e30846f5e33e5 (patch) | |
tree | d8788eab6bb91d0b06a15d50f93f19d9d42ba96c /linguistics/src/test/java/com/yahoo/language | |
parent | a2c9cd4bc04f1a3eaa31524b3970b96be5c2eda9 (diff) |
Revert "Merge pull request #17754 from vespa-engine/revert-17747-bratseth/special-tokens-take-2"
This reverts commit a2c9cd4bc04f1a3eaa31524b3970b96be5c2eda9, reversing
changes made to 8c61a373af0066fbdf1cca354c24b197c7347321.
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language')
-rw-r--r-- | linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java | 40 |
1 files changed, 40 insertions, 0 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java new file mode 100644 index 00000000000..47c3ba7933c --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.process; + +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SpecialTokensTestCase { + + @Test + public void testSpecialTokensConfig() { + var builder = new SpecialtokensConfig.Builder(); + var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder(); + tokenBuilder.name("default"); + + var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); + tokenListBuilder1.token("c++"); + tokenListBuilder1.replace("cpp"); + tokenBuilder.tokens(tokenListBuilder1); + + var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); + tokenListBuilder2.token("..."); + tokenBuilder.tokens(tokenListBuilder2); + + builder.tokenlist(tokenBuilder); + + var registry = new SpecialTokenRegistry(builder.build()); + + var defaultTokens = registry.getSpecialTokens("default"); + assertEquals("default", defaultTokens.name()); + assertEquals(2, defaultTokens.asMap().size()); + assertEquals("cpp", defaultTokens.asMap().get("c++")); + assertEquals("...", defaultTokens.asMap().get("...")); + } + +} |