diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-05-04 22:55:28 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-05-04 22:55:28 +0200 |
commit | 9bb76fccc1c128920650bd5a55f4ee4a2af554e7 (patch) | |
tree | c3f1c1f77083975756e2b146e0860c1b9d2656d8 /linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java | |
parent | 8430510510f28a2888a37bf6b07e3526486e072c (diff) |
Revert "Revert "Bratseth/special tokens""
Diffstat (limited to 'linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java')
-rw-r--r-- | linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java | 40 |
1 files changed, 40 insertions, 0 deletions
diff --git a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java new file mode 100644 index 00000000000..47c3ba7933c --- /dev/null +++ b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java @@ -0,0 +1,40 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.language.process; + +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SpecialTokensTestCase { + + @Test + public void testSpecialTokensConfig() { + var builder = new SpecialtokensConfig.Builder(); + var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder(); + tokenBuilder.name("default"); + + var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); + tokenListBuilder1.token("c++"); + tokenListBuilder1.replace("cpp"); + tokenBuilder.tokens(tokenListBuilder1); + + var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder(); + tokenListBuilder2.token("..."); + tokenBuilder.tokens(tokenListBuilder2); + + builder.tokenlist(tokenBuilder); + + var registry = new SpecialTokenRegistry(builder.build()); + + var defaultTokens = registry.getSpecialTokens("default"); + assertEquals("default", defaultTokens.name()); + assertEquals(2, defaultTokens.asMap().size()); + assertEquals("cpp", defaultTokens.asMap().get("c++")); + assertEquals("...", defaultTokens.asMap().get("...")); + } + +} |