aboutsummaryrefslogtreecommitdiffstats
path: root/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
diff options
context:
space:
mode:
Diffstat (limited to 'container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java')
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java50
1 files changed, 24 insertions, 26 deletions
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
index 5b660f78429..1ff5574ec03 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
@@ -10,7 +10,7 @@ import com.yahoo.language.process.SpecialTokenRegistry;
import com.yahoo.language.process.SpecialTokens;
import com.yahoo.prelude.query.parser.Token;
import com.yahoo.prelude.query.parser.Tokenizer;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Collections;
@@ -29,9 +29,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE;
import static com.yahoo.prelude.query.parser.Token.Kind.STAR;
import static com.yahoo.prelude.query.parser.Token.Kind.UNDERSCORE;
import static com.yahoo.prelude.query.parser.Token.Kind.WORD;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
/**
* Tests the tokenizer
@@ -41,7 +39,7 @@ import static org.junit.Assert.assertTrue;
public class TokenizerTestCase {
@Test
- public void testPlainTokenization() {
+ void testPlainTokenization() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
@@ -76,14 +74,14 @@ public class TokenizerTestCase {
}
@Test
- public void testOutsideBMPCodepoints() {
+ void testOutsideBMPCodepoints() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
List<?> tokens = tokenizer.tokenize("\ud841\udd47");
assertEquals(new Token(WORD, "\ud841\udd47"), tokens.get(0));
}
@Test
- public void testOneSpecialToken() {
+ void testOneSpecialToken() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
@@ -93,7 +91,7 @@ public class TokenizerTestCase {
}
@Test
- public void testSpecialTokenCombination() {
+ void testSpecialTokenCombination() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
@@ -121,7 +119,7 @@ public class TokenizerTestCase {
* separated by space, as special token recognition happens before tokenization
*/
@Test
- public void testSpecialTokenCJK() {
+ void testSpecialTokenCJK() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSubstringSpecialTokens(true);
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing"));
@@ -146,7 +144,7 @@ public class TokenizerTestCase {
}
@Test
- public void testSpecialTokenCaseInsensitive() {
+ void testSpecialTokenCaseInsensitive() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
@@ -162,7 +160,7 @@ public class TokenizerTestCase {
}
@Test
- public void testSpecialTokenNonMatch() {
+ void testSpecialTokenNonMatch() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
@@ -187,7 +185,7 @@ public class TokenizerTestCase {
}
@Test
- public void testSpecialTokenConfigurationDefault() {
+ void testSpecialTokenConfigurationDefault() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
@@ -212,7 +210,7 @@ public class TokenizerTestCase {
}
@Test
- public void testSpecialTokenConfigurationOther() {
+ void testSpecialTokenConfigurationOther() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("other"));
@@ -247,7 +245,7 @@ public class TokenizerTestCase {
}
@Test
- public void testTokenReplacing() {
+ void testTokenReplacing() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing"));
@@ -273,7 +271,7 @@ public class TokenizerTestCase {
}
@Test
- public void testExactMatchTokenization() {
+ void testExactMatchTokenization() {
SearchDefinition sd = new SearchDefinition("testsd");
Index index1 = new Index("testexact1");
@@ -286,8 +284,8 @@ public class TokenizerTestCase {
IndexFacts facts = new IndexFacts(new IndexModel(sd));
IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet());
- Tokenizer tokenizer=new Tokenizer(new SimpleLinguistics());
- List<?> tokens=tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*& b:c", "default", session);
+ Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
+ List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*& b:c", "default", session);
// tokenizer.print();
assertEquals(new Token(WORD, "normal"), tokens.get(0));
assertEquals(new Token(SPACE, " "), tokens.get(1));
@@ -317,7 +315,7 @@ public class TokenizerTestCase {
}
@Test
- public void testExactMatchTokenizationTerminatorTerminatesQuery() {
+ void testExactMatchTokenizationTerminatorTerminatesQuery() {
SearchDefinition sd = new SearchDefinition("testsd");
Index index1 = new Index("testexact1");
@@ -354,7 +352,7 @@ public class TokenizerTestCase {
}
@Test
- public void testExactMatchTokenizationWithTerminatorTerminatedByEndOfString() {
+ void testExactMatchTokenizationWithTerminatorTerminatedByEndOfString() {
SearchDefinition sd = new SearchDefinition("testsd");
Index index1 = new Index("testexact1");
@@ -391,7 +389,7 @@ public class TokenizerTestCase {
}
@Test
- public void testExactMatchTokenizationEndsByColon() {
+ void testExactMatchTokenizationEndsByColon() {
SearchDefinition sd = new SearchDefinition("testsd");
Index index1 = new Index("testexact1");
@@ -429,7 +427,7 @@ public class TokenizerTestCase {
}
@Test
- public void testExactMatchHeuristics() {
+ void testExactMatchHeuristics() {
SearchDefinition sd = new SearchDefinition("testsd");
Index index1 = new Index("testexact1");
@@ -542,7 +540,7 @@ public class TokenizerTestCase {
assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
assertEquals(new Token(COLON, ":"), tokens.get(1));
assertEquals(new Token(STAR, "*"), tokens.get(2));
- assertEquals(new Token(WORD, "foobar*!100"),tokens.get(3));
+ assertEquals(new Token(WORD, "foobar*!100"), tokens.get(3));
assertEquals(new Token(EXCLAMATION, "!"), tokens.get(4));
tokens = tokenizer.tokenize("testexact1:foo(bar)", facts);
@@ -582,7 +580,7 @@ public class TokenizerTestCase {
tokens = tokenizer.tokenize("testexact1:\"news search\" testexact2:resolved", facts);
assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
assertEquals(new Token(COLON, ":"), tokens.get(1));
- assertEquals(new Token(WORD, "news search"),tokens.get(2));
+ assertEquals(new Token(WORD, "news search"), tokens.get(2));
assertEquals(new Token(SPACE, " "), tokens.get(3));
assertEquals(new Token(WORD, "testexact2"), tokens.get(4));
assertEquals(new Token(COLON, ":"), tokens.get(5));
@@ -592,7 +590,7 @@ public class TokenizerTestCase {
assertEquals(new Token(LBRACE, "("), tokens.get(0));
assertEquals(new Token(WORD, "testexact1"), tokens.get(1));
assertEquals(new Token(COLON, ":"), tokens.get(2));
- assertEquals(new Token(WORD, "news search"),tokens.get(3));
+ assertEquals(new Token(WORD, "news search"), tokens.get(3));
assertEquals(new Token(SPACE, " "), tokens.get(4));
assertEquals(new Token(WORD, "testexact1"), tokens.get(5));
assertEquals(new Token(COLON, ":"), tokens.get(6));
@@ -614,7 +612,7 @@ public class TokenizerTestCase {
tokens = tokenizer.tokenize("testexact1:\"news search\"!200", facts);
assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
assertEquals(new Token(COLON, ":"), tokens.get(1));
- assertEquals(new Token(WORD, "news search"),tokens.get(2));
+ assertEquals(new Token(WORD, "news search"), tokens.get(2));
assertEquals(new Token(EXCLAMATION, "!"), tokens.get(3));
assertEquals(new Token(NUMBER, "200"), tokens.get(4));
@@ -705,7 +703,7 @@ public class TokenizerTestCase {
}
@Test
- public void testSingleQuoteAsWordCharacter() {
+ void testSingleQuoteAsWordCharacter() {
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));