Stem by linguistics in rule bases

Also add a @language directive to stem in other languages than english.

15 files changed, 221 insertions, 104 deletions

diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java
index ca5bb4d4cd2..bf99a709df3 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java
@@ -4,6 +4,7 @@ package com.yahoo.prelude.semantics.parser.test;
 import java.util.Iterator;
 
 import com.yahoo.javacc.UnicodeUtilities;
+import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.semantics.RuleBase;
 import com.yahoo.prelude.semantics.RuleImporter;
 import com.yahoo.prelude.semantics.parser.ParseException;
@@ -24,8 +25,8 @@ public class SemanticsParserTestCase {
 
     @Test
     public void testRuleReading() throws java.io.IOException, ParseException {
-        RuleBase rules=new RuleImporter().importFile(ROOT + "rules.sr");
-        Iterator<?> i=rules.ruleIterator();
+        RuleBase rules = new RuleImporter(new SimpleLinguistics()).importFile(ROOT + "rules.sr");
+        Iterator<?> i = rules.ruleIterator();
         assertEquals("[listing] [preposition] [place] -> listing:[listing] place:[place]!150",
                      i.next().toString());
         assertEquals("[listing] [place] +> place:[place]",
@@ -36,10 +37,10 @@ public class SemanticsParserTestCase {
                      i.next().toString());
         assertEquals("digital camera -> digicamera",
                      i.next().toString());
-        assertEquals("(parameter.ranking='cat'), (parameter.ranking='cat0') -> one",i.next().toString());
+        assertEquals("(parameter.ranking='cat'), (parameter.ranking='cat0') -> one", i.next().toString());
         assertFalse(i.hasNext());
 
-        i=rules.conditionIterator();
+        i = rules.conditionIterator();
         assertEquals("[listing] :- restaurant, shop, cafe, hotel",
                      i.next().toString());
         assertEquals("[preposition] :- in, at, near",
@@ -53,7 +54,7 @@ public class SemanticsParserTestCase {
         assertFalse(i.hasNext());
 
         assertTrue(rules.isDefault());
-        assertEquals(ROOT + "semantics.fsa",rules.getAutomataFile());
+        assertEquals(ROOT + "semantics.fsa", rules.getAutomataFile());
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java
index ac1791ae91a..394752f8aa1 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java
@@ -30,7 +30,7 @@ public class BacktrackingTestCase {
 
     static {
         try {
-            searcher = new SemanticSearcher(new RuleImporter().importFile(root + "backtrackingrules.sr"));
+            searcher = new SemanticSearcher(new RuleImporter(new SimpleLinguistics()).importFile(root + "backtrackingrules.sr"));
         }
         catch (Exception e) {
             throw new RuntimeException(e);
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java
index 86ee9b5948b..eb69372c22b 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java
@@ -1,6 +1,8 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics.test;
 
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics;
 import com.yahoo.search.Query;
 import com.yahoo.prelude.semantics.RuleBase;
 import com.yahoo.prelude.semantics.engine.Evaluation;
@@ -24,15 +26,17 @@ public class ConditionTestCase {
 
     @Test
     public void testTermCondition() {
-        TermCondition term=new TermCondition("foo");
-        Query query=new Query("?query=foo");
+        var linguistics = new RuleBaseLinguistics(new SimpleLinguistics());
+        TermCondition term = new TermCondition("foo", linguistics);
+        Query query = new Query("?query=foo");
         assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation()));
     }
 
     @Test
     public void testSequenceCondition() {
-        TermCondition term1 = new TermCondition("foo");
-        TermCondition term2 = new TermCondition("bar");
+        var linguistics = new RuleBaseLinguistics(new SimpleLinguistics());
+        TermCondition term1 = new TermCondition("foo", linguistics);
+        TermCondition term2 = new TermCondition("bar",linguistics);
         SequenceCondition sequence = new SequenceCondition();
         sequence.addCondition(term1);
         sequence.addCondition(term2);
@@ -46,8 +50,9 @@ public class ConditionTestCase {
 
     @Test
     public void testChoiceCondition() {
-        TermCondition term1 = new TermCondition("foo");
-        TermCondition term2 = new TermCondition("bar");
+        var linguistics = new RuleBaseLinguistics(new SimpleLinguistics());
+        TermCondition term1 = new TermCondition("foo", linguistics);
+        TermCondition term2 = new TermCondition("bar", linguistics);
         ChoiceCondition choice = new ChoiceCondition();
         choice.addCondition(term1);
         choice.addCondition(term2);
@@ -61,7 +66,8 @@ public class ConditionTestCase {
 
     @Test
     public void testNamedConditionReference() {
-        TermCondition term = new TermCondition("foo");
+        var linguistics = new RuleBaseLinguistics(new SimpleLinguistics());
+        TermCondition term = new TermCondition("foo", linguistics);
         NamedCondition named = new NamedCondition("cond",term);
         ConditionReference reference = new ConditionReference("cond");
 
@@ -69,8 +75,7 @@ public class ConditionTestCase {
         ProductionRule rule = new ReplacingProductionRule();
         rule.setCondition(reference);
         rule.setProduction(new ProductionList());
-        RuleBase ruleBase = new RuleBase();
-        ruleBase.setName("test");
+        RuleBase ruleBase = new RuleBase("test", linguistics.linguistics());
         ruleBase.addCondition(named);
         ruleBase.addRule(rule);
         ruleBase.initialize();
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java
index 80c9e898302..6d5b9459833 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java
@@ -37,7 +37,7 @@ public class ConfigurationTestCase {
 
     static {
         semanticRulesConfig = new ConfigGetter<>(SemanticRulesConfig.class).getConfig("file:" + root + "semantic-rules.cfg");
-        searcher=new SemanticSearcher(semanticRulesConfig);
+        searcher = new SemanticSearcher(semanticRulesConfig, new SimpleLinguistics());
     }
 
     protected void assertSemantics(String result, String input, String baseName) {
@@ -54,46 +54,46 @@ public class ConfigurationTestCase {
 
     @Test
     public void testReadingConfigurationRuleBase() {
-        RuleBase parent=searcher.getRuleBase("parent");
+        RuleBase parent = searcher.getRuleBase("parent");
         assertNotNull(parent);
-        assertEquals("parent",parent.getName());
-        assertEquals("semantic-rules.cfg",parent.getSource());
+        assertEquals("parent", parent.getName());
+        assertEquals("semantic-rules.cfg", parent.getSource());
     }
 
     @Test
-    public void testParent() throws Exception {
-        assertSemantics("vehiclebrand:audi","audi cars","parent");
-        assertSemantics("vehiclebrand:alfa","alfa bus","parent");
-        assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","parent.sr");
-        assertSemantics("AND vw car",       "vw cars","parent");
-        assertSemantics("AND skoda car",    "skoda cars","parent.sr");
+    public void testParent() {
+        assertSemantics("vehiclebrand:audi", "audi cars", "parent");
+        assertSemantics("vehiclebrand:alfa", "alfa bus", "parent");
+        assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "parent.sr");
+        assertSemantics("AND vw car",        "vw cars", "parent");
+        assertSemantics("AND skoda car",     "skoda cars", "parent.sr");
     }
 
     @Test
-    public void testChild1() throws Exception {
-        assertSemantics("vehiclebrand:skoda","audi cars","child1.sr");
-        assertSemantics("vehiclebrand:alfa", "alfa bus","child1");
-        assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child1");
-        assertSemantics("vehiclebrand:skoda","vw cars","child1");
-        assertSemantics("AND skoda car",     "skoda cars","child1");
+    public void testChild1() {
+        assertSemantics("vehiclebrand:skoda", "audi cars", "child1.sr");
+        assertSemantics("vehiclebrand:alfa",  "alfa bus", "child1");
+        assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "child1");
+        assertSemantics("vehiclebrand:skoda", "vw cars", "child1");
+        assertSemantics("AND skoda car",      "skoda cars", "child1");
     }
 
     @Test
-    public void testChild2() throws Exception {
-        assertSemantics("vehiclebrand:audi","audi cars","child2");
-        assertSemantics("vehiclebrand:alfa","alfa bus","child2.sr");
-        assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child2.sr");
-        assertSemantics("AND vw car","vw cars","child2");
-        assertSemantics("vehiclebrand:skoda","skoda cars","child2");
+    public void testChild2() {
+        assertSemantics("vehiclebrand:audi", "audi cars", "child2");
+        assertSemantics("vehiclebrand:alfa", "alfa bus", "child2.sr");
+        assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "child2.sr");
+        assertSemantics("AND vw car", "vw cars", "child2");
+        assertSemantics("vehiclebrand:skoda", "skoda cars", "child2");
     }
 
     @Test
-    public void testGrandchild() throws Exception {
-        assertSemantics("vehiclebrand:skoda","audi cars","grandchild.sr");
-        assertSemantics("vehiclebrand:alfa","alfa bus","grandchild");
-        assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","grandchild");
-        assertSemantics("vehiclebrand:skoda","vw cars","grandchild");
-        assertSemantics("vehiclebrand:skoda","skoda cars","grandchild");
+    public void testGrandchild() {
+        assertSemantics("vehiclebrand:skoda", "audi cars", "grandchild.sr");
+        assertSemantics("vehiclebrand:alfa", "alfa bus", "grandchild");
+        assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "grandchild");
+        assertSemantics("vehiclebrand:skoda", "vw cars", "grandchild");
+        assertSemantics("vehiclebrand:skoda", "skoda cars", "grandchild");
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java
index fb86beaa9bc..76c8c3966b7 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics.test;
 
+import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.semantics.RuleBaseException;
 import com.yahoo.prelude.semantics.RuleImporter;
 import com.yahoo.prelude.semantics.parser.ParseException;
@@ -14,18 +15,18 @@ import static org.junit.Assert.fail;
  */
 public class DuplicateRuleTestCase {
 
-    private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/";
+    private final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/";
 
     @Test
     public void testDuplicateRuleBaseLoading() throws java.io.IOException, ParseException  {
         if (System.currentTimeMillis() > 0) return; // TODO: Include this test...
 
         try {
-            new RuleImporter().importFile(root + "rules.sr");
+            new RuleImporter(new SimpleLinguistics()).importFile(root + "rules.sr");
             fail("Did not detect duplicate condition names");
         }
         catch (RuleBaseException e) {
-            assertEquals("Duplicate condition 'something' in 'duplicaterules.sr'",e.getMessage());
+            assertEquals("Duplicate condition 'something' in 'duplicaterules.sr'", e.getMessage());
         }
     }
 
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java
index d93fd218259..e9364074281 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java
@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.StringTokenizer;
 
 import com.yahoo.component.chain.Chain;
+import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.search.Query;
 import com.yahoo.prelude.semantics.RuleBase;
 import com.yahoo.prelude.semantics.RuleBaseException;
@@ -24,7 +25,6 @@ import static org.junit.Assert.fail;
 /**
  * @author bratseth
  */
-@SuppressWarnings("deprecation")
 public class InheritanceTestCase {
 
     private static final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/";
@@ -34,10 +34,10 @@ public class InheritanceTestCase {
 
     static {
         try {
-            parent = RuleBase.createFromFile(root + "inheritingrules/parent.sr", null);
-            child1 = RuleBase.createFromFile(root + "inheritingrules/child1.sr", null);
-            child2 = RuleBase.createFromFile(root + "inheritingrules/child2.sr", null);
-            grandchild = RuleBase.createFromFile(root + "inheritingrules/grandchild.sr", null);
+            parent = RuleBase.createFromFile(root + "inheritingrules/parent.sr", null, new SimpleLinguistics());
+            child1 = RuleBase.createFromFile(root + "inheritingrules/child1.sr", null, new SimpleLinguistics());
+            child2 = RuleBase.createFromFile(root + "inheritingrules/child2.sr", null, new SimpleLinguistics());
+            grandchild = RuleBase.createFromFile(root + "inheritingrules/grandchild.sr", null, new SimpleLinguistics());
             grandchild.setDefault(true);
 
             searcher = new SemanticSearcher(parent, child1, child2, grandchild);
@@ -77,7 +77,7 @@ public class InheritanceTestCase {
     public void testInclusionOrderAndContentDump() {
         StringTokenizer lines = new StringTokenizer(grandchild.toContentString(),"\n",false);
         assertEquals("vw -> audi", lines.nextToken());
-        assertEquals("cars -> car", lines.nextToken());
+        assertEquals("car -> car", lines.nextToken());
         assertEquals("[brand] [vehicle] -> vehiclebrand:[brand]", lines.nextToken());
         assertEquals("vehiclebrand:bmw +> expensivetv", lines.nextToken());
         assertEquals("vehiclebrand:audi -> vehiclebrand:skoda", lines.nextToken());
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java
index cd5743c6d77..376da065f4d 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java
@@ -20,66 +20,65 @@ public class ParameterTestCase extends RuleBaseAbstractTestCase {
     /** Tests parameter literal matching */
     @Test
     public void testLiteralEquals() {
-        assertSemantics("a","a");
-        assertSemantics("RANK a foo:a","a&ranking=category");
-        assertSemantics("a","a&ranking=somethingelse");
-        assertSemantics("a","a&otherparam=category");
+        assertSemantics("a", "a");
+        assertSemantics("RANK a foo:a", "a&ranking=category");
+        assertSemantics("a", "a&ranking=somethingelse");
+        assertSemantics("a", "a&otherparam=category");
     }
 
     /** Tests parameter matching of larger */
     @Test
     public void testLarger() {
-        assertSemantics("a","a");
-        assertSemantics("AND a largepage","a&hits=11");
-        assertSemantics("AND a largepage","a&hits=12");
+        assertSemantics("a", "a");
+        assertSemantics("AND a largepage", "a&hits=11");
+        assertSemantics("AND a largepage", "a&hits=12");
     }
 
     /** Tests parameter containment matching */
     @Test
     public void testContainsAsList() {
         assertSemantics("a","a");
-        assertSemantics("AND a intent:music","a&search=music");
-        assertSemantics("AND a intent:music","a&search=music,books");
-        assertSemantics("AND a intent:music","a&search=kanoos,music,books");
+        assertSemantics("AND a intent:music", "a&search=music");
+        assertSemantics("AND a intent:music", "a&search=music,books");
+        assertSemantics("AND a intent:music", "a&search=kanoos,music,books");
     }
 
     /** Tests parameter production */
     @Test
     public void testParameterProduction() {
-        assertParameterSemantics("AND a b c","a b c","search","[letters, alphabet]");
-        assertParameterSemantics("AND a c d","a c d","search","[letters, someletters]");
-        assertParameterSemantics("+(AND a d e) -letter:c","a d e","search","[someletters]");
-        assertParameterSemantics("AND a d f","a d f","rank-profile","foo");
-        assertParameterSemantics("AND a f g","a f g","grouping.nolearning","true");
+        assertParameterSemantics("AND a b c", "a b c", "search", "[letters, alphabet]");
+        assertParameterSemantics("AND a c d", "a c d", "search", "[letters, someletters]");
+        assertParameterSemantics("+(AND a d e) -letter:c", "a d e", "search", "[someletters]");
+        assertParameterSemantics("AND a d f", "a d f", "rank-profile", "foo");
+        assertParameterSemantics("AND a f g", "a f g", "grouping.nolearning", "true");
     }
 
     @Test
     public void testMultipleAlternativeParameterValuesInCondition() {
-        assertInputRankParameterSemantics("one","foo","cat");
-        assertInputRankParameterSemantics("one","foo","cat0");
-        assertInputRankParameterSemantics("one","bar","cat");
-        assertInputRankParameterSemantics("one","bar","cat0");
-        assertInputRankParameterSemantics("AND one one","foo+bar","cat0");
-        assertInputRankParameterSemantics("AND fuki sushi","fuki+sushi","cat0");
+        assertInputRankParameterSemantics("one", "foo", "cat");
+        assertInputRankParameterSemantics("one", "foo", "cat0");
+        assertInputRankParameterSemantics("one", "bar", "cat");
+        assertInputRankParameterSemantics("one", "bar", "cat0");
+        assertInputRankParameterSemantics("AND one one", "foo+bar", "cat0");
+        assertInputRankParameterSemantics("AND fuki sushi", "fuki+sushi", "cat0");
     }
 
-    private void assertInputRankParameterSemantics(String producedQuery,String inputQuery,
-                                               String rankParameterValue) {
-        assertInputRankParameterSemantics(producedQuery,inputQuery,rankParameterValue,0);
+    private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, String rankParameterValue) {
+        assertInputRankParameterSemantics(producedQuery, inputQuery, rankParameterValue, 0);
     }
 
-    private void assertInputRankParameterSemantics(String producedQuery,String inputQuery,
-                                                   String rankParameterValue,int tracelevel) {
-        Query query=new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel);
+    private void assertInputRankParameterSemantics(String producedQuery, String inputQuery,
+                                                   String rankParameterValue, int tracelevel) {
+        Query query = new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel);
         query.getRanking().setProfile(rankParameterValue);
         query.properties().set("tracelevel.rules", tracelevel);
         assertSemantics(producedQuery, query);
     }
 
-    private void assertParameterSemantics(String producedQuery,String inputQuery,
-                                          String producedParameterName,String producedParameterValue) {
-        Query query=assertSemantics(producedQuery,inputQuery);
-        assertEquals(producedParameterValue,query.properties().getString(producedParameterName));
+    private void assertParameterSemantics(String producedQuery, String inputQuery,
+                                          String producedParameterName, String producedParameterValue) {
+        Query query = assertSemantics(producedQuery, inputQuery);
+        assertEquals(producedParameterValue, query.properties().getString(producedParameterName));
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java
index 8b883759215..b91e9441a2b 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java
@@ -1,6 +1,8 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics.test;
 
+import com.yahoo.language.simple.SimpleLinguistics;
+import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics;
 import com.yahoo.search.Query;
 import com.yahoo.prelude.semantics.RuleBase;
 import com.yahoo.prelude.semantics.engine.Evaluation;
@@ -25,7 +27,8 @@ public class ProductionRuleTestCase {
 
     @Test
     public void testProductionRule() {
-        TermCondition term = new TermCondition("sony");
+        var linguistics = new RuleBaseLinguistics(new SimpleLinguistics());
+        TermCondition term = new TermCondition("sony", linguistics);
         NamedCondition named = new NamedCondition("brand", term);
         ConditionReference reference = new ConditionReference("brand");
 
@@ -38,8 +41,7 @@ public class ProductionRuleTestCase {
         rule.setProduction(productionList);
 
         // To initialize the condition reference...
-        RuleBase ruleBase = new RuleBase();
-        ruleBase.setName("test");
+        RuleBase ruleBase = new RuleBase("test", linguistics.linguistics());
         ruleBase.addCondition(named);
         ruleBase.addRule(rule);
         ruleBase.initialize();
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java
index baccb73cd93..84e47edae29 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java
@@ -2,6 +2,7 @@
 package com.yahoo.prelude.semantics.test;
 
 import com.yahoo.component.chain.Chain;
+import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.search.Query;
 import com.yahoo.prelude.semantics.RuleBase;
 import com.yahoo.prelude.semantics.RuleBaseException;
@@ -16,7 +17,7 @@ import java.util.List;
 import static org.junit.Assert.assertEquals;
 
 /**
- * Tests semantic searching
+ * DO NOT USE. Use RuleBaseTester instead
  *
  * @author bratseth
  */
@@ -37,7 +38,7 @@ public abstract class RuleBaseAbstractTestCase {
         try {
             if (automataFileName != null)
                 automataFileName = root + automataFileName;
-            RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName);
+            RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName, new SimpleLinguistics());
             return new SemanticSearcher(ruleBase);
         } catch (Exception e) {
             throw new RuleBaseException("Initialization of rule base '" + ruleBaseName + "' failed",e);
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java
new file mode 100644
index 00000000000..cc9e758a0e0
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java
@@ -0,0 +1,79 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.semantics.test;
+
+import com.yahoo.component.chain.Chain;
+import com.yahoo.language.opennlp.OpenNlpLinguistics;
+import com.yahoo.prelude.semantics.RuleBase;
+import com.yahoo.prelude.semantics.RuleBaseException;
+import com.yahoo.prelude.semantics.SemanticSearcher;
+import com.yahoo.search.Query;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.test.QueryTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Helper for testing with a rule base.
+ * Replace subclassing of RuleBaseAbstractTestCase by this.
+ *
+ * @author bratseth
+ */
+public class RuleBaseTester {
+
+    private final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/";
+    private final SemanticSearcher searcher;
+
+    public RuleBaseTester(String ruleBaseName) {
+        this(ruleBaseName, null);
+    }
+
+    public RuleBaseTester(String ruleBaseName, String automataFileName) {
+        searcher = createSearcher(ruleBaseName, automataFileName);
+    }
+
+    private SemanticSearcher createSearcher(String ruleBaseName,String automataFileName) {
+        try {
+            if (automataFileName != null)
+                automataFileName = root + automataFileName;
+            RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName, new OpenNlpLinguistics());
+            return new SemanticSearcher(ruleBase);
+        } catch (Exception e) {
+            throw new RuleBaseException("Initialization of rule base '" + ruleBaseName + "' failed", e);
+        }
+    }
+
+    public Query assertSemantics(String result, String input) {
+        return assertSemantics(result, input, 0);
+    }
+
+    public Query assertSemantics(String result, String input, int tracelevel) {
+        return assertSemantics(result, input, tracelevel, Query.Type.ALL);
+    }
+
+    public Query assertSemantics(String result, String input, int tracelevel, Query.Type queryType) {
+        Query query = new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=" + tracelevel +
+                                "&language=und&type=" + queryType);
+        return assertSemantics(result, query);
+    }
+
+    public Query assertSemantics(String result, Query query) {
+        createExecution(searcher).search(query);
+        assertEquals(result, query.getModel().getQueryTree().getRoot().toString());
+        return query;
+    }
+
+    private Execution createExecution(Searcher searcher) {
+        return new Execution(chainedAsSearchChain(searcher), Execution.Context.createContextStub());
+    }
+
+    private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) {
+        List<Searcher> searchers = new ArrayList<>();
+        searchers.add(topOfChain);
+        return new Chain<>(searchers);
+    }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java
index 29cc5c6e23a..76b2d3991c1 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java
@@ -142,11 +142,6 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase {
     }
 
     @Test
-    public void testPluralReplaceBecomesSingular() {
-        assertSemantics("AND from:paris to:texas","pariss to texass");
-    }
-
-    @Test
     public void testOrProduction() {
         assertSemantics("OR something somethingelse", "something");
     }
@@ -155,7 +150,7 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase {
     @Test
     public void testWeightedSetItem() {
         Query q = new Query();
-        WeightedSetItem weightedSet=new WeightedSetItem("fieldName");
+        WeightedSetItem weightedSet = new WeightedSetItem("fieldName");
         weightedSet.addToken("a", 1);
         weightedSet.addToken("b", 2);
         q.getModel().getQueryTree().setRoot(weightedSet);
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java
index 6702a1ca1d9..b8efbf7422b 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java
@@ -4,34 +4,52 @@ package com.yahoo.prelude.semantics.test;
 import org.junit.Test;
 
 /**
- * Tests a case reported by tularam
+ * Tests stemming.
  *
  * @author bratseth
  */
-public class StemmingTestCase extends RuleBaseAbstractTestCase {
+public class StemmingTestCase {
 
-    public StemmingTestCase() {
-        super("stemming.sr");
+    @Test
+    public void testRewritingDueToStemmingInQuery() {
+        var tester = new RuleBaseTester("stemming.sr");
+        tester.assertSemantics("+(AND i:vehicle TRUE) -i:s", "i:cars -i:s");
     }
 
     @Test
-    public void testRewritingDueToStemmingInQuery() {
-        assertSemantics("+(AND i:vehicle TRUE) -i:s","i:cars -i:s");
+    public void testNoRewritingDueToStemmingInQueryWhenStemmingDisabled() {
+        var tester = new RuleBaseTester("stemming-none.sr");
+        tester.assertSemantics("+i:cars -i:s", "i:cars -i:s");
     }
 
     @Test
     public void testRewritingDueToStemmingInRule() {
-        assertSemantics("+(AND i:animal TRUE) -i:s","i:horse -i:s");
+        var tester = new RuleBaseTester("stemming.sr");
+        tester.assertSemantics("+(AND i:animal TRUE) -i:s", "i:horse -i:s");
+    }
+
+    @Test
+    public void testNoRewritingDueToStemmingInRuleWhenStemmingDisabled() {
+        var tester = new RuleBaseTester("stemming-none.sr");
+        tester.assertSemantics("+i:horse -i:s", "i:horse -i:s");
     }
 
     @Test
     public void testRewritingDueToExactMatch() {
-        assertSemantics("+(AND i:arts i:sciences TRUE) -i:s","i:as -i:s");
+        var tester = new RuleBaseTester("stemming.sr");
+        tester.assertSemantics("+(AND i:arts i:sciences TRUE) -i:s", "i:as -i:s");
+    }
+
+    @Test
+    public void testEnglishStemming() {
+        var tester = new RuleBaseTester("stemming.sr");
+        tester.assertSemantics("i:drive", "i:going");
     }
 
     @Test
-    public void testNoRewritingBecauseShortWordsAreNotStemmed() {
-        assertSemantics("+i:a -i:s","i:a -i:s");
+    public void testFrenchStemming() {
+        var tester = new RuleBaseTester("stemming-french.sr");
+        tester.assertSemantics("i:going", "i:going");
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr
new file mode 100644
index 00000000000..1ccafd04344
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr
@@ -0,0 +1,8 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@stemming(true)
+@language(fr)
+
+i:as -> i:arts i:sciences;
+i:car -> i:vehicle;
+i:horses -> i:animal;
+i:go -> i:drive;
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr
new file mode 100644
index 00000000000..44f6e40a308
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr
@@ -0,0 +1,6 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@stemming(false)
+
+i:car -> i:vehicle;
+i:horses -> i:animal;
+i:go -> i:drive;
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr
index f68706646c2..ea73e385b3a 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr
@@ -1,5 +1,7 @@
 # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 @stemming(true)
+
 i:as -> i:arts i:sciences;
 i:car -> i:vehicle;
 i:horses -> i:animal;
+i:go -> i:drive;