Support producing all clauses in a referenced condition

26 files changed, 389 insertions, 251 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java
index 55adc3cba8d..0c4dff15744 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java
@@ -22,9 +22,7 @@ public abstract class CompositeIndexedItem extends CompositeTaggableItem impleme
 
     private String index = "";
 
-    /**
-     * The name of the index this belongs to, or "" (never null) if not specified
-     **/
+    /** The name of the index this belongs to, or "" (never null) if not specified */
     public String getIndexName() {
         return index;
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java
index 8e137d99951..4980b035876 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java
@@ -1,10 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics;
 
-import com.yahoo.language.Language;
 import com.yahoo.language.Linguistics;
-import com.yahoo.language.process.StemMode;
-import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics;
 import com.yahoo.prelude.semantics.rule.CompositeCondition;
 import com.yahoo.prelude.semantics.rule.Condition;
 import com.yahoo.prelude.semantics.rule.NamedCondition;
@@ -76,12 +73,9 @@ public class RuleBase {
      */
     private boolean usesAutomata = false;
 
-    private RuleBaseLinguistics linguistics;
-
     /** Creates an empty rule base */
-    public RuleBase(String name, Linguistics linguistics) {
+    public RuleBase(String name) {
         this.name = name;
-        this.linguistics = new RuleBaseLinguistics(StemMode.BEST, Language.ENGLISH, linguistics);
     }
 
     /**
@@ -284,7 +278,7 @@ public class RuleBase {
     }
 
     /**
-     * Set to truew if this uses an automata, even if an automata is not present right now.
+     * Set to true if this uses an automata, even if an automata is not present right now.
      * Useful to validate without having automatas available
      */
     void setUsesAutomata(boolean usesAutomata) { this.usesAutomata = usesAutomata; }
@@ -342,7 +336,7 @@ public class RuleBase {
     }
 
     // TODO: Values are not added right now
-    protected void annotatePhrase(PhraseMatcher.Phrase phrase,Query query,int traceLevel) {
+    protected void annotatePhrase(PhraseMatcher.Phrase phrase, Query query, int traceLevel) {
         for (StringTokenizer tokens = new StringTokenizer(phrase.getData(), "|", false); tokens.hasMoreTokens(); ) {
             String token = tokens.nextToken();
             int semicolonIndex = token.indexOf(";");
@@ -357,12 +351,12 @@ public class RuleBase {
             phrase.getItem(0).addAnnotation(annotation, phrase);
             if (traceLevel >= 4)
                 query.trace("   Annotating '" + phrase + "' as " + annotation +
-                        (value.equals("") ? "" :"=" + value),false,1);
+                            (value.equals("") ? "" :"=" + value),false,1);
         }
     }
 
     private void makeReferences() {
-        for (Iterator<ProductionRule> i=ruleIterator(); i.hasNext(); ) {
+        for (Iterator<ProductionRule> i = ruleIterator(); i.hasNext(); ) {
             ProductionRule rule = i.next();
             rule.makeReferences(this);
         }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java
index acbf9a7ffb6..6e7286cb8dc 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java
@@ -99,7 +99,7 @@ public class RuleImporter {
             reader = IOUtils.createReader(fileName, "utf-8");
             File file = new File(fileName);
             String absoluteFileName = file.getAbsolutePath();
-            var ruleBase = new RuleBase(stripLastName(file.getName()), linguistics);
+            var ruleBase = new RuleBase(stripLastName(file.getName()));
             privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase);
             return ruleBase;
         }
@@ -205,7 +205,7 @@ public class RuleImporter {
     /** Imports an unitialized rule base */
     public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException {
         if (config == null) throw new IllegalStateException("Must initialize with config if importing from config");
-        RuleBase ruleBase = new RuleBase(ruleBaseConfig.name(), linguistics);
+        RuleBase ruleBase = new RuleBase(ruleBaseConfig.name());
         return privateImportFromReader(new StringReader(ruleBaseConfig.rules()),
                                        "semantic-rules.cfg",
                                        ruleBaseConfig.automata(),ruleBase);
@@ -234,7 +234,7 @@ public class RuleImporter {
     public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException {
         try {
             if (ruleBase == null)
-                ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName, linguistics);
+                ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName);
             ruleBase.setSource(sourceName.replace('\\', '/'));
             new SemanticsParser(reader, linguistics).semanticRules(ruleBase, this);
             if (automataFile != null && !automataFile.isEmpty())
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java
index 85bcdf8bd20..7bfcc08f21b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java
@@ -2,6 +2,7 @@
 package com.yahoo.prelude.semantics.engine;
 
 import com.yahoo.prelude.query.*;
+import com.yahoo.prelude.semantics.RuleBase;
 import com.yahoo.search.Query;
 import com.yahoo.search.query.QueryTree;
 
@@ -32,11 +33,13 @@ public class Evaluation {
     /** The rule evaluation context, can be reset once the rule is evaluated */
     private final RuleEvaluation ruleEvaluation;
 
+    private final RuleBase ruleBase;
+
     /**
      * The amount of context information to collect about this evaluation.
      * 0 means no context information, higher numbers means more context information.
      */
-    private int traceLevel = 0;
+    private final int traceLevel;
 
     private String traceIndentation = "";
 
@@ -49,8 +52,8 @@ public class Evaluation {
     /** Should we allow stemmed matches? */
     private boolean stemming = true;
 
-    public Evaluation(Query query) {
-        this(query,0);
+    public Evaluation(Query query, RuleBase ruleBase) {
+        this(query, ruleBase, 0);
     }
 
     /**
@@ -59,13 +62,17 @@ public class Evaluation {
      * @param query the query this evaluation is for
      * @param traceLevel the amount of tracing to do
      */
-    public Evaluation(Query query, int traceLevel) {
+    public Evaluation(Query query, RuleBase ruleBase, int traceLevel) {
         this.query = query;
+        this.ruleBase = ruleBase;
         this.traceLevel = traceLevel;
         reset();
         ruleEvaluation = new RuleEvaluation(this);
     }
 
+    /** Returns the rule base this evaluates over */
+    public RuleBase ruleBase() { return ruleBase; }
+
     /** Resets the item iterator to point to the first item */
     public void reset() {
         if (flattenedItems != null)
@@ -134,11 +141,11 @@ public class Evaluation {
     /** Adds an item to the query being evaluated in a way consistent with the query type */
     // TODO: Add this functionality to Query?
     public void addItem(Item item, TermType termType) {
-        Item root= query.getModel().getQueryTree().getRoot();
-        if (root==null)
+        Item root = query.getModel().getQueryTree().getRoot();
+        if (root == null)
             query.getModel().getQueryTree().setRoot(item);
         else
-            query.getModel().getQueryTree().setRoot(combineItems(root,item,termType));
+            query.getModel().getQueryTree().setRoot(combineItems(root, item, termType));
     }
 
     /** Removes this item */
@@ -151,18 +158,18 @@ public class Evaluation {
      * equal items
      */
     public void removeItemByIdentity(Item item) {
-        int position=findIndexByIdentity(item);
-        if (position>=0)
+        int position = findIndexByIdentity(item);
+        if (position >= 0)
             item.getParent().removeItem(position);
         else
             item.getParent().removeItem(item); // Fallback to removeField by equal()
     }
 
     private int findIndexByIdentity(Item item) {
-        int position=0;
-        for (Iterator<Item> i=item.getParent().getItemIterator(); i.hasNext(); ) {
-            Item child=i.next();
-            if (item==child) {
+        int position = 0;
+        for (Iterator<Item> i = item.getParent().getItemIterator(); i.hasNext(); ) {
+            Item child = i.next();
+            if (item == child) {
                 return position;
             }
             position++;
@@ -172,7 +179,7 @@ public class Evaluation {
 
     /** Removes an item, prefers the one at/close to the given position if there are multiple ones */
     public void removeItem(int position,Item item) {
-        Item removeCandidate=item.getParent().getItem(position);
+        Item removeCandidate = item.getParent().getItem(position);
         if (removeCandidate.equals(item)) // Remove based on position
             item.getParent().removeItem(position);
         else
@@ -189,7 +196,7 @@ public class Evaluation {
         if (!(item instanceof SegmentItem)) {
             return item;
         }
-        CompositeItem converted = null;
+        CompositeItem converted;
         if (item instanceof AndSegmentItem) {
             converted = new AndItem();
         } else if (item instanceof PhraseSegmentItem) {
@@ -238,16 +245,21 @@ public class Evaluation {
     /**
      * Inserts an item to the query being evaluated in a way consistent with the query type
      *
-     * @param item the item to insert
+     * @param items the item to insert
      * @param parent the parent of this item, or null to set the root
      * @param index the index at which to insert this into the parent
      * @param desiredParentType the desired type of the composite which contains item when this returns
      */
-    public void insertItem(Item item, CompositeItem parent, int index, TermType desiredParentType) {
+    public void insertItems(List<Item> items, CompositeItem parent, int index, TermType desiredParentType) {
         if (isEmpty(parent)) {
-            CompositeItem newParent = (CompositeItem)desiredParentType.createItemClass();
-            newParent.addItem(item);
-            query.getModel().getQueryTree().setRoot(newParent);
+            if (items.size() == 1 && desiredParentType.hasItemClass(items.get(0).getClass())) {
+                query.getModel().getQueryTree().setRoot(items.get(0));
+            }
+            else {
+                CompositeItem newParent = (CompositeItem) desiredParentType.createItemClass();
+                items.forEach(item -> newParent.addItem(item));
+                query.getModel().getQueryTree().setRoot(newParent);
+            }
             return;
         }
 
@@ -260,17 +272,31 @@ public class Evaluation {
         }
 
         if (( desiredParentType == TermType.DEFAULT || desiredParentType.hasItemClass(parent.getClass()) )
-             && equalIndexNameIfParentIsPhrase(item, parent)) {
-            addItem(parent, index, item, desiredParentType);
+             && equalIndexNameIfParentIsPhrase(items, parent)) {
+            for (Item item : items)
+                addItem(parent, index, item, desiredParentType);
+        }
+        else if (parent.items().isEmpty()) {
+            CompositeItem parentsParent = parent.getParent();
+            CompositeItem newParent = newParent(desiredParentType);
+            items.forEach(item -> newParent.addItem(item));
+            parentsParent.setItem(parentsParent.getItemIndex(parent), newParent);
+        }
+        else if (items.size() ==1 && desiredParentType.hasItemClass(items.get(0).getClass())) { // This will never happen
+            for (Item item : items)
+                addItem(parent, index, item, desiredParentType);
         }
         else if (incompatible(desiredParentType, parent)) {
-            insertIncompatibleItem(item, parent, query, desiredParentType);
+            for (Item item : items)
+                insertIncompatibleItem(item, parent, query, desiredParentType);
         }
         else {
-            insertIncompatibleItemAsParent(item, parent, query, desiredParentType);
+            for (Item item : items)
+                insertIncompatibleItemAsParent(item, parent, query, desiredParentType);
         }
     }
 
+    /** Returns true if this item represents an empty query *tree*. */
     private boolean isEmpty(Item item) {
         if (item == null) return true;
         if (item instanceof QueryTree && ((QueryTree) item).isEmpty()) return true;
@@ -278,8 +304,9 @@ public class Evaluation {
     }
 
     /** Returns true if the desired type cannot have childCandidate as a child */
-    private boolean incompatible(TermType desiredParentType, Item childCandidate) {
-        return desiredParentType == TermType.EQUIV && childCandidate.getItemType() != Item.ItemType.EQUIV;
+    private boolean incompatible(TermType desiredParentType, CompositeItem parent) {
+        return desiredParentType == TermType.EQUIV
+               && (parent.getItemType() != Item.ItemType.EQUIV && parent.getItemType() != Item.ItemType.PHRASE);
     }
 
     private void addItem(CompositeItem parent, int index, Item item, TermType desiredParentType) {
@@ -287,7 +314,7 @@ public class Evaluation {
             if (index == 0 && parent.getItem(0) == null) { // Case 1: The current positive is null and we are adding a positive
                 parent.setItem(0, item);
             }
-            else if (index<=1 && !(parent.getItem(0) instanceof CompositeItem))  { // Case 2: The positive must become a composite
+            else if (index <= 1 && !(parent.getItem(0) instanceof CompositeItem))  { // Case 2: The positive must become a composite
                 CompositeItem positiveComposite = (CompositeItem)desiredParentType.createItemClass();
                 positiveComposite.addItem(parent.getItem(0));
                 positiveComposite.addItem(index, item);
@@ -313,50 +340,46 @@ public class Evaluation {
     }
 
     /** A special purpose check used to simplify the above */
-    private boolean equalIndexNameIfParentIsPhrase(Item item,CompositeItem parent) {
+    private boolean equalIndexNameIfParentIsPhrase(List<Item> items, CompositeItem parent) {
         if ( ! (parent instanceof PhraseItem)) return true;
-        if ( ! (item instanceof IndexedItem)) return true;
+        var phrase = (PhraseItem)parent;
 
-        return ((PhraseItem)parent).getIndexName().equals(((IndexedItem)item).getIndexName());
+        for (Item item : items) {
+            if ( ! (item instanceof IndexedItem)) continue;
+            var indexedItem = (IndexedItem)item;
+            if (! indexedItem.getIndexName().equals(phrase.getIndexName())) return false;
+        }
+        return true;
     }
 
     private void insertIncompatibleItem(Item item, CompositeItem parent, Query query, TermType desiredParentType) {
-        CompositeItem newParent;
-        if (desiredParentType == TermType.DEFAULT)
-            newParent = new AndItem();
-        else
-            newParent = (CompositeItem)desiredParentType.createItemClass();
+        CompositeItem newParent = newParent(desiredParentType);
 
         newParent.addItem(item);
         parent.addItem(newParent);
     }
 
-    private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) {
-        // Create new parent
-        CompositeItem newParent;
-        if (desiredParentType == TermType.DEFAULT)
-            newParent = new AndItem();
-        else
-            newParent = (CompositeItem)desiredParentType.createItemClass();
+    private CompositeItem newParent(TermType desiredParentType) {
+        return desiredParentType == TermType.DEFAULT ? new AndItem() : (CompositeItem)desiredParentType.createItemClass();
+    }
 
-        // Save previous parent parent
+    private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) {
         CompositeItem parentsParent = parent.getParent();
 
+        CompositeItem newParent = newParent(desiredParentType);
+        if (! (parentsParent instanceof QueryTree) && parentsParent.getItemType() == newParent.getItemType()) { // Collapse
+            newParent = parentsParent;
+        }
+
         // Add items to new parent
         newParent.addItem(parent);
         newParent.addItem(item);
 
-        // Insert new parent as root or child of old parents parent
-        if (parentsParent == null) {
-            query.getModel().getQueryTree().setRoot(newParent);
-
-        }
-        else {
+        if (newParent != parentsParent) // Insert new parent as root or child of old parent's parent
             parentsParent.setItem(parentsParent.getItemIndex(parent), newParent);
-        }
     }
 
-    private Item combineItems(Item first,Item second,TermType termType) {
+    private Item combineItems(Item first, Item second, TermType termType) {
         if (first instanceof NullItem) {
             return second;
         } else if (first instanceof NotItem) {
@@ -378,6 +401,10 @@ public class Evaluation {
                 return composite;
             }
             else {
+                if (combined instanceof EquivItem) {
+                    first = makeEquivCompatible(first);
+                    second = makeEquivCompatible(second);
+                }
                 combined.addItem(first);
                 combined.addItem(second); // Also works for nots
                 return combined;
@@ -394,6 +421,22 @@ public class Evaluation {
         }
     }
 
+    private Item makeEquivCompatible(Item item) {
+        if (item instanceof AndItem || item instanceof WeakAndItem) {
+            PhraseItem phrase = new PhraseItem();
+            List<Item> children = ((CompositeItem)item).items();
+            if (children.isEmpty()) return phrase;
+            String index = ((IndexedItem)children.get(0)).getIndexName();
+            for (var child : ((CompositeItem)item).items())
+                phrase.addItem(child);
+            phrase.setIndexName(index);
+            return phrase;
+        }
+        else {
+            return item; // Compatible, or can't be made so
+        }
+    }
+
     private CompositeItem createType(TermType termType) {
         if (termType == TermType.DEFAULT) {
             if (query.getModel().getType() == Query.Type.ANY)
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java
index 6098fbee327..6ed09926ccb 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java
@@ -60,7 +60,12 @@ public class Match {
 
     /** Returns a new item representing this match */
     public Item toItem(String label) {
-        var newItem = new WordItem(getReplaceValue(), label);
+        return toItem(label, getReplaceValue());
+    }
+
+    /** Returns a new item representing this match */
+    public Item toItem(String label, String term) {
+        var newItem = new WordItem(term, label);
         newItem.setWeight(item.getWeight());
         return newItem;
     }
@@ -81,4 +86,9 @@ public class Match {
         return true;
     }
 
+    @Override
+    public String toString() {
+        return "match of " + item + " at " + position + " to be replaced by " + replaceValue;
+    }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java
index 605f3a23e10..b7083414e85 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java
@@ -8,7 +8,7 @@ import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.PhraseItem;
 
 /**
- * The Matches referenced by a particular context name in a rule evaluation
+ * The matches referenced by a particular context name in a rule evaluation
  *
  * @author bratseth
  */
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java
index dd6610d1184..e2756afbb2e 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java
@@ -37,7 +37,7 @@ public class RuleEngine {
         //       Probably create indices on the first term like Prolog implementations use to
 
         boolean matchedAnything = false;
-        Evaluation evaluation = new Evaluation(query, traceLevel);
+        Evaluation evaluation = new Evaluation(query, rules, traceLevel);
         if (traceLevel >= 2)
             evaluation.trace(2,"Evaluating query '" + evaluation.getQuery().getModel().getQueryTree().getRoot() + "':");
         for (ListIterator<ProductionRule> i = rules.ruleIterator(); i.hasNext(); ) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java
index 2a00843a85c..1da16b4d277 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java
@@ -190,8 +190,9 @@ public class RuleEvaluation {
     // TODO: Simplistic yet. Nedd to support context nesting etc.
     public void entering(String context) {
         if (context == null) return;
-        if (matchReferences != null && matchReferences.contains(context))
+        if (matchReferences != null && matchReferences.contains(context)) {
             currentContext = context;
+        }
     }
 
     public void leaving(String context) {
@@ -209,7 +210,7 @@ public class RuleEvaluation {
      */
     public void addMatch(FlattenedItem item, String replaceString) {
         evaluation.makeParentMutable(item.getItem());
-        Match match = new Match(item,replaceString);
+        Match match = new Match(item, replaceString);
         if (currentContext != null) {
             ReferencedMatches matches = getReferencedMatches(currentContext);
             if (matches == null) {
@@ -241,8 +242,8 @@ public class RuleEvaluation {
     public Evaluation getEvaluation() { return evaluation; }
 
     /** Adds an item to the query being evaluated in a way consistent with the query type */
-    public void addItem(Item item, TermType termType) {
-        evaluation.addItem(item,termType);
+    public void addItems(List<Item> items, TermType termType) {
+        items.forEach(item -> evaluation.addItem(item, termType));
     }
 
     public void removeItem(Item item) {
@@ -262,13 +263,13 @@ public class RuleEvaluation {
     /**
      * Inserts an item to the query being evaluated in a way consistent with the query type
      *
-     * @param item the item to insert
+     * @param items the items to insert
      * @param parent the parent of this item, or null to set the root
      * @param index the index at which to insert this into the parent
      * @param termType the kind of item to index, this decides the resulting structure
      */
-    public void insertItem(Item item, CompositeItem parent, int index, TermType termType) {
-        evaluation.insertItem(item, parent, index, termType);
+    public void insertItems(List<Item> items, CompositeItem parent, int index, TermType termType) {
+        evaluation.insertItems(items, parent, index, termType);
     }
 
     /** Returns a read-only view of the items of this */
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java
index 77bffa5778e..a49272a73ea 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics.rule;
 
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -14,28 +15,28 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation;
  */
 public abstract class CompositeCondition extends Condition {
 
-    private List<Condition> conditions=new java.util.ArrayList<>();
+    private final List<Condition> conditions = new java.util.ArrayList<>();
 
     public CompositeCondition() {
     }
 
     public void preMatchHook(RuleEvaluation e) {
         super.preMatchHook(e);
-        if (e.getTraceLevel()>=3) {
-            e.trace(3,"Evaluating '" + this + "'"  + " at " + e.currentItem());
+        if (e.getTraceLevel() >= 3) {
+            e.trace(3, "Evaluating '" + this + "'"  + " at " + e.currentItem());
             e.indentTrace();
         }
     }
 
     public void postMatchHook(RuleEvaluation e) {
-        if (e.getTraceLevel()>=3) {
+        if (e.getTraceLevel() >= 3) {
             e.unindentTrace();
         }
     }
 
     protected boolean hasOpenChoicepoint(RuleEvaluation evaluation) {
-        for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) {
-            Condition subCondition=i.next();
+        for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) {
+            Condition subCondition = i.next();
             if (subCondition.hasOpenChoicepoint(evaluation))
                 return true;
         }
@@ -48,8 +49,8 @@ public abstract class CompositeCondition extends Condition {
     }
 
     /** Sets the condition at the given index */
-    public void setCondition(int index,Condition condition) {
-        conditions.set(index,condition);
+    public void setCondition(int index, Condition condition) {
+        conditions.set(index, condition);
     }
 
     /** Returns the number of subconditions */
@@ -74,7 +75,7 @@ public abstract class CompositeCondition extends Condition {
      * @throws IndexOutOfBoundsException if there is no condition at this index
      */
     public Condition removeCondition(int i) {
-        Condition condition=conditions.remove(i);
+        Condition condition = conditions.remove(i);
         condition.setParent(null);
         return condition;
     }
@@ -82,20 +83,22 @@ public abstract class CompositeCondition extends Condition {
     /** Returns an iterator of the immediate children of this condition */
     public Iterator<Condition> conditionIterator() { return conditions.iterator(); }
 
+    public List<Condition> conditions() { return Collections.unmodifiableList(conditions); }
+
     public void makeReferences(RuleBase rules) {
-        for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) {
-            Condition condition=i.next();
+        for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) {
+            Condition condition = i.next();
             condition.makeReferences(rules);
         }
     }
 
     /** Whether this should be output with parentheses, default is parent!=null */
     protected boolean useParentheses() {
-        return getParent()!=null;
+        return getParent() != null;
     }
 
     protected String toInnerString(String conditionSeparator) {
-        if (getLabel()!=null)
+        if (getLabel() != null)
             return getLabel() + ":(" + conditionsToString(conditionSeparator) + ")";
         else if (useParentheses())
             return "(" + conditionsToString(conditionSeparator) + ")";
@@ -104,8 +107,8 @@ public abstract class CompositeCondition extends Condition {
     }
 
     protected final String conditionsToString(String conditionSeparator) {
-        StringBuilder buffer=new StringBuilder();
-        for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) {
+        StringBuilder buffer = new StringBuilder();
+        for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) {
             buffer.append(i.next().toString());
             if (i.hasNext())
                 buffer.append(conditionSeparator);
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java
index 0d3c93619b5..8d6016cb060 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java
@@ -14,20 +14,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation;
 public abstract class Condition {
 
     /** The parent of this condition, or null if this is not nested */
-    private CompositeCondition parent=null;
+    private CompositeCondition parent = null;
 
     /**
      * The label of this condition, or null if none.
      * Specified by label:condition
      * The label is also the default context is no context is speficied explicitly
      */
-    private String label=null;
+    private String label;
 
     /**
      * The name space refered by this match, or null if the default (query)
      * Specified by namespace.condition in rules.
      */
-    private String nameSpace=null;
+    private String nameSpace = null;
 
     /**
      * The name of the context created by this, or null if none
@@ -36,9 +36,9 @@ public abstract class Condition {
     private String contextName;
 
     /** Position constraints of the terms matched by this condition */
-    private Anchor anchor=Anchor.NONE;
+    private Anchor anchor = Anchor.NONE;
 
-    public static enum Anchor {
+    public enum Anchor {
         NONE, START, END, BOTH;
         public static Anchor create(boolean start,boolean end) {
             if (start && end) return Anchor.BOTH;
@@ -49,32 +49,32 @@ public abstract class Condition {
     }
 
     public Condition() {
-        this(null,null);
+        this(null, null);
     }
 
     public Condition(String label) {
-        this(label,null);
+        this(label, null);
     }
 
-    public Condition(String label,String context) {
-        this.label=label;
-        this.contextName=context;
+    public Condition(String label, String context) {
+        this.label = label;
+        this.contextName = context;
     }
 
     /**
      * Sets the name whatever is matched by this condition can be refered as, or null
-     * to make it unreferable
+     * to make it nonreferable
      */
-    public void setContextName(String contextName) { this.contextName=contextName; }
+    public void setContextName(String contextName) { this.contextName = contextName; }
 
     /**
-     * Returns the name whatever is matched by this condition can be refered as, or null
+     * Returns the name whatever is matched by this condition can be referred as, or null
      * if it is unreferable
      */
     public String getContextName() { return contextName; }
 
     /** Returns whether this is referable, returns context!=null by default */
-    protected boolean isReferable() { return contextName!=null; }
+    protected boolean isReferable() { return contextName != null; }
 
     /** Sets the label of this. Set to null to use the default */
     public String getLabel() { return label; }
@@ -95,15 +95,15 @@ public abstract class Condition {
     void setParent(CompositeCondition parent) { this.parent=parent; }
 
     /** Sets a positional constraint on this condition */
-    public void setAnchor(Anchor anchor) { this.anchor=anchor; }
+    public void setAnchor(Anchor anchor) { this.anchor = anchor; }
 
     /** Returns the positional constraint on this anchor. This is never null */
     public Anchor getAnchor() { return anchor; }
 
     /**
-     * <p>Returns whether this condition matches the given evaluation
+     * Returns whether this condition matches the given evaluation
      * at the <i>current</i> location of the evaluation. Calls the doesMatch
-     * method of each condition subtype.</p>
+     * method of each condition subtype.
      */
     public final boolean matches(RuleEvaluation e) {
         // TODO: With this algoritm, each choice point will move to the next choice on each reevaluation
@@ -138,32 +138,32 @@ public abstract class Condition {
 
     /** Check start anchor. Trace level 4 if no match */
     protected boolean matchesStartAnchor(RuleEvaluation e) {
-        if (anchor!=Anchor.START && anchor!=Anchor.BOTH) return true;
-        if (e.getPosition()==0) return true;
-        if (e.getTraceLevel()>=4)
-            e.trace(4,this + " must be at the start, which " + e.currentItem() + " isn't");
+        if (anchor != Anchor.START && anchor != Anchor.BOTH) return true;
+        if (e.getPosition() == 0) return true;
+        if (e.getTraceLevel() >= 4)
+            e.trace(4, this + " must be at the start, which " + e.currentItem() + " isn't");
         return false;
     }
 
     /** Check start anchor. Trace level 4 if no match */
     protected boolean matchesEndAnchor(RuleEvaluation e) {
-        if (anchor!=Anchor.END && anchor!=Anchor.BOTH) return true;
-        if (e.getPosition()>=e.items().size()) return true;
-        if (e.getTraceLevel()>=4)
-            e.trace(4,this + " must be at the end, which " + e.currentItem() + " isn't");
+        if (anchor != Anchor.END && anchor != Anchor.BOTH) return true;
+        if (e.getPosition() >= e.items().size()) return true;
+        if (e.getTraceLevel() >= 4)
+            e.trace(4, this + " must be at the end, which " + e.currentItem() + " isn't");
         return false;
     }
 
-    protected void traceResult(boolean matches,RuleEvaluation e) {
-        if (matches && e.getTraceLevel()>=3)
-            e.trace(3,"Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem());
-        if (!matches && e.getTraceLevel()>=4)
-            e.trace(4,"Did not match '" + this + "' at " + e.currentItem());
+    protected void traceResult(boolean matches, RuleEvaluation e) {
+        if (matches && e.getTraceLevel() >= 3)
+            e.trace(3, "Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem());
+        if (!matches && e.getTraceLevel() >= 4)
+            e.trace(4, "Did not match '" + this + "' at " + e.currentItem());
     }
 
     protected String getMatchInfoString(RuleEvaluation e) {
-        String matchInfo=getMatchInfo(e);
-        if (matchInfo==null) return "";
+        String matchInfo = getMatchInfo(e);
+        if (matchInfo == null) return "";
         return " as '" + matchInfo + "'";
     }
 
@@ -212,28 +212,28 @@ public abstract class Condition {
     public void makeReferences(RuleBase rules) { }
 
     protected String getLabelString() {
-        if (label==null) return "";
+        if (label == null) return "";
         return label + ":";
     }
 
     /** Whether the label matches the current item, true if there is no current item */
     protected boolean labelMatches(RuleEvaluation e) {
-        FlattenedItem flattenedItem=e.currentItem();
-        if (flattenedItem==null) return true;
-        TermItem item=flattenedItem.getItem();
-        if (item==null) return true;
-        return labelMatches(item,e);
-    }
-
-    protected boolean labelMatches(TermItem evaluationTerm,RuleEvaluation e) {
-        String indexName=evaluationTerm.getIndexName();
-        String label=getLabel();
-        if (label==null)
-            label=e.getCurrentLabel();
-        if ("".equals(indexName) && label==null) return true;
+        FlattenedItem flattenedItem = e.currentItem();
+        if (flattenedItem == null) return true;
+        TermItem item = flattenedItem.getItem();
+        if (item == null) return true;
+        return labelMatches(item, e);
+    }
+
+    protected boolean labelMatches(TermItem evaluationTerm, RuleEvaluation e) {
+        String indexName = evaluationTerm.getIndexName();
+        String label = getLabel();
+        if (label == null)
+            label = e.getCurrentLabel();
+        if ("".equals(indexName) && label == null) return true;
         if (indexName.equals(label)) return true;
-        if (e.getTraceLevel()>=4)
-            e.trace(4,"'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label);
+        if (e.getTraceLevel() >= 4)
+            e.trace(4, "'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label);
         return false;
     }
 
@@ -242,13 +242,14 @@ public abstract class Condition {
 
     protected boolean isDefaultContextName() { return false; }
 
+    @Override
     public String toString() {
-        String contextString="";
-        String nameSpaceString="";
-        if (contextName!=null && !isDefaultContextName())
-            contextString=contextName + "/";
-        if (getNameSpace()!=null)
-            nameSpaceString=getNameSpace() + ".";
+        String contextString = "";
+        String nameSpaceString = "";
+        if (contextName != null && !isDefaultContextName())
+            contextString = contextName + "/";
+        if (getNameSpace() != null)
+            nameSpaceString = getNameSpace() + ".";
         return contextString + nameSpaceString + toInnerString();
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java
index bfddb55566c..e468526fbe0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java
@@ -1,7 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics.rule;
 
-import com.yahoo.prelude.query.TermItem;
 import com.yahoo.prelude.querytransform.PhraseMatcher;
 import com.yahoo.prelude.semantics.RuleBase;
 import com.yahoo.prelude.semantics.RuleBaseException;
@@ -11,8 +10,6 @@ import com.yahoo.prelude.semantics.engine.FlattenedItem;
 import com.yahoo.prelude.semantics.engine.RuleEvaluation;
 import com.yahoo.protect.Validator;
 
-import java.util.Map;
-
 /**
  * A reference to a named condition
  *
@@ -24,7 +21,7 @@ public class ConditionReference extends Condition {
     private String conditionName;
 
     /**
-     * The actual condition references by this, or null if not initialized or not found,
+     * The actual condition referenced by this, or null if not initialized or not found,
      * or if this is really an automata reference
      */
     private NamedCondition namedCondition;
@@ -33,7 +30,7 @@ public class ConditionReference extends Condition {
      * True if this condition should be looked up in the automata
      * annotations of the item instead of by reference to another item
      */
-    private boolean automataLookup=false;
+    private boolean automataLookup = false;
 
     public ConditionReference(String conditionName) {
         this(null,conditionName);
@@ -42,37 +39,36 @@ public class ConditionReference extends Condition {
     public ConditionReference(String label,String conditionName) {
         super(label);
         Validator.ensureNotNull("Name of referenced condition",conditionName);
-        this.conditionName=conditionName;
+        this.conditionName = conditionName;
         setContextName(conditionName);
     }
 
     /** Returns the name of the referenced rule, never null */
     public String getConditionName() { return conditionName; }
 
-    public void setConditionName(String name) { this.conditionName=name; }
+    public void setConditionName(String name) { this.conditionName = name; }
 
     public boolean doesMatch(RuleEvaluation e) {
         if (automataLookup) return automataMatch(e);
 
-        if (namedCondition==null)
-            throw new EvaluationException("Condition reference '" + conditionName +
-                                          "' not found or not initialized");
+        if (namedCondition == null)
+            throw new EvaluationException("Condition reference '" + conditionName + "' not found or not initialized");
 
         return namedCondition.matches(e);
     }
 
     private boolean automataMatch(RuleEvaluation e) {
-        FlattenedItem current=e.currentItem();
-        if (current==null) return false;
+        FlattenedItem current = e.currentItem();
+        if (current == null) return false;
 
-        Object annotation=current.getItem().getAnnotation(conditionName);
-        if (annotation==null) return false;
+        Object annotation = current.getItem().getAnnotation(conditionName);
+        if (annotation == null) return false;
         if (! (annotation instanceof PhraseMatcher.Phrase)) return false;
 
-        PhraseMatcher.Phrase phrase=(PhraseMatcher.Phrase)annotation;
+        PhraseMatcher.Phrase phrase = (PhraseMatcher.Phrase)annotation;
 
-        Choicepoint choicePoint=e.getChoicepoint(this,true);
-        boolean matches=automataMatchPhrase(phrase,e);
+        Choicepoint choicePoint = e.getChoicepoint(this,true);
+        boolean matches = automataMatchPhrase(phrase,e);
 
         if (!matches && e.isInNegation()) { // TODO: Temporary hack! Works for single items only
             e.addMatch(current,null);
@@ -84,40 +80,41 @@ public class ConditionReference extends Condition {
         return matches;
     }
 
-    private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase,RuleEvaluation e) {
-        for (PhraseMatcher.Phrase.MatchIterator i=phrase.itemIterator(); i.hasNext(); ) {
+    private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase, RuleEvaluation e) {
+        for (PhraseMatcher.Phrase.MatchIterator i = phrase.itemIterator(); i.hasNext(); ) {
             i.next();
-            FlattenedItem current=e.currentItem();
-            if (current==null) return false;
+            FlattenedItem current = e.currentItem();
+            if (current == null) return false;
             if (!labelMatches(e.currentItem().getItem(),e)) return false;
             if (!e.isInNegation())
-                e.addMatch(current,i.getReplace());
+                e.addMatch(current, i.getReplace());
             e.next();
         }
-        if (phrase.getLength()>phrase.getBackedLength()) return false; // The underlying composite item has changed
+        if (phrase.getLength() > phrase.getBackedLength()) return false; // The underlying composite item has changed
         return true;
     }
 
     public void makeReferences(RuleBase ruleBase) {
-        namedCondition=ruleBase.getCondition(conditionName);
-        if (namedCondition==null) { // Then this may reference some automata value, if we have an automata
+        namedCondition = ruleBase.getCondition(conditionName);
+        if (namedCondition == null) { // Then this may reference some automata value, if we have an automata
             if (ruleBase.usesAutomata())
-                automataLookup=true;
+                automataLookup = true;
             else
                 throw new RuleBaseException("Referenced condition '" + conditionName +
-                                                                        "' does not exist in " + ruleBase);
+                                            "' does not exist in " + ruleBase);
         }
     }
 
     protected boolean hasOpenChoicepoint(RuleEvaluation e) {
-        if (namedCondition==null) return false;
+        if (namedCondition == null) return false;
         return namedCondition.getCondition().hasOpenChoicepoint(e);
     }
 
     protected boolean isDefaultContextName() {
-        return getContextName()==null || getContextName().equals(conditionName);
+        return getContextName() == null || getContextName().equals(conditionName);
     }
 
+    @Override
     protected String toInnerString() {
         return "[" + conditionName + "]";
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java
index 0d5ce78baf0..af7aab23b85 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java
@@ -44,7 +44,7 @@ public class LiteralPhraseProduction extends TermProduction {
     /** Returns a read only view of the terms produced by this, never null */
     public List<String> getTerms() { return Collections.unmodifiableList(terms); }
 
-    public void produce(RuleEvaluation e,int offset) {
+    public void produce(RuleEvaluation e, int offset) {
         PhraseItem newPhrase = new PhraseItem();
         newPhrase.setIndexName(getLabel());
         for (String term : terms)
@@ -52,13 +52,13 @@ public class LiteralPhraseProduction extends TermProduction {
 
         if (replacing) {
             Match matched = e.getNonreferencedMatch(0);
-            insertMatch(e, matched, newPhrase, offset);
+            insertMatch(e, matched, List.of(newPhrase), offset);
         }
         else {
             newPhrase.setWeight(getWeight());
             if (e.getTraceLevel() >= 6)
                 e.trace(6, "Adding '" + newPhrase + "'");
-            e.addItem(newPhrase, getTermType());
+            e.addItems(List.of(newPhrase), getTermType());
         }
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java
index ed21c9643c3..66507719a11 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java
@@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match;
 import com.yahoo.prelude.semantics.engine.RuleEvaluation;
 import com.yahoo.protect.Validator;
 
+import java.util.List;
+
 /**
  * A literal term produced by a production rule
  *
@@ -63,13 +65,13 @@ public class LiteralTermProduction extends TermProduction {
         if (replacing) {
             Match matched = e.getNonreferencedMatch(0);
             newItem.setWeight(matched.getItem().getWeight());
-            insertMatch(e, matched, newItem, offset);
+            insertMatch(e, matched, List.of(newItem), offset);
         }
         else {
             newItem.setWeight(getWeight());
             if (e.getTraceLevel() >= 6)
                 e.trace(6, "Adding '" + newItem + "'");
-            e.addItem(newItem, getTermType());
+            e.addItems(List.of(newItem), getTermType());
         }
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java
index a267d274d5a..cbb0b4ab10a 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java
@@ -33,7 +33,7 @@ public class NamedCondition {
             e.indentTrace();
         }
 
-        boolean matches=condition.matches(e);
+        boolean matches = condition.matches(e);
 
         if (e.getTraceLevel() >= 3) {
             e.unindentTrace();
@@ -50,6 +50,7 @@ public class NamedCondition {
      * This string representation can always be reparsed to produce an
      * identical rule to this one.
      */
+    @Override
     public String toString() {
         return "[" + conditionName + "] :- " + condition.toString();
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java
index a971020ea90..debc3150959 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java
@@ -13,20 +13,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation;
 public abstract class Production {
 
     /** True to add, false to replace, default true */
-    protected boolean replacing=true;
+    protected boolean replacing = true;
 
     /** The (0-base) position of this term in the productions of this rule */
-    private int position=0;
+    private int position = 0;
 
     /** The weight (strength) of this production as a percentage (default is 100) */
-    private int weight=100;
+    private int weight = 100;
 
     /** Creates a produced template term with no label and the default type */
     public Production() {
     }
 
     /** True to replace, false to add, if this production can do both. Default true. */
-    public void setReplacing(boolean replacing) { this.replacing=replacing; }
+    public void setReplacing(boolean replacing) { this.replacing = replacing; }
 
     public int getPosition() { return position; }
 
@@ -45,7 +45,7 @@ public abstract class Production {
      * @param offset the offset position at which to produce this. Offsets are used to produce multiple items
      *        at one position, inserted in the right order.
      */
-    public abstract void produce(RuleEvaluation e,int offset);
+    public abstract void produce(RuleEvaluation e, int offset);
 
     /**
      * Called to add the references into the condition of this rule made by this production
@@ -55,6 +55,7 @@ public abstract class Production {
     void addMatchReferences(Set<String> matchReferences) { }
 
     /** All instances of this produces a parseable string output */
+    @Override
     public final String toString() {
         return toInnerString() + (getWeight()!=100 ? ("!" + getWeight()) : "");
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java
index 46f855ca95d..5d20075cd97 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java
@@ -54,6 +54,7 @@ public class ProductionList {
         }
     }
 
+    @Override
     public String toString() {
         StringBuilder buffer = new StringBuilder();
         for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java
index 20ebc41e57f..b320d10b43d 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java
@@ -19,18 +19,18 @@ public abstract class ProductionRule {
     private Condition condition;
 
     /** What is produced when this rule is true */
-    private ProductionList production=new ProductionList();
+    private ProductionList production = new ProductionList();
 
     /** The set of match name Strings which the production part of this rule references */
-    private Set<String> matchReferences=new java.util.LinkedHashSet<>();
+    private final Set<String> matchReferences = new java.util.LinkedHashSet<>();
 
     /** Sets what must be true for this rule to be true */
-    public void setCondition(Condition condition) { this.condition=condition; }
+    public void setCondition(Condition condition) { this.condition = condition; }
 
     public Condition getCondition() { return condition; }
 
     /** Sets what is produced when this rule is true */
-    public void setProduction(ProductionList production) { this.production=production; }
+    public void setProduction(ProductionList production) { this.production = production; }
 
     public ProductionList getProduction() { return production; }
 
@@ -64,6 +64,7 @@ public abstract class ProductionRule {
      * This string representation can always be reparsed to produce an
      * identical rule to this one.
      */
+    @Override
     public String toString() {
         return condition.toString() + " " + getSymbol() + " " + production.toString();
     }
@@ -80,21 +81,27 @@ public abstract class ProductionRule {
      * This default implementation returns false;
      */
     public boolean isLoop() {
-        // TODO: There are many more possible loops, we should probably detect
-        // a few more obvious ones
+        // TODO: There are many more possible loops, we should probably detect a few more obvious ones
         if (conditionIsEllipsAndOtherNameSpacesOnly(getCondition())) return true;
+        if (producesItself()) return true;
         return false;
     }
 
     private boolean conditionIsEllipsAndOtherNameSpacesOnly(Condition condition) {
         if (condition instanceof EllipsisCondition) return true;
         if (! (condition instanceof CompositeCondition)) return false;
-        for (Iterator<Condition> i=((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) {
-            Condition child= i.next();
-            if (child.getNameSpace()==null && conditionIsEllipsAndOtherNameSpacesOnly(child))
+        for (Iterator<Condition> i = ((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) {
+            Condition child = i.next();
+            if (child.getNameSpace() == null && conditionIsEllipsAndOtherNameSpacesOnly(child))
                 return true;
         }
         return false;
     }
 
+    private boolean producesItself() {
+        return production.productionList()
+                         .stream()
+                         .anyMatch(p -> (p instanceof ReferenceTermProduction) && ((ReferenceTermProduction)p).producesAll());
+    }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java
index af7abf325e7..00562d3953e 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java
@@ -1,15 +1,19 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics.rule;
 
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
 import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PhraseItem;
 import com.yahoo.prelude.query.TermType;
 import com.yahoo.prelude.semantics.engine.EvaluationException;
 import com.yahoo.prelude.semantics.engine.Match;
 import com.yahoo.prelude.semantics.engine.ReferencedMatches;
 import com.yahoo.prelude.semantics.engine.RuleEvaluation;
-import com.yahoo.protect.Validator;
 
 /**
  * A term produced by a production rule which takes its actual term value
@@ -19,90 +23,114 @@ import com.yahoo.protect.Validator;
  */
 public class ReferenceTermProduction extends TermProduction {
 
-    private String reference;
+    private final String reference;
+    private final boolean produceAll;
 
     /**
      * Creates a new produced reference term
      *
      * @param reference the label of the condition this should take it's value from
      */
-    public ReferenceTermProduction(String reference) {
+    public ReferenceTermProduction(String reference, boolean produceAll) {
         super();
-        setReference(reference);
+        this.reference = Objects.requireNonNull(reference, "Reference cannot be null");
+        this.produceAll = produceAll;
     }
 
     /**
      * Creates a new produced reference term
      *
-     * @param reference the label of the condition this should take it's value from
+     * @param reference the label of the condition this should take its value from
      * @param termType the type of the term to produce
      */
-    public ReferenceTermProduction(String reference, TermType termType) {
+    public ReferenceTermProduction(String reference, TermType termType, boolean produceAll) {
         super(termType);
-        setReference(reference);
+        this.reference = Objects.requireNonNull(reference, "Reference cannot be null");
+        this.produceAll = produceAll;
     }
 
     /**
      * Creates a new produced reference term
      *
      * @param label the label of the produced term
-     * @param reference the label of the condition this should take it's value from
+     * @param reference the label of the condition this should take its value from
      */
-    public ReferenceTermProduction(String label, String reference) {
+    public ReferenceTermProduction(String label, String reference, boolean produceAll) {
         super(label);
-        setReference(reference);
+        this.reference = Objects.requireNonNull(reference, "Reference cannot be null");
+        this.produceAll = produceAll;
     }
 
     /**
      * Creates a new produced reference term
      *
      * @param label the label of the produced term
-     * @param reference the label of the condition this should take it's value from
+     * @param reference the label of the condition this should take its value from
      * @param termType the type of term to produce
      */
-    public ReferenceTermProduction(String label, String reference, TermType termType) {
+    public ReferenceTermProduction(String label, String reference, TermType termType, boolean produceAll) {
         super(label, termType);
-        setReference(reference);
-    }
-
-    /** The label of the condition this should take its value from, never null */
-    public void setReference(String reference) {
-        Validator.ensureNotNull("reference name of a produced reference term",reference);
-        this.reference = reference;
+        this.reference = Objects.requireNonNull(reference, "Reference cannot be null");
+        this.produceAll = produceAll;
     }
 
     /** Returns the label of the condition this should take its value from, never null */
     public String getReference() { return reference; }
 
+    public boolean producesAll() { return produceAll; }
+
+    @Override
     void addMatchReferences(Set<String> matchReferences) {
         matchReferences.add(reference);
     }
 
-    public void produce(RuleEvaluation e, int offset) {
+    public void produce(RuleEvaluation e, int ignored) {
         ReferencedMatches referencedMatches = e.getReferencedMatches(reference);
         if (referencedMatches == null)
             throw new EvaluationException("Referred match '" + reference + "' not found");
         if (replacing) {
-            replaceMatches(e, referencedMatches);
+            e.removeMatches(referencedMatches);
+        }
+
+        var match = referencedMatches.matchIterator().next();
+        if (produceAll) {
+            // produce all terms in the condition
+            NamedCondition namedCondition = e.getEvaluation().ruleBase().getCondition(referencedMatches.getContextName());
+            ChoiceCondition choices = (ChoiceCondition)namedCondition.getCondition();
+            List<Item> items = new ArrayList<>();
+            for (Iterator<Condition> i = choices.conditionIterator(); i.hasNext();) {
+                Condition condition = i.next();
+                if (condition instanceof TermCondition) {
+                    items.add(match.toItem(getLabel(), ((TermCondition)condition).term()));
+                }
+                else if (condition instanceof SequenceCondition) {
+                    PhraseItem phrase = new PhraseItem(getLabel());
+                    for (var term : ((SequenceCondition)condition).conditions())
+                        phrase.addItem(match.toItem(getLabel(), ((TermCondition)term).term()));
+                    items.add(phrase);
+                }
+                else {
+                    // Until we validate this at construction time
+                    throw new EvaluationException("Could not produce all terms in " + namedCondition + " as it is " +
+                                                  "not a term or sequence condition");
+                }
+            }
+            produce(e, match, items, 0);
         }
         else {
-            addMatches(e, referencedMatches);
+            // produce just the matching term
+            produce(e, match, List.of(referencedMatches.toItem(getLabel())), 0);
         }
     }
 
-    public void replaceMatches(RuleEvaluation e, ReferencedMatches referencedMatches) {
-        Item referencedItem = referencedMatches.toItem(getLabel());
-        if (referencedItem == null) return;
-        e.removeMatches(referencedMatches);
-        insertMatch(e, referencedMatches.matchIterator().next(), referencedItem, 0);
-    }
-
-    private void addMatches(RuleEvaluation e, ReferencedMatches referencedMatches) {
-        Item referencedItem = referencedMatches.toItem(getLabel());
-        if (referencedItem == null) return;
-        e.addItem(referencedItem, getTermType());
+    private void produce(RuleEvaluation e, Match match, List<Item> items, int offset) {
+        if (replacing)
+            insertMatch(e, match, items, offset);
+        else
+            e.addItems(items, getTermType());
     }
 
+    @Override
     public String toInnerTermString() {
         return getLabelString() + "[" + reference + "]";
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java
index a2bbf72a53b..eaff66a0bb0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java
@@ -1,7 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.semantics.rule;
 
-import com.yahoo.prelude.query.TermItem;
 import com.yahoo.prelude.semantics.engine.NameSpace;
 import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics;
 import com.yahoo.prelude.semantics.engine.RuleEvaluation;
@@ -49,6 +48,9 @@ public class TermCondition extends Condition {
         }
     }
 
+    public String term() { return term; }
+
+    @Override
     public String toInnerString() {
         return getLabelString() + term;
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java
index 29e4982ac17..41d15bc9262 100644
--- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java
+++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java
@@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match;
 import com.yahoo.prelude.semantics.engine.RuleEvaluation;
 import com.yahoo.protect.Validator;
 
+import java.util.List;
+
 /**
  * A new term produced by a production rule
  *
@@ -59,9 +61,9 @@ public abstract class TermProduction extends Production {
      * Inserts newItem at the position of this match
      * TODO: Move to ruleevaluation
      */
-    protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) {
+    protected void insertMatch(RuleEvaluation e, Match matched, List<Item> newItems, int offset) {
         if (getWeight() != 100)
-            newItem.setWeight(getWeight());
+            newItems.forEach(item -> item.setWeight(getWeight()));
         int insertPosition = matched.getPosition() + offset;
 
         // This check is necessary (?) because earlier items may have been removed
@@ -71,9 +73,9 @@ public abstract class TermProduction extends Production {
             insertPosition = matched.getParent().getItemCount();
         }
 
-        e.insertItem(newItem, matched.getParent(), insertPosition,getTermType());
+        e.insertItems(newItems, matched.getParent(), insertPosition, getTermType());
         if (e.getTraceLevel() >= 6)
-            e.trace(6, "Inserted item '" + newItem + "' at position " + insertPosition + " producing " +
+            e.trace(6, "Inserted items '" + newItems + "' at position " + insertPosition + " producing " +
                        e.getEvaluation().getQuery().getModel().getQueryTree());
     }
 
diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj
index 46117374e59..39ea6435393 100644
--- a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj
+++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj
@@ -69,6 +69,7 @@ TOKEN :
     <EQUALS: "="> |
     <EXCLAMATION: "!"> |
     <INCLUDEDIRECTIVE: "@include"> |
+    <LANGUAGEDIRECTIVE: "@language"> |
     <LARGER: ">"> |
     <LARGEREQUALS: ">="> |
     <LEFTBRACE: "("> |
@@ -86,8 +87,8 @@ TOKEN :
     <SLASH: "/"> |
     <SMALLER: "<"> |
     <SMALLEREQUALS: "<="> |
+    <STAR: "*"> |
     <STEMMINGDIRECTIVE: "@stemming"> |
-    <LANGUAGEDIRECTIVE: "@language"> |
     <SUPERDIRECTIVE: "@super"> |
     <IDENTIFIER: (~[
 "\u0000"-"\u002f","\u003a"-"\u003f","\u005b"-"\u005d","\u007b"-"\u00a7","\u00a9","\u00ab"-"\u00ae","\u00b0"-"\u00b3","\u00b6"-"\u00b7","\u00b9","\u00bb"-"\u00bf",
@@ -312,10 +313,14 @@ NamespaceProduction namespaceProduction() :
 ReferenceTermProduction referenceTermProduction() :
 {
     String reference;
+    boolean produceAll = false;
 }
 {
-    <LEFTSQUAREBRACKET> reference = referenceIdentifier() <RIGHTSQUAREBRACKET>
-    { return new ReferenceTermProduction(reference); }
+    <LEFTSQUAREBRACKET>
+        reference = referenceIdentifier()
+        (<STAR> { produceAll = true; })?
+    <RIGHTSQUAREBRACKET>
+    { return new ReferenceTermProduction(reference, produceAll); }
 }
 
 LiteralTermProduction literalTermProduction() :
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java
index eb69372c22b..17eb4120b84 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java
@@ -29,7 +29,7 @@ public class ConditionTestCase {
         var linguistics = new RuleBaseLinguistics(new SimpleLinguistics());
         TermCondition term = new TermCondition("foo", linguistics);
         Query query = new Query("?query=foo");
-        assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation()));
+        assertTrue(term.matches(new Evaluation(query, null).freshRuleEvaluation()));
     }
 
     @Test
@@ -41,11 +41,11 @@ public class ConditionTestCase {
         sequence.addCondition(term1);
         sequence.addCondition(term2);
         Query query = new Query("?query=foo+bar");
-        assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query).freshRuleEvaluation()));
+        assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query, null).freshRuleEvaluation()));
         Query query2 = new Query("?query=foo");
-        assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2).freshRuleEvaluation()));
+        assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2, null).freshRuleEvaluation()));
         Query query3 = new Query("?query=bar");
-        assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3).freshRuleEvaluation()));
+        assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3, null).freshRuleEvaluation()));
     }
 
     @Test
@@ -57,11 +57,11 @@ public class ConditionTestCase {
         choice.addCondition(term1);
         choice.addCondition(term2);
         Query query1 = new Query("?query=foo+bar");
-        assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1).freshRuleEvaluation()));
+        assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1, null).freshRuleEvaluation()));
         Query query2 = new Query("?query=foo");
-        assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2).freshRuleEvaluation()));
+        assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2, null).freshRuleEvaluation()));
         Query query3 = new Query("?query=bar");
-        assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3).freshRuleEvaluation()));
+        assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3, null).freshRuleEvaluation()));
     }
 
     @Test
@@ -75,13 +75,13 @@ public class ConditionTestCase {
         ProductionRule rule = new ReplacingProductionRule();
         rule.setCondition(reference);
         rule.setProduction(new ProductionList());
-        RuleBase ruleBase = new RuleBase("test", linguistics.linguistics());
+        RuleBase ruleBase = new RuleBase("test");
         ruleBase.addCondition(named);
         ruleBase.addRule(rule);
         ruleBase.initialize();
 
         Query query = new Query("?query=foo");
-        assertTrue(query + "  matches " + reference,reference.matches(new Evaluation(query).freshRuleEvaluation()));
+        assertTrue(query + "  matches " + reference,reference.matches(new Evaluation(query, null).freshRuleEvaluation()));
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java
index b91e9441a2b..bafb2cb6a73 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java
@@ -32,7 +32,7 @@ public class ProductionRuleTestCase {
         NamedCondition named = new NamedCondition("brand", term);
         ConditionReference reference = new ConditionReference("brand");
 
-        TermProduction termProduction = new ReferenceTermProduction("brand", "brand");
+        TermProduction termProduction = new ReferenceTermProduction("brand", "brand", false);
         ProductionList productionList = new ProductionList();
         productionList.addProduction(termProduction);
 
@@ -41,7 +41,7 @@ public class ProductionRuleTestCase {
         rule.setProduction(productionList);
 
         // To initialize the condition reference...
-        RuleBase ruleBase = new RuleBase("test", linguistics.linguistics());
+        RuleBase ruleBase = new RuleBase("test");
         ruleBase.addCondition(named);
         ruleBase.addRule(rule);
         ruleBase.initialize();
@@ -49,7 +49,7 @@ public class ProductionRuleTestCase {
         assertTrue("Brand is referenced", rule.matchReferences().contains("brand"));
 
         Query query = new Query("?query=sony");
-        RuleEvaluation e = new Evaluation(query).freshRuleEvaluation();
+        RuleEvaluation e = new Evaluation(query, null).freshRuleEvaluation();
         assertTrue(rule.matches(e));
         rule.produce(e);
         assertEquals("AND brand:sony", query.getModel().getQueryTree().getRoot().toString());
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java
new file mode 100644
index 00000000000..8f69be2f710
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java
@@ -0,0 +1,33 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.semantics.test;
+
+import org.junit.Test;
+
+/**
+ * @author bratseth
+ */
+public class SynonymTestCase {
+
+    @Test
+    public void testReplacingBySynonyms() {
+        var tester = new RuleBaseTester("synonyms.sr");
+        tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:foo");
+        tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:bar");
+        tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:baz");
+        tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:word");
+        tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:a index1:phrase");
+        tester.assertSemantics("index1:other", "index1:other");
+    }
+
+    @Test
+    public void testAddingSynonyms() {
+        var tester = new RuleBaseTester("synonyms.sr");
+        tester.assertSemantics("EQUIV index2:foo index2:baz index2:bar", "index2:foo");
+        tester.assertSemantics("EQUIV index2:bar index2:foo index2:baz", "index2:bar");
+        tester.assertSemantics("EQUIV index2:baz index2:foo index2:bar", "index2:baz");
+        tester.assertSemantics("EQUIV index2:word index2:\"a phrase\"", "index2:word");
+        tester.assertSemantics("EQUIV index2:\"a phrase\" index2:word", "index2:a index2:phrase");
+        tester.assertSemantics("index2:other", "index2:other");
+    }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr
index 9a147887207..32f8e86b59f 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr
@@ -5,6 +5,4 @@ equiv1 +> =equiv2 =equiv3;
 
 testfield:[test] -> =testfield:e1 =testfield:e2 =testfield:e3;
 
-synonymfield:[test] -> =[test];
-
 [test] :- foo, bar, baz;
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr
new file mode 100644
index 00000000000..4220f807733
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr
@@ -0,0 +1,11 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+index1:[synonyms1] -> =index1:[synonyms1*]; # Replace by equiv(foo, bar, baz) when the query contains foo, bar or baz
+index1:[synonyms2] -> =index1:[synonyms2*]; # with phrase
+
+
+index2:[synonyms1] +> =index2:[synonyms1*]; # Add equiv(foo, bar, baz) when the query contains foo, bar or baz
+index2:[synonyms2] +> =index2:[synonyms2*]; # with phrase
+
+[synonyms1] :- foo, baz, bar;
+[synonyms2] :- word, a phrase;