Make defaultIndex part of parser state

10 files changed, 96 insertions, 123 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
index d682ca9564e..ad1e8525b4b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
@@ -7,7 +7,6 @@ import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.query.AndSegmentItem;
 import com.yahoo.prelude.query.CompositeItem;
-import com.yahoo.prelude.query.IndexedItem;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.NullItem;
 import com.yahoo.prelude.query.PhraseItem;
@@ -17,7 +16,6 @@ import com.yahoo.search.query.QueryTree;
 import com.yahoo.search.query.parser.Parsable;
 import com.yahoo.search.query.parser.ParserEnvironment;
 
-import java.util.Iterator;
 import java.util.List;
 import java.util.ListIterator;
 
@@ -29,6 +27,7 @@ import java.util.ListIterator;
  */
 public abstract class AbstractParser implements CustomParser {
 
+
     /** The current submodes of this parser */
     protected Submodes submodes = new Submodes();
 
@@ -41,6 +40,8 @@ public abstract class AbstractParser implements CustomParser {
     /** The IndexFacts.Session of this query */
     protected IndexFacts.Session indexFacts;
 
+    protected String defaultIndex;
+
     /**
      * The counter for braces in URLs, braces in URLs are accepted so long as
      * they are balanced.
@@ -134,38 +135,34 @@ public abstract class AbstractParser implements CustomParser {
 
     @Override
     public final Item parse(String queryToParse, String filterToParse, Language parsingLanguage,
-                            IndexFacts.Session indexFacts, String defaultIndexName) {
-        return parse(queryToParse, filterToParse, parsingLanguage, indexFacts, defaultIndexName, null);
+                            IndexFacts.Session indexFacts, String defaultIndex) {
+        return parse(queryToParse, filterToParse, parsingLanguage, indexFacts, defaultIndex, null);
     }
 
     private Item parse(String queryToParse, String filterToParse, Language parsingLanguage,
-                       IndexFacts.Session indexFacts, String defaultIndexName, Parsable parsable) {
+                       IndexFacts.Session indexFacts, String defaultIndex, Parsable parsable) {
         if (queryToParse == null) return null;
-
-        tokenize(queryToParse, defaultIndexName, indexFacts, parsingLanguage);
+        tokenize(queryToParse, defaultIndex, indexFacts, parsingLanguage);
 
         if (parsingLanguage == null && parsable != null) {
-            String detectionText = generateLanguageDetectionTextFrom(tokens, indexFacts, defaultIndexName);
+            String detectionText = generateLanguageDetectionTextFrom(tokens, indexFacts, defaultIndex);
             if (detectionText.isEmpty()) // heuristic detection text extraction is fallible
                 detectionText = queryToParse;
             parsingLanguage = parsable.getOrDetectLanguage(detectionText);
         }
-        setState(parsingLanguage, indexFacts);
 
-        Item root = parseItems(defaultIndexName);
+        setState(parsingLanguage, indexFacts, defaultIndex);
+        Item root = parseItems();
+
         if (filterToParse != null) {
             AnyParser filterParser = new AnyParser(environment);
             if (root == null) {
-                root = filterParser.parseFilter(filterToParse, parsingLanguage, indexFacts);
+                root = filterParser.parseFilter(filterToParse, parsingLanguage, indexFacts, defaultIndex);
             } else {
-                root = filterParser.applyFilter(root, filterToParse, parsingLanguage, indexFacts);
+                root = filterParser.applyFilter(root, filterToParse, parsingLanguage, indexFacts, defaultIndex);
             }
         }
-        root = simplifyPhrases(root);
-        if (defaultIndexName != null) {
-            assignDefaultIndex(indexFacts.getCanonicName(defaultIndexName), root);
-        }
-        return root;
+        return simplifyPhrases(root);
     }
 
     /**
@@ -232,28 +229,7 @@ public abstract class AbstractParser implements CustomParser {
         return kind.equals(tokenOrNull.kind);
     }
 
-    protected abstract  Item parseItems(String defaultIndexName);
-
-    /**
-     * Assigns the default index to query terms having no default index. The
-     * parser _should_ have done this, for some reason it doesn't.
-     *
-     * @param defaultIndex the default index to assign
-     * @param item         the item to check
-     */
-    private static void assignDefaultIndex(String defaultIndex, Item item) {
-        if (defaultIndex == null || item == null) return;
-
-        if (item instanceof IndexedItem indexName) {
-            if ("".equals(indexName.getIndexName()))
-                indexName.setIndexName(defaultIndex);
-        } 
-        else if (item instanceof CompositeItem) {
-            Iterator<Item> items = ((CompositeItem)item).getItemIterator();
-            while (items.hasNext())
-                assignDefaultIndex(defaultIndex, items.next());
-        }
-    }
+    protected abstract Item parseItems();
 
     /**
      * Unicode normalizes some piece of natural language text. The chosen form
@@ -265,10 +241,11 @@ public abstract class AbstractParser implements CustomParser {
         return environment.getLinguistics().getNormalizer().normalize(input);
     }
 
-    protected void setState(Language queryLanguage, IndexFacts.Session indexFacts) {
+    protected void setState(Language queryLanguage, IndexFacts.Session indexFacts, String defaultIndex) {
         this.indexFacts = indexFacts;
-        language = queryLanguage;
-        submodes.reset();
+        this.defaultIndex = defaultIndex;
+        this.language = queryLanguage;
+        this.submodes.reset();
     }
 
     /**
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java
index 22bdf145e90..8f98763a838 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java
@@ -31,7 +31,8 @@ public class AdvancedParser extends StructuredParser {
         super(environment);
     }
 
-    protected Item parseItems(String defaultIndexName) {
+    @Override
+    protected Item parseItems() {
         return advancedItems(true);
     }
 
@@ -64,45 +65,40 @@ public class AdvancedParser extends StructuredParser {
         boolean expectingOperator = false;
 
         do {
-            item = null;
-
+            item = indexableItem().getFirst();
             if (item == null) {
-                item = indexableItem();
-                if (item == null) {
-                    item = compositeItem();
-                    itemIsComposite = true;
-                } else {
-                    itemIsComposite = false;
-                }
-                if (item != null) {
-                    Item newTop = null;
+                item = compositeItem();
+                itemIsComposite = true;
+            } else {
+                itemIsComposite = false;
+            }
+            if (item != null) {
+                Item newTop = null;
 
-                    if (expectingOperator) {
-                        newTop = handleAdvancedOperator(topLevelItem, item,
-                                topLevelIsClosed);
-                    }
-                    if (newTop != null) { // Operator found
-                        topLevelIsClosed = false;
-                        expectingOperator = false;
-                        topLevelItem = newTop;
-                    } else if (topLevelItem == null) {
-                        topLevelItem = item;
-                        if (itemIsComposite) {
-                            topLevelIsClosed = true;
-                        }
-                        expectingOperator = true;
-                    } else if (topLevelItem instanceof CompositeItem && !(topLevelItem instanceof SegmentItem)) {
-                        ((CompositeItem) topLevelItem).addItem(item);
-                        expectingOperator = true;
-                    } else {
-                        AndItem and = new AndItem();
-
-                        and.addItem(topLevelItem);
-                        and.addItem(item);
-                        topLevelItem = and;
-                        topLevelIsClosed = false;
-                        expectingOperator = true;
+                if (expectingOperator) {
+                    newTop = handleAdvancedOperator(topLevelItem, item, topLevelIsClosed);
+                }
+                if (newTop != null) { // Operator found
+                    topLevelIsClosed = false;
+                    expectingOperator = false;
+                    topLevelItem = newTop;
+                } else if (topLevelItem == null) {
+                    topLevelItem = item;
+                    if (itemIsComposite) {
+                        topLevelIsClosed = true;
                     }
+                    expectingOperator = true;
+                } else if (topLevelItem instanceof CompositeItem && !(topLevelItem instanceof SegmentItem)) {
+                    ((CompositeItem) topLevelItem).addItem(item);
+                    expectingOperator = true;
+                } else {
+                    AndItem and = new AndItem();
+
+                    and.addItem(topLevelItem);
+                    and.addItem(item);
+                    topLevelItem = and;
+                    topLevelIsClosed = false;
+                    expectingOperator = true;
                 }
             }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
index 09583f420cb..9a60eaef76b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
@@ -41,16 +41,16 @@ public class AllParser extends SimpleParser {
     }
 
     @Override
-    protected Item parseItems(String defaultIndexName) {
+    protected Item parseItems() {
         int position = tokens.getPosition();
         try {
-            return parseItemsBody(defaultIndexName);
+            return parseItemsBody();
         } finally {
             tokens.setPosition(position);
         }
     }
 
-    protected Item parseItemsBody(String defaultIndexName) {
+    protected Item parseItemsBody() {
         // Algorithm: Collect positive, negative, and and'ed items, then combine.
         CompositeItem and = null;
         NotItem not = null; // Store negatives here as we go
@@ -65,7 +65,7 @@ public class AllParser extends SimpleParser {
 
             current = positiveItem();
             if (current == null)
-                current = indexableItem(defaultIndexName);
+                current = indexableItem().getFirst();
             if (current == null)
                 current = compositeItem();
 
@@ -129,8 +129,9 @@ public class AllParser extends SimpleParser {
         try {
             if ( ! tokens.skip(MINUS)) return null;
             if (tokens.currentIsNoIgnore(SPACE)) return null;
-
-            item = indexableItem();
+            var itemAndExplicitIndex = indexableItem();
+            item = itemAndExplicitIndex.getFirst();
+            boolean explicitIndex = itemAndExplicitIndex.getSecond();
             if (item == null) {
                 item = compositeItem();
 
@@ -155,11 +156,11 @@ public class AllParser extends SimpleParser {
             // but interpret -(N) as a negative item matching a positive number
             // but interpret --N as a negative item matching a negative number
             if (item instanceof IntItem &&
-                ((IntItem)item).getIndexName().isEmpty() &&
+                ! explicitIndex &&
                 ! isComposited &&
-                ! ((IntItem)item).getNumber().startsWith(("-")))
+                ! ((IntItem)item).getNumber().startsWith(("-"))) {
                 item = null;
-
+            }
             return item;
         } finally {
             if (item == null) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java
index 603c3258ce2..bf778409364 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java
@@ -29,12 +29,13 @@ public class AnyParser extends SimpleParser {
         super(environment);
     }
 
-    protected Item parseItems(String defaultIndexName) {
-        return anyItems(true, defaultIndexName);
+    @Override
+    protected Item parseItems() {
+        return anyItems(true);
     }
 
-    Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) {
-        setState(queryLanguage, indexFacts);
+    Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts, String defaultIndex) {
+        setState(queryLanguage, indexFacts, defaultIndex);
         tokenize(filter, null, indexFacts, queryLanguage);
 
         Item filterRoot = anyItems(true);
@@ -53,7 +54,7 @@ public class AnyParser extends SimpleParser {
             if ( ! tokens.skipMultiple(MINUS)) return null;
             if (tokens.currentIsNoIgnore(SPACE)) return null;
 
-            item = indexableItem();
+            item = indexableItem().getFirst();
 
             if (item == null) {
                 item = compositeItem();
@@ -121,8 +122,8 @@ public class AnyParser extends SimpleParser {
         }
     }
 
-    Item applyFilter(Item root, String filter, Language queryLanguage, IndexFacts.Session indexFacts) {
-        setState(queryLanguage, indexFacts);
+    Item applyFilter(Item root, String filter, Language queryLanguage, IndexFacts.Session indexFacts, String defaultIndex) {
+        setState(queryLanguage, indexFacts, defaultIndex);
         tokenize(filter, null, indexFacts, queryLanguage);
         return filterItems(root);
     }
@@ -153,7 +154,7 @@ public class AnyParser extends SimpleParser {
                 root = addNotFilter(root, item);
             }
             if (item == null) {
-                item = indexableItem();
+                item = indexableItem().getFirst();
                 root = addRankFilter(root, item);
             }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java
index 72eb56dd0fb..01b5b943829 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java
@@ -16,7 +16,8 @@ public class PhraseParser extends AbstractParser {
         super(environment);
     }
 
-    protected Item parseItems(String defaultIndex) {
+    @Override
+    protected Item parseItems() {
         return forcedPhrase();
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java
index 0f94e4ce21d..b7355c43f81 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java
@@ -42,12 +42,12 @@ abstract class SimpleParser extends StructuredParser {
      * If there's a explicit composite and some other terms,
      * a rank terms combines them
      */
-    protected Item anyItems(boolean topLevel, String defaultIndexName) {
+    protected Item anyItems(boolean topLevel) {
         int position = tokens.getPosition();
         Item item = null;
 
         try {
-            item = anyItemsBody(topLevel, defaultIndexName);
+            item = anyItemsBody(topLevel);
             return item;
         } finally {
             if (item == null) {
@@ -56,11 +56,7 @@ abstract class SimpleParser extends StructuredParser {
         }
     }
 
-    protected Item anyItems(boolean topLevel) {
-        return anyItems(topLevel, null);
-    }
-
-    private Item anyItemsBody(boolean topLevel, String defaultIndexName) {
+    private Item anyItemsBody(boolean topLevel) {
         Item topLevelItem = null;
         NotItem not = null;
         Item item;
@@ -101,7 +97,7 @@ abstract class SimpleParser extends StructuredParser {
             }
 
             if (item == null) {
-                item = indexableItem(defaultIndexName);
+                item = indexableItem().getFirst();
                 if (item != null) {
                     if (topLevelItem == null) {
                         topLevelItem = item;
@@ -186,7 +182,7 @@ abstract class SimpleParser extends StructuredParser {
                 return null;
             }
 
-            item = indexableItem();
+            item = indexableItem().getFirst();
 
             if (item == null) {
                 item = compositeItem();
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
index 0d6a06961d1..88490237fc7 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.query.parser;
 
+import com.yahoo.collections.Pair;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.query.AndItem;
 import com.yahoo.prelude.query.AndSegmentItem;
@@ -69,19 +70,22 @@ abstract class StructuredParser extends AbstractParser {
         submodes.setFromIndex(indexName, indexFacts);
     }
 
-    protected Item indexableItem() {
-        return indexableItem(null);
-    }
-
-    protected Item indexableItem(String defaultIndexName) {
+    /**
+     * Returns an item and whether it had an explicit index ('indexname:' prefix).
+     *
+     * @return an item and whether it has an explicit index, or a Pair with the first element null if none
+     */
+    protected Pair<Item, Boolean> indexableItem() {
         int position = tokens.getPosition();
         Item item = null;
 
         try {
+            boolean explicitIndex = false;
             String indexName = indexPrefix();
-            if (Objects.isNull(indexName)) {
-                indexName = defaultIndexName;
-            }
+            if (indexName != null)
+                explicitIndex = true;
+            else
+                indexName = this.defaultIndex;
             setSubmodeFromIndex(indexName, indexFacts);
 
             item = number();
@@ -103,7 +107,6 @@ abstract class StructuredParser extends AbstractParser {
             if (item != null) {
                 weight = weightSuffix();
             }
-
             if (indexName != null && item != null) {
                 item.setIndexName(indexName);
             }
@@ -112,7 +115,7 @@ abstract class StructuredParser extends AbstractParser {
                 item.setWeight(weight);
             }
 
-            return item;
+            return new Pair<>(item, explicitIndex);
         } finally {
             if (item == null) {
                 tokens.setPosition(position);
@@ -126,8 +129,7 @@ abstract class StructuredParser extends AbstractParser {
             if (tokens.currentIsNoIgnore(SPACE)) {
                 return false;
             }
-            if (tokens.currentIsNoIgnore(NUMBER)
-                    || tokens.currentIsNoIgnore(WORD)) {
+            if (tokens.currentIsNoIgnore(NUMBER) || tokens.currentIsNoIgnore(WORD)) {
                 return true;
             }
             tokens.skipNoIgnore();
@@ -303,7 +305,6 @@ abstract class StructuredParser extends AbstractParser {
             tokens.skip(LSQUAREBRACKET);
             if (item == null)
                 tokens.skipNoIgnore(SPACE);
-
             // TODO: Better definition of start and end of numeric items
             if (item == null && tokens.currentIsNoIgnore(MINUS) && (tokens.currentNoIgnore(1).kind == NUMBER)) {
                 tokens.skipNoIgnore();
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java
index eefbe5fa0d0..dbbc321d057 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java
@@ -22,7 +22,7 @@ public final class TokenizeParser extends AbstractParser {
     }
 
     @Override
-    protected Item parseItems(String defaultIndex) {
+    protected Item parseItems() {
         WeakAndItem weakAnd = new WeakAndItem();
         Token token;
         while (null != (token = tokens.next())) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java
index 40497d94a6d..aff28179050 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java
@@ -28,7 +28,7 @@ public class WebParser extends AllParser {
     }
 
     @Override
-    protected Item parseItemsBody(String defaultIndexName) {
+    protected Item parseItemsBody() {
         // Algorithm: Collect positive, negative, and'ed and or'ed elements, then combine.
         CompositeItem and = null;
         OrItem or = null;
@@ -45,7 +45,7 @@ public class WebParser extends AllParser {
 
             current = positiveItem();
             if (current == null)
-                current = indexableItem(defaultIndexName);
+                current = indexableItem().getFirst();
 
             if (current != null) {
                 if (and != null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
index 95ecf3c2dba..06b6eca5f84 100644
--- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java
@@ -195,7 +195,7 @@ public class QueryTestCase {
         assertTrue(p.hashCode() != q.hashCode());
     }
 
-    /** Test using the defauultindex feature */
+    /** Test using the defaultindex feature */
     @Test
     void testDefaultIndex() {
         Query q = newQuery("?query=hi hello keyword:kanoo " +