Revert "Reapply "Bratseth/special tokens""

7 files changed, 326 insertions, 27 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
index 732466748eb..902be7e15dd 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java
@@ -19,6 +19,7 @@ import java.util.*;
  * @author bratseth
  * @author Steinar Knutsen
  */
+@SuppressWarnings("deprecation")
 public abstract class AbstractParser implements CustomParser {
 
     /** The current submodes of this parser */
@@ -47,7 +48,7 @@ public abstract class AbstractParser implements CustomParser {
      * of these may be active at the same time. SubModes are activated or
      * deactivated by specifying special indexes in the query.
      */
-    static final class Submodes {
+    final class Submodes {
 
         /**
          * Url mode allows "_" and "-" as word characters. Default is false
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
new file mode 100644
index 00000000000..be2d9f9f68b
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
@@ -0,0 +1,137 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.query.parser;
+
+import com.yahoo.config.subscription.ConfigGetter;
+import com.yahoo.config.subscription.ConfigSubscriber;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+
+/**
+ * A <i>registry</i> which is responsible for knowing the current
+ * set of special tokens. The default registry returns empty token lists
+ * for all names. Usage of this registry is multithread safe.
+ *
+ * @author bratseth
+ */
+public class SpecialTokenRegistry {
+
+    /** The log of this */
+    private static final Logger log = Logger.getLogger(SpecialTokenRegistry.class.getName());
+
+    private static final SpecialTokens nullSpecialTokens = new SpecialTokens();
+
+    /**
+     * The current authorative special token lists, indexed on name.
+     * These lists are unmodifiable and used directly by clients of this
+     */
+    private Map<String,SpecialTokens> specialTokenMap = new HashMap<>();
+
+    private boolean frozen = false;
+
+    /**
+     * Creates an empty special token registry which
+     * does not subscribe to any configuration
+     */
+    public SpecialTokenRegistry() {}
+
+    /**
+     * Create a special token registry which subscribes to the specialtokens
+     * configuration. Only used for testing.
+     */
+    public SpecialTokenRegistry(String configId) {
+        try {
+            build(new ConfigGetter<>(SpecialtokensConfig.class).getConfig(configId));
+        } catch (Exception e) {
+            log.config(
+                    "No special tokens are configured (" + e.getMessage() + ")");
+        }
+    }
+
+    /**
+     * Create a special token registry from a configuration object. This is the production code path.
+     */
+    public SpecialTokenRegistry(SpecialtokensConfig config) {
+        if (config != null) {
+            build(config);
+        }
+        freeze();
+    }
+
+    private void freeze() {
+        frozen = true;
+    }
+
+    private void build(SpecialtokensConfig config) {
+        List<SpecialTokens> list = new ArrayList<>();
+        for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
+            Tokenlist tokenList = i.next();
+            SpecialTokens tokens = new SpecialTokens(tokenList.name());
+
+            for (Iterator<Tokens> j = tokenList.tokens().iterator(); j.hasNext();) {
+                Tokens token = j.next();
+                tokens.addSpecialToken(token.token(), token.replace());
+            }
+            tokens.freeze();
+            list.add(tokens);
+        }
+        addSpecialTokens(list);
+    }
+
+    /**
+     * Adds a SpecialTokens instance to the registry. That is, add the
+     * tokens contained for the name of the SpecialTokens instance
+     * given.
+     *
+     * @param specialTokens the SpecialTokens object to add
+     */
+    public void addSpecialTokens(SpecialTokens specialTokens) {
+        ensureNotFrozen();
+        List<SpecialTokens> list = new ArrayList<>();
+        list.add(specialTokens);
+        addSpecialTokens(list);
+
+    }
+
+    private void ensureNotFrozen() {
+        if (frozen) {
+            throw new IllegalStateException("Tried to modify a frozen SpecialTokenRegistry instance.");
+        }
+    }
+
+    private void addSpecialTokens(List<SpecialTokens> list) {
+        HashMap<String,SpecialTokens> tokens = new HashMap<>(specialTokenMap);
+        for(SpecialTokens t: list) {
+            tokens.put(t.getName(),t);
+        }
+        specialTokenMap = tokens;
+    }
+
+
+    /**
+     * Returns the currently authorative list of special tokens for
+     * a given name.
+     *
+     * @param  name the name of the special tokens to return
+     *         null, the empth string or the string "default" returns
+     *         the default ones
+     * @return a read-only list of SpecialToken instances, an empty list if this name
+     *         has no special tokens
+     */
+    public SpecialTokens getSpecialTokens(String name) {
+        if (name == null || name.trim().equals("")) {
+            name = "default";
+        }
+        SpecialTokens specialTokens = specialTokenMap.get(name);
+
+        if (specialTokens == null) {
+            return nullSpecialTokens;
+        }
+        return specialTokens;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
new file mode 100644
index 00000000000..f45ecefefa6
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
@@ -0,0 +1,167 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.query.parser;
+
+import java.util.logging.Level;
+import com.yahoo.prelude.query.Substring;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * A list of special tokens - string that should be treated as word
+ * no matter what they contain. Special tokens are case insensitive.
+ *
+ * @author bratseth
+ */
+public class SpecialTokens {
+
+    private static final Logger log = Logger.getLogger(SpecialTokens.class.getName());
+
+    private final String name;
+
+    private final List<SpecialToken> specialTokens = new ArrayList<>();
+
+    private boolean frozen = false;
+
+    private int currentMaximumLength = 0;
+
+    /** Creates a null list of special tokens */
+    public SpecialTokens() {
+        this.name = "(null)";
+    }
+
+    public SpecialTokens(String name) {
+        this.name = name;
+    }
+
+    /** Returns the name of this special tokens list */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Adds a special token to this
+     *
+     * @param token the special token string to add
+     * @param replace the token to replace instances of the special token with, or null to keep the token
+     */
+    public void addSpecialToken(String token, String replace) {
+        ensureNotFrozen();
+        if (!caseIndependentLength(token)) {
+            return;
+        }
+        // TODO are special tokens correctly unicode normalized in reagards to query parsing?
+        final SpecialToken specialTokenToAdd = new SpecialToken(token, replace);
+        currentMaximumLength = Math.max(currentMaximumLength, specialTokenToAdd.token.length());
+        specialTokens.add(specialTokenToAdd);
+        Collections.sort(specialTokens);
+    }
+
+    private boolean caseIndependentLength(String token) {
+        // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
+        String asLow = toLowerCase(token);
+        // TODO put along with the global toLowerCase
+        String asHigh = token.toUpperCase(Locale.ENGLISH);
+        if (asLow.length() != token.length() || asHigh.length() != token.length()) {
+            log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token."
+                    + " Please report this message in a bug to the Vespa team.");
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    /**
+     * Returns the special token starting at the start of the given string, or null if no
+     * special token starts at this string
+     *
+     * @param string the string to search for a special token at the start position
+     * @param substring true to allow the special token to be followed by a character which does not
+     *        mark the end of a token
+     */
+    public SpecialToken tokenize(String string, boolean substring) {
+        // XXX detonator pattern token.length may be != the length of the
+        // matching data in string, ref caseIndependentLength(String)
+        final String input = toLowerCase(string.substring(0, Math.min(string.length(), currentMaximumLength)));
+        for (Iterator<SpecialToken> i = specialTokens.iterator(); i.hasNext();) {
+            SpecialTokens.SpecialToken special = i.next();
+
+            if (input.startsWith(special.token())) {
+                if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string))
+                    return special;
+            }
+        }
+        return null;
+    }
+
+    private boolean tokenEndsAt(int position,String string) {
+        return !Character.isLetterOrDigit(string.charAt(position));
+    }
+
+    /** Returns the number of special tokens in this */
+    public int size() {
+        return specialTokens.size();
+    }
+
+    private void ensureNotFrozen() {
+        if (frozen) {
+            throw new IllegalStateException("Tried to modify a frozen SpecialTokens instance.");
+        }
+    }
+
+    public void freeze() {
+        frozen = true;
+    }
+
+    /** An immutable special token */
+    public final static class SpecialToken implements Comparable<SpecialToken> {
+
+        private String token;
+
+        private String replace;
+
+        public SpecialToken(String token, String replace) {
+            this.token = toLowerCase(token);
+            if (replace == null || replace.trim().equals("")) {
+                this.replace = this.token;
+            } else {
+                this.replace = toLowerCase(replace);
+            }
+        }
+
+        /** Returns the special token */
+        public String token() {
+            return token;
+        }
+
+        /** Returns the right replace value, never null or an empty string */
+        public String replace() {
+            return replace;
+        }
+
+        @Override
+        public int compareTo(SpecialToken other) {
+            if (this.token().length() < other.token().length()) return 1;
+            if (this.token().length() == other.token().length()) return 0;
+            return -1;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (other == this) return true;
+            if ( ! (other instanceof SpecialToken)) return false;
+            return Objects.equals(this.token, ((SpecialToken)other).token);
+        }
+
+        @Override
+        public int hashCode() { return token.hashCode(); }
+
+        public Token toToken(int start, String rawSource) {
+            return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe?
+        }
+
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
index b71bd57539f..2dc2254df68 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
@@ -3,7 +3,6 @@ package com.yahoo.prelude.query.parser;
 
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.CharacterClasses;
-import com.yahoo.language.process.SpecialTokens;
 import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.query.Substring;
@@ -201,7 +200,7 @@ public final class Tokenizer {
         }
         StringBuilder tmp = new StringBuilder();
         for (int i = 0; i < tokencnt; i++) {
-            Token useToken = tokens.get(backtrack + i);
+            Token useToken = tokens.get(backtrack+i);
             tmp.append(useToken.image);
         }
         String indexName = tmp.toString();
@@ -217,20 +216,20 @@ public final class Tokenizer {
     }
 
     private int consumeSpecialToken(int start) {
-        SpecialTokens.Token token = getSpecialToken(start);
-        if (token == null) return start;
-        tokens.add(toToken(token, start, source));
-        return start + token.token().length();
+        SpecialTokens.SpecialToken specialToken=getSpecialToken(start);
+        if (specialToken==null) return start;
+        tokens.add(specialToken.toToken(start,source));
+        return start + specialToken.token().length();
     }
 
-    private SpecialTokens.Token getSpecialToken(int start) {
+    private SpecialTokens.SpecialToken getSpecialToken(int start) {
         if (specialTokens == null) return null;
         return specialTokens.tokenize(source.substring(start), substringSpecialTokens);
     }
 
     private int consumeExact(int start,Index index) {
         if (index.getExactTerminator() == null) return consumeHeuristicExact(start);
-        return consumeToTerminator(start, index.getExactTerminator());
+        return consumeToTerminator(start,index.getExactTerminator());
     }
 
     private boolean looksLikeExactEnd(int end) {
@@ -468,7 +467,7 @@ public final class Tokenizer {
     /** Consumes a word or number <i>and/or possibly</i> a special token starting within this word or number */
     private int consumeWordOrNumber(int start, Index currentIndex) {
         int tokenEnd = start;
-        SpecialTokens.Token substringToken = null;
+        SpecialTokens.SpecialToken substringSpecialToken = null;
         boolean digitsOnly = true;
         // int underscores = 0;
         // boolean underscoresOnly = true;
@@ -476,8 +475,8 @@ public final class Tokenizer {
 
         while (tokenEnd < source.length()) {
             if (substringSpecialTokens) {
-                substringToken = getSpecialToken(tokenEnd);
-                if (substringToken != null) break;
+                substringSpecialToken = getSpecialToken(tokenEnd);
+                if (substringSpecialToken != null) break;
             }
 
             int c = source.codePointAt(tokenEnd);
@@ -525,11 +524,11 @@ public final class Tokenizer {
             }
         }
 
-        if (substringToken == null)
+        if (substringSpecialToken == null)
             return --tokenEnd;
         // TODO: test the logic around tokenEnd with friends
-        addToken(toToken(substringToken, tokenEnd, source));
-        return --tokenEnd + substringToken.token().length();
+        addToken(substringSpecialToken.toToken(tokenEnd, source));
+        return --tokenEnd + substringSpecialToken.token().length();
     }
 
     private void addToken(Token.Kind kind, String word, int start, int end) {
@@ -540,11 +539,4 @@ public final class Tokenizer {
         tokens.add(token);
     }
 
-    public Token toToken(SpecialTokens.Token specialToken, int start, String rawSource) {
-        return new Token(Token.Kind.WORD,
-                         specialToken.replacement(),
-                         true,
-                         new Substring(start, start + specialToken.token().length(), rawSource)); // XXX: Unsafe?
-    }
-
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
index df96d314455..94b9bf6ce65 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
@@ -4,7 +4,7 @@ package com.yahoo.search.query.parser;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.IndexFacts;
-import com.yahoo.language.process.SpecialTokens;
+import com.yahoo.prelude.query.parser.SpecialTokens;
 import com.yahoo.search.Searcher;
 import com.yahoo.search.searchchain.Execution;
 
@@ -18,7 +18,7 @@ public final class ParserEnvironment {
 
     private IndexFacts indexFacts = new IndexFacts();
     private Linguistics linguistics = new SimpleLinguistics();
-    private SpecialTokens specialTokens = SpecialTokens.empty();
+    private SpecialTokens specialTokens = new SpecialTokens();
 
     public IndexFacts getIndexFacts() {
         return indexFacts;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
index 0574fc660c3..84fe88d0292 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
@@ -6,7 +6,7 @@ import com.yahoo.language.Linguistics;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.Ping;
 import com.yahoo.prelude.Pong;
-import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
 import com.yahoo.processing.Processor;
 import com.yahoo.processing.Request;
 import com.yahoo.processing.Response;
@@ -17,6 +17,8 @@ import com.yahoo.search.cluster.PingableSearcher;
 import com.yahoo.search.rendering.RendererRegistry;
 import com.yahoo.search.statistics.TimeTracker;
 
+import java.util.logging.Logger;
+
 /**
  * <p>An execution of a search chain. This keeps track of the call state for an execution (in the calling thread)
  * of the searchers of a search chain.</p>
@@ -109,7 +111,7 @@ public class Execution extends com.yahoo.processing.execution.Execution {
         public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts,
                        SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics)
         {
-            owner = null;
+            owner=null;
             // The next time something is added here, compose into wrapper objects. Many arguments...
 
             // Four methods need to be updated when adding something:
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
index a813229c984..31b6d06f78e 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java
@@ -13,7 +13,7 @@ import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
-import com.yahoo.language.process.SpecialTokenRegistry;
+import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
 import com.yahoo.processing.rendering.Renderer;
 import com.yahoo.search.Searcher;
 import com.yahoo.search.config.IndexInfoConfig;