Make immutable

5 files changed, 101 insertions, 177 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
index f1798948361..9c735a031d7 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokenRegistry.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.query.parser;
 
-import com.yahoo.config.subscription.ConfigGetter;
 import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
 import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
 import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
@@ -11,55 +10,36 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
- * A <i>registry</i> which is responsible for knowing the current
- * set of special tokens. The default registry returns empty token lists
- * for all names. Usage of this registry is multithread safe.
+ * A registry which is responsible for knowing the current
+ * set of special tokens.Usage of this registry is multithread safe.
  *
  * @author bratseth
  */
 public class SpecialTokenRegistry {
 
-    /** The log of this */
-    private static final Logger log = Logger.getLogger(SpecialTokenRegistry.class.getName());
-
-    private static final SpecialTokens nullSpecialTokens = new SpecialTokens();
-
     /**
      * The current special token lists, indexed on name.
      * These lists are unmodifiable and used directly by clients of this
      */
-    private Map<String, SpecialTokens> specialTokenMap = new HashMap<>();
+    private Map<String, SpecialTokens> specialTokenMap;
 
     private boolean frozen = false;
 
-    /**
-     * Creates an empty special token registry which
-     * does not subscribe to any configuration
-     */
-    public SpecialTokenRegistry() {}
-
-    /**
-     * Create a special token registry which subscribes to the specialtokens
-     * configuration. Only used for testing.
-     */
-    public SpecialTokenRegistry(String configId) {
-        try {
-            build(new ConfigGetter<>(SpecialtokensConfig.class).getConfig(configId));
-        } catch (Exception e) {
-            log.config("No special tokens are configured (" + e.getMessage() + ")");
-        }
+    /** Creates an empty special token registry */
+    public SpecialTokenRegistry() {
+        this(List.of());
     }
 
-    /**
-     * Create a special token registry from a configuration object. This is the production code path.
-     */
+    /** Create a special token registry from a configuration object. */
     public SpecialTokenRegistry(SpecialtokensConfig config) {
-        if (config != null) {
-            build(config);
-        }
+        this(specialTokensFrom(config));
+    }
+
+    public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) {
+        specialTokenMap = specialTokensList.stream().collect(Collectors.toMap(t -> t.name(), t -> t));
         freeze();
     }
 
@@ -67,35 +47,19 @@ public class SpecialTokenRegistry {
         frozen = true;
     }
 
-    private void build(SpecialtokensConfig config) {
-        List<SpecialTokens> list = new ArrayList<>();
+    private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) {
+        List<SpecialTokens> specialTokensList = new ArrayList<>();
         for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
-            Tokenlist tokenList = i.next();
-            SpecialTokens tokens = new SpecialTokens(tokenList.name());
+            Tokenlist tokenListConfig = i.next();
 
-            for (Iterator<Tokens> j = tokenList.tokens().iterator(); j.hasNext();) {
-                Tokens token = j.next();
-                tokens.addSpecialToken(token.token(), token.replace());
+            List<SpecialTokens.Token> tokenList = new ArrayList<>();
+            for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) {
+                Tokens tokenConfig = j.next();
+                tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace()));
             }
-            tokens.freeze();
-            list.add(tokens);
+            specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList));
         }
-        addSpecialTokens(list);
-    }
-
-    /**
-     * Adds a SpecialTokens instance to the registry. That is, add the
-     * tokens contained for the name of the SpecialTokens instance
-     * given.
-     *
-     * @param specialTokens the SpecialTokens object to add
-     */
-    public void addSpecialTokens(SpecialTokens specialTokens) {
-        ensureNotFrozen();
-        List<SpecialTokens> list = new ArrayList<>();
-        list.add(specialTokens);
-        addSpecialTokens(list);
-
+        return specialTokensList;
     }
 
     private void ensureNotFrozen() {
@@ -104,14 +68,6 @@ public class SpecialTokenRegistry {
         }
     }
 
-    private void addSpecialTokens(List<SpecialTokens> list) {
-        HashMap<String,SpecialTokens> tokens = new HashMap<>(specialTokenMap);
-        for(SpecialTokens t: list) {
-            tokens.put(t.getName(),t);
-        }
-        specialTokenMap = tokens;
-    }
-
     /**
      * Returns the list of special tokens for a given name.
      *
@@ -122,15 +78,9 @@ public class SpecialTokenRegistry {
      *         has no special tokens
      */
     public SpecialTokens getSpecialTokens(String name) {
-        if (name == null || name.trim().equals("")) {
+        if (name == null || name.trim().equals(""))
             name = "default";
-        }
-        SpecialTokens specialTokens = specialTokenMap.get(name);
-
-        if (specialTokens == null) {
-            return nullSpecialTokens;
-        }
-        return specialTokens;
+        return specialTokenMap.getOrDefault(name, SpecialTokens.empty());
     }
 
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
index 6f1fad1a6c1..4b29b50f095 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
@@ -1,11 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.query.parser;
 
-import java.util.logging.Level;
-import com.yahoo.prelude.query.Substring;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
 
-import java.util.*;
-import java.util.logging.Logger;
+import com.yahoo.prelude.query.Substring;
 
 import static com.yahoo.language.LinguisticsCase.toLowerCase;
 
@@ -17,61 +20,28 @@ import static com.yahoo.language.LinguisticsCase.toLowerCase;
  */
 public class SpecialTokens {
 
-    private static final Logger log = Logger.getLogger(SpecialTokens.class.getName());
+    private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of());
 
     private final String name;
-
-    private final List<SpecialToken> specialTokens = new ArrayList<>();
-
-    private boolean frozen = false;
-
-    private int currentMaximumLength = 0;
-
-    /** Creates a null list of special tokens */
-    public SpecialTokens() {
-        this.name = "(null)";
-    }
-
-    public SpecialTokens(String name) {
+    private final List<Token> tokens;
+    private final int maximumLength;
+
+    public SpecialTokens(String name,  List<Token> tokens) {
+        tokens.stream().peek(token -> token.validate());
+        List<Token> mutableTokens = new ArrayList<>(tokens);
+        Collections.sort(mutableTokens);
+        this.tokens = List.copyOf(mutableTokens);
         this.name = name;
+        this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0);
     }
 
     /** Returns the name of this special tokens list */
-    public String getName() {
+    public String name() {
         return name;
     }
 
-    /**
-     * Adds a special token to this
-     *
-     * @param token the special token string to add
-     * @param replace the token to replace instances of the special token with, or null to keep the token
-     */
-    public void addSpecialToken(String token, String replace) {
-        ensureNotFrozen();
-        if (!caseIndependentLength(token)) {
-            return;
-        }
-        // TODO: Are special tokens correctly unicode normalized in regards to query parsing?
-        final SpecialToken specialTokenToAdd = new SpecialToken(token, replace);
-        currentMaximumLength = Math.max(currentMaximumLength, specialTokenToAdd.token.length());
-        specialTokens.add(specialTokenToAdd);
-        Collections.sort(specialTokens);
-    }
-
-    private boolean caseIndependentLength(String token) {
-        // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
-        String asLow = toLowerCase(token);
-        // TODO: Put along with the global toLowerCase
-        String asHigh = token.toUpperCase(Locale.ENGLISH);
-        if (asLow.length() != token.length() || asHigh.length() != token.length()) {
-            log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token." +
-                                  " Please report this message in a bug to the Vespa team.");
-            return false;
-        } else {
-            return true;
-        }
-    }
+    /** Returns a sorted immutable list of the special tokens in this */
+    public List<Token> tokens() { return tokens; }
 
     /**
      * Returns the special token starting at the start of the given string, or null if no
@@ -81,12 +51,12 @@ public class SpecialTokens {
      * @param substring true to allow the special token to be followed by a character which does not
      *        mark the end of a token
      */
-    public SpecialToken tokenize(String string, boolean substring) {
+    public Token tokenize(String string, boolean substring) {
         // XXX detonator pattern token.length may be != the length of the
         // matching data in string, ref caseIndependentLength(String)
-        String input = toLowerCase(string.substring(0, Math.min(string.length(), currentMaximumLength)));
-        for (Iterator<SpecialToken> i = specialTokens.iterator(); i.hasNext();) {
-            SpecialTokens.SpecialToken special = i.next();
+        String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength)));
+        for (Iterator<Token> i = tokens.iterator(); i.hasNext();) {
+            Token special = i.next();
 
             if (input.startsWith(special.token())) {
                 if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string))
@@ -100,49 +70,36 @@ public class SpecialTokens {
         return !Character.isLetterOrDigit(string.charAt(position));
     }
 
-    /** Returns the number of special tokens in this */
-    public int size() {
-        return specialTokens.size();
-    }
-
-    private void ensureNotFrozen() {
-        if (frozen) {
-            throw new IllegalStateException("Tried to modify a frozen SpecialTokens instance.");
-        }
-    }
-
-    public void freeze() {
-        frozen = true;
-    }
+    public static SpecialTokens empty() { return empty; }
 
     /** An immutable special token */
-    public final static class SpecialToken implements Comparable<SpecialToken> {
+    public final static class Token implements Comparable<Token> {
 
         private final String token;
+        private final String replacement;
 
-        private final String replace;
+        /** Creates a special token */
+        public Token(String token) {
+            this(token, null);
+        }
 
-        public SpecialToken(String token, String replace) {
+        /** Creates a special token which will be represented by the given replacement token */
+        public Token(String token, String replacement) {
             this.token = toLowerCase(token);
-            if (replace == null || replace.trim().equals("")) {
-                this.replace = this.token;
-            } else {
-                this.replace = toLowerCase(replace);
-            }
+            if (replacement == null || replacement.trim().equals(""))
+                this.replacement = this.token;
+            else
+                this.replacement = toLowerCase(replacement);
         }
 
         /** Returns the special token */
-        public String token() {
-            return token;
-        }
+        public String token() { return token; }
 
-        /** Returns the right replace value, never null or an empty string */
-        public String replace() {
-            return replace;
-        }
+        /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */
+        public String replacement() { return replacement; }
 
         @Override
-        public int compareTo(SpecialToken other) {
+        public int compareTo(Token other) {
             if (this.token().length() < other.token().length()) return 1;
             if (this.token().length() == other.token().length()) return 0;
             return -1;
@@ -151,15 +108,27 @@ public class SpecialTokens {
         @Override
         public boolean equals(Object other) {
             if (other == this) return true;
-            if ( ! (other instanceof SpecialToken)) return false;
-            return Objects.equals(this.token, ((SpecialToken)other).token);
+            if ( ! (other instanceof Token)) return false;
+            return Objects.equals(this.token, ((Token)other).token);
         }
 
         @Override
         public int hashCode() { return token.hashCode(); }
 
-        public Token toToken(int start, String rawSource) {
-            return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe?
+        @Override
+        public String toString() {
+            return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'");
+        }
+
+        private void validate() {
+            // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
+            String asLow = toLowerCase(token);
+            // TODO: Put along with the global toLowerCase
+            String asHigh = token.toUpperCase(Locale.ENGLISH);
+            if (asLow.length() != token.length() || asHigh.length() != token.length()) {
+                throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " +
+                                                   "Please report this to the Vespa team.");
+            }
         }
 
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
index 686ceefc460..f0656efa59a 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
@@ -200,7 +200,7 @@ public final class Tokenizer {
         }
         StringBuilder tmp = new StringBuilder();
         for (int i = 0; i < tokencnt; i++) {
-            Token useToken = tokens.get(backtrack+i);
+            Token useToken = tokens.get(backtrack + i);
             tmp.append(useToken.image);
         }
         String indexName = tmp.toString();
@@ -216,13 +216,13 @@ public final class Tokenizer {
     }
 
     private int consumeSpecialToken(int start) {
-        SpecialTokens.SpecialToken specialToken = getSpecialToken(start);
-        if (specialToken == null) return start;
-        tokens.add(specialToken.toToken(start, source));
-        return start + specialToken.token().length();
+        SpecialTokens.Token token = getSpecialToken(start);
+        if (token == null) return start;
+        tokens.add(toToken(token, start, source));
+        return start + token.token().length();
     }
 
-    private SpecialTokens.SpecialToken getSpecialToken(int start) {
+    private SpecialTokens.Token getSpecialToken(int start) {
         if (specialTokens == null) return null;
         return specialTokens.tokenize(source.substring(start), substringSpecialTokens);
     }
@@ -467,7 +467,7 @@ public final class Tokenizer {
     /** Consumes a word or number <i>and/or possibly</i> a special token starting within this word or number */
     private int consumeWordOrNumber(int start, Index currentIndex) {
         int tokenEnd = start;
-        SpecialTokens.SpecialToken substringSpecialToken = null;
+        SpecialTokens.Token substringToken = null;
         boolean digitsOnly = true;
         // int underscores = 0;
         // boolean underscoresOnly = true;
@@ -475,8 +475,8 @@ public final class Tokenizer {
 
         while (tokenEnd < source.length()) {
             if (substringSpecialTokens) {
-                substringSpecialToken = getSpecialToken(tokenEnd);
-                if (substringSpecialToken != null) break;
+                substringToken = getSpecialToken(tokenEnd);
+                if (substringToken != null) break;
             }
 
             int c = source.codePointAt(tokenEnd);
@@ -524,11 +524,11 @@ public final class Tokenizer {
             }
         }
 
-        if (substringSpecialToken == null)
+        if (substringToken == null)
             return --tokenEnd;
         // TODO: test the logic around tokenEnd with friends
-        addToken(substringSpecialToken.toToken(tokenEnd, source));
-        return --tokenEnd + substringSpecialToken.token().length();
+        addToken(toToken(substringToken, tokenEnd, source));
+        return --tokenEnd + substringToken.token().length();
     }
 
     private void addToken(Token.Kind kind, String word, int start, int end) {
@@ -539,4 +539,11 @@ public final class Tokenizer {
         tokens.add(token);
     }
 
+    public Token toToken(SpecialTokens.Token specialToken, int start, String rawSource) {
+        return new Token(Token.Kind.WORD,
+                         specialToken.replacement(),
+                         true,
+                         new Substring(start, start + specialToken.token().length(), rawSource)); // XXX: Unsafe?
+    }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
index 94b9bf6ce65..1b35296082e 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java
@@ -18,7 +18,7 @@ public final class ParserEnvironment {
 
     private IndexFacts indexFacts = new IndexFacts();
     private Linguistics linguistics = new SimpleLinguistics();
-    private SpecialTokens specialTokens = new SpecialTokens();
+    private SpecialTokens specialTokens = SpecialTokens.empty();
 
     public IndexFacts getIndexFacts() {
         return indexFacts;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
index 84fe88d0292..7adfccc2ed1 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
@@ -17,8 +17,6 @@ import com.yahoo.search.cluster.PingableSearcher;
 import com.yahoo.search.rendering.RendererRegistry;
 import com.yahoo.search.statistics.TimeTracker;
 
-import java.util.logging.Logger;
-
 /**
  * <p>An execution of a search chain. This keeps track of the call state for an execution (in the calling thread)
  * of the searchers of a search chain.</p>