Revert "Revert "Revert "Bratseth/special tokens"""

8 files changed, 13 insertions, 336 deletions

diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json
index b77b03664d4..58b838d7332 100644
--- a/linguistics/abi-spec.json
+++ b/linguistics/abi-spec.json
@@ -427,57 +427,6 @@
     ],
     "fields": []
   },
-  "com.yahoo.language.process.SpecialTokenRegistry": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>()",
-      "public void <init>(com.yahoo.vespa.configdefinition.SpecialtokensConfig)",
-      "public void <init>(java.util.List)",
-      "public com.yahoo.language.process.SpecialTokens getSpecialTokens(java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.language.process.SpecialTokens$Token": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public",
-      "final"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String)",
-      "public void <init>(java.lang.String, java.lang.String)",
-      "public java.lang.String token()",
-      "public java.lang.String replacement()",
-      "public int compareTo(com.yahoo.language.process.SpecialTokens$Token)",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
-      "public java.lang.String toString()",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.language.process.SpecialTokens": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.util.List)",
-      "public java.lang.String name()",
-      "public java.util.Map asMap()",
-      "public com.yahoo.language.process.SpecialTokens$Token tokenize(java.lang.String, boolean)",
-      "public static com.yahoo.language.process.SpecialTokens empty()"
-    ],
-    "fields": []
-  },
   "com.yahoo.language.process.StemList": {
     "superClass": "java.util.AbstractList",
     "interfaces": [],
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
index 73518876c3f..e1185cb2457 100644
--- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
@@ -4,7 +4,6 @@ package com.yahoo.language.opennlp;
 import com.yahoo.language.Language;
 import com.yahoo.language.LinguisticsCase;
 import com.yahoo.language.process.Normalizer;
-import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.language.process.StemMode;
 import com.yahoo.language.process.Token;
 import com.yahoo.language.process.TokenType;
@@ -33,21 +32,15 @@ public class OpenNlpTokenizer implements Tokenizer {
     private final Normalizer normalizer;
     private final Transformer transformer;
     private final SimpleTokenizer simpleTokenizer;
-    private final SpecialTokenRegistry specialTokenRegistry;
 
     public OpenNlpTokenizer() {
         this(new SimpleNormalizer(), new SimpleTransformer());
     }
 
     public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) {
-        this(normalizer, transformer, new SpecialTokenRegistry(List.of()));
-    }
-
-    public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) {
         this.normalizer = normalizer;
         this.transformer = transformer;
-        this.specialTokenRegistry = specialTokenRegistry;
-        this.simpleTokenizer = new SimpleTokenizer(normalizer, transformer, specialTokenRegistry);
+        simpleTokenizer = new SimpleTokenizer(normalizer, transformer);
     }
 
     @Override
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java
deleted file mode 100644
index b6335d67967..00000000000
--- a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.language.process;
-
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * Immutable named lists of "special tokens" - strings which should override the normal tokenizer semantics
- * and be tokenized into a single token.
- *
- * @author bratseth
- */
-public class SpecialTokenRegistry {
-
-    /**
-     * The current special token lists, indexed on name.
-     * These lists are unmodifiable and used directly by clients of this
-     */
-    private final Map<String, SpecialTokens> specialTokenMap;
-
-    /** Creates an empty special token registry */
-    public SpecialTokenRegistry() {
-        this(List.of());
-    }
-
-    /** Create a special token registry from a configuration object. */
-    public SpecialTokenRegistry(SpecialtokensConfig config) {
-        this(specialTokensFrom(config));
-    }
-
-    public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) {
-        specialTokenMap = specialTokensList.stream().collect(Collectors.toUnmodifiableMap(t -> t.name(), t -> t));
-    }
-
-    private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) {
-        List<SpecialTokens> specialTokensList = new ArrayList<>();
-        for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
-            Tokenlist tokenListConfig = i.next();
-
-            List<SpecialTokens.Token> tokenList = new ArrayList<>();
-            for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) {
-                Tokens tokenConfig = j.next();
-                tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace()));
-            }
-            specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList));
-        }
-        return specialTokensList;
-    }
-
-    /**
-     * Returns the list of special tokens for a given name.
-     *
-     * @param  name the name of the special tokens to return
-     *         null, the empty string or the string "default" returns
-     *         the default ones
-     * @return a read-only list of SpecialToken instances, an empty list if this name
-     *         has no special tokens
-     */
-    public SpecialTokens getSpecialTokens(String name) {
-        if (name == null || name.trim().equals(""))
-            name = "default";
-        return specialTokenMap.getOrDefault(name, SpecialTokens.empty());
-    }
-
-}
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java
deleted file mode 100644
index 465d9b754b3..00000000000
--- a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java
+++ /dev/null
@@ -1,141 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.language.process;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-import static com.yahoo.language.LinguisticsCase.toLowerCase;
-
-/**
- * An immutable list of special tokens - strings which should override the normal tokenizer semantics
- * and be tokenized into a single token. Special tokens are case insensitive.
- *
- * @author bratseth
- */
-public class SpecialTokens {
-
-    private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of());
-
-    private final String name;
-    private final int maximumLength;
-    private final List<Token> tokens;
-    private final Map<String, String> tokenMap;
-
-    public SpecialTokens(String name,  List<Token> tokens) {
-        tokens.stream().peek(token -> token.validate());
-        List<Token> mutableTokens = new ArrayList<>(tokens);
-        Collections.sort(mutableTokens);
-        this.name = name;
-        this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0);
-        this.tokens = List.copyOf(mutableTokens);
-        this.tokenMap = tokens.stream().collect(Collectors.toUnmodifiableMap(t -> t.token(), t -> t.replacement()));
-    }
-
-    /** Returns the name of this special tokens list */
-    public String name() {
-        return name;
-    }
-
-    /**
-     * Returns the tokens of this as an immutable map from token to replacement.
-     * Tokens which do not have a replacement token maps to themselves.
-     */
-    public Map<String, String> asMap() { return tokenMap; }
-
-    /**
-     * Returns the special token starting at the start of the given string, or null if no
-     * special token starts at this string
-     *
-     * @param string the string to search for a special token at the start position
-     * @param substring true to allow the special token to be followed by a character which does not
-     *        mark the end of a token
-     */
-    public Token tokenize(String string, boolean substring) {
-        // XXX detonator pattern token.length may be != the length of the
-        // matching data in string, ref caseIndependentLength(String)
-        String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength)));
-        for (Iterator<Token> i = tokens.iterator(); i.hasNext();) {
-            Token special = i.next();
-
-            if (input.startsWith(special.token())) {
-                if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string))
-                    return special;
-            }
-        }
-        return null;
-    }
-
-    private boolean tokenEndsAt(int position, String string) {
-        return !Character.isLetterOrDigit(string.charAt(position));
-    }
-
-    public static SpecialTokens empty() { return empty; }
-
-    /** An immutable special token */
-    public final static class Token implements Comparable<Token> {
-
-        private final String token;
-        private final String replacement;
-
-        /** Creates a special token */
-        public Token(String token) {
-            this(token, null);
-        }
-
-        /** Creates a special token which will be represented by the given replacement token */
-        public Token(String token, String replacement) {
-            this.token = toLowerCase(token);
-            if (replacement == null || replacement.trim().equals(""))
-                this.replacement = this.token;
-            else
-                this.replacement = toLowerCase(replacement);
-        }
-
-        /** Returns the special token */
-        public String token() { return token; }
-
-        /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */
-        public String replacement() { return replacement; }
-
-        @Override
-        public int compareTo(Token other) {
-            if (this.token().length() < other.token().length()) return 1;
-            if (this.token().length() == other.token().length()) return 0;
-            return -1;
-        }
-
-        @Override
-        public boolean equals(Object other) {
-            if (other == this) return true;
-            if ( ! (other instanceof Token)) return false;
-            return Objects.equals(this.token, ((Token)other).token);
-        }
-
-        @Override
-        public int hashCode() { return token.hashCode(); }
-
-        @Override
-        public String toString() {
-            return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'");
-        }
-
-        private void validate() {
-            // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
-            String asLow = toLowerCase(token);
-            // TODO: Put along with the global toLowerCase
-            String asHigh = token.toUpperCase(Locale.ENGLISH);
-            if (asLow.length() != token.length() || asHigh.length() != token.length()) {
-                throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " +
-                                                   "Please report this to the Vespa team.");
-            }
-        }
-
-    }
-
-}
diff --git a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java
index ad154d1b003..57a5b6edb68 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/TokenType.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/TokenType.java
@@ -4,7 +4,7 @@ package com.yahoo.language.process;
 /**
  * An enumeration of token types.
  *
- * @author Mathias Mølster Lidal
+ * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a>
  */
 public enum TokenType {
 
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
index 4ffe2a866d8..e1a04b2985d 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
@@ -11,14 +11,10 @@ import com.yahoo.language.process.GramSplitter;
 import com.yahoo.language.process.Normalizer;
 import com.yahoo.language.process.Segmenter;
 import com.yahoo.language.process.SegmenterImpl;
-import com.yahoo.language.process.SpecialTokenRegistry;
 import com.yahoo.language.process.Stemmer;
 import com.yahoo.language.process.StemmerImpl;
 import com.yahoo.language.process.Tokenizer;
 import com.yahoo.language.process.Transformer;
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
-
-import java.util.List;
 
 /**
  * Factory of simple linguistic processor implementations.
@@ -35,7 +31,6 @@ public class SimpleLinguistics implements Linguistics {
     private final Detector detector;
     private final CharacterClasses characterClasses;
     private final GramSplitter gramSplitter;
-    private final SpecialTokenRegistry specialTokenRegistry = new SpecialTokenRegistry(List.of());
 
     @Inject
     public SimpleLinguistics() {
@@ -50,7 +45,7 @@ public class SimpleLinguistics implements Linguistics {
     public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); }
 
     @Override
-    public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer, specialTokenRegistry); }
+    public Tokenizer getTokenizer() { return new SimpleTokenizer(normalizer, transformer); }
 
     @Override
     public Normalizer getNormalizer() { return normalizer; }
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
index 740307c0cca..7df432f496d 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java
@@ -23,13 +23,11 @@ import java.util.logging.Level;
  */
 public class SimpleTokenizer implements Tokenizer {
 
-    private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName());
     private final static int SPACE_CODE = 32;
-
     private final Normalizer normalizer;
     private final Transformer transformer;
     private final KStemmer stemmer = new KStemmer();
-    private final SpecialTokenRegistry specialTokenRegistry;
+    private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName());
 
     public SimpleTokenizer() {
         this(new SimpleNormalizer(), new SimpleTransformer());
@@ -40,13 +38,8 @@ public class SimpleTokenizer implements Tokenizer {
     }
 
     public SimpleTokenizer(Normalizer normalizer, Transformer transformer) {
-        this(normalizer, transformer, new SpecialTokenRegistry(List.of()));
-    }
-
-    public SimpleTokenizer(Normalizer normalizer, Transformer transformer, SpecialTokenRegistry specialTokenRegistry) {
         this.normalizer = normalizer;
         this.transformer = transformer;
-        this.specialTokenRegistry = specialTokenRegistry;
     }
 
     @Override
@@ -63,8 +56,8 @@ public class SimpleTokenizer implements Tokenizer {
                 String original = input.substring(prev, next);
                 String token = processToken(original, language, stemMode, removeAccents);
                 tokens.add(new SimpleToken(original).setOffset(prev)
-                                                    .setType(prevType)
-                                                    .setTokenString(token));
+                                                .setType(prevType)
+                                                .setTokenString(token));
                 prev = next;
                 prevType = nextType;
             }
@@ -74,20 +67,20 @@ public class SimpleTokenizer implements Tokenizer {
     }
 
     private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) {
-        String original = token;
-        log.log(Level.FINEST, () -> "processToken '" + original + "'");
+        final String original = token;
+        log.log(Level.FINEST, () -> "processToken '"+original+"'");
         token = normalizer.normalize(token);
         token = LinguisticsCase.toLowerCase(token);
         if (removeAccents)
             token = transformer.accentDrop(token, language);
         if (stemMode != StemMode.NONE) {
-            String oldToken = token;
+            final String oldToken = token;
             token = stemmer.stem(token);
-            String newToken = token;
-            log.log(Level.FINEST, () -> "stem '" + oldToken+"' to '" + newToken+"'");
+            final String newToken = token;
+            log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'");
         }
-        String result = token;
-        log.log(Level.FINEST, () -> "processed token is: " + result);
+        final String result = token;
+        log.log(Level.FINEST, () -> "processed token is: "+result);
         return result;
     }
 
diff --git a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java
deleted file mode 100644
index 47c3ba7933c..00000000000
--- a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.language.process;
-
-import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author bratseth
- */
-public class SpecialTokensTestCase {
-
-    @Test
-    public void testSpecialTokensConfig() {
-        var builder = new SpecialtokensConfig.Builder();
-        var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder();
-        tokenBuilder.name("default");
-
-        var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
-        tokenListBuilder1.token("c++");
-        tokenListBuilder1.replace("cpp");
-        tokenBuilder.tokens(tokenListBuilder1);
-
-        var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
-        tokenListBuilder2.token("...");
-        tokenBuilder.tokens(tokenListBuilder2);
-
-        builder.tokenlist(tokenBuilder);
-
-        var registry = new SpecialTokenRegistry(builder.build());
-
-        var defaultTokens = registry.getSpecialTokens("default");
-        assertEquals("default", defaultTokens.name());
-        assertEquals(2, defaultTokens.asMap().size());
-        assertEquals("cpp", defaultTokens.asMap().get("c++"));
-        assertEquals("...", defaultTokens.asMap().get("..."));
-    }
-
-}