Move specialtokens to linguistics

4 files changed, 299 insertions, 0 deletions

diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json
index 58b838d7332..34c388b8a2e 100644
--- a/linguistics/abi-spec.json
+++ b/linguistics/abi-spec.json
@@ -427,6 +427,57 @@
     ],
     "fields": []
   },
+  "com.yahoo.language.process.SpecialTokenRegistry": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>()",
+      "public void <init>(com.yahoo.vespa.configdefinition.SpecialtokensConfig)",
+      "public void <init>(java.util.List)",
+      "public com.yahoo.language.process.SpecialTokens getSpecialTokens(java.lang.String)"
+    ],
+    "fields": []
+  },
+  "com.yahoo.language.process.SpecialTokens$Token": {
+    "superClass": "java.lang.Object",
+    "interfaces": [
+      "java.lang.Comparable"
+    ],
+    "attributes": [
+      "public",
+      "final"
+    ],
+    "methods": [
+      "public void <init>(java.lang.String)",
+      "public void <init>(java.lang.String, java.lang.String)",
+      "public java.lang.String token()",
+      "public java.lang.String replacement()",
+      "public int compareTo(com.yahoo.language.process.SpecialTokens$Token)",
+      "public boolean equals(java.lang.Object)",
+      "public int hashCode()",
+      "public java.lang.String toString()",
+      "public bridge synthetic int compareTo(java.lang.Object)"
+    ],
+    "fields": []
+  },
+  "com.yahoo.language.process.SpecialTokens": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>(java.lang.String, java.util.List)",
+      "public java.lang.String name()",
+      "public java.util.List tokens()",
+      "public com.yahoo.language.process.SpecialTokens$Token tokenize(java.lang.String, boolean)",
+      "public static com.yahoo.language.process.SpecialTokens empty()"
+    ],
+    "fields": []
+  },
   "com.yahoo.language.process.StemList": {
     "superClass": "java.util.AbstractList",
     "interfaces": [],
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java
new file mode 100644
index 00000000000..b65c3ba663c
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokenRegistry.java
@@ -0,0 +1,72 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.process;
+
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist;
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * A registry which is responsible for knowing the current
+ * set of special tokens.Usage of this registry is multithread safe.
+ *
+ * @author bratseth
+ */
+public class SpecialTokenRegistry {
+
+    /**
+     * The current special token lists, indexed on name.
+     * These lists are unmodifiable and used directly by clients of this
+     */
+    private final Map<String, SpecialTokens> specialTokenMap;
+
+    /** Creates an empty special token registry */
+    public SpecialTokenRegistry() {
+        this(List.of());
+    }
+
+    /** Create a special token registry from a configuration object. */
+    public SpecialTokenRegistry(SpecialtokensConfig config) {
+        this(specialTokensFrom(config));
+    }
+
+    public SpecialTokenRegistry(List<SpecialTokens> specialTokensList) {
+        specialTokenMap = specialTokensList.stream().collect(Collectors.toMap(t -> t.name(), t -> t));
+    }
+
+    private static List<SpecialTokens> specialTokensFrom(SpecialtokensConfig config) {
+        List<SpecialTokens> specialTokensList = new ArrayList<>();
+        for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext();) {
+            Tokenlist tokenListConfig = i.next();
+
+            List<SpecialTokens.Token> tokenList = new ArrayList<>();
+            for (Iterator<Tokens> j = tokenListConfig.tokens().iterator(); j.hasNext();) {
+                Tokens tokenConfig = j.next();
+                tokenList.add(new SpecialTokens.Token(tokenConfig.token(), tokenConfig.replace()));
+            }
+            specialTokensList.add(new SpecialTokens(tokenListConfig.name(), tokenList));
+        }
+        return specialTokensList;
+    }
+
+    /**
+     * Returns the list of special tokens for a given name.
+     *
+     * @param  name the name of the special tokens to return
+     *         null, the empty string or the string "default" returns
+     *         the default ones
+     * @return a read-only list of SpecialToken instances, an empty list if this name
+     *         has no special tokens
+     */
+    public SpecialTokens getSpecialTokens(String name) {
+        if (name == null || name.trim().equals(""))
+            name = "default";
+        return specialTokenMap.getOrDefault(name, SpecialTokens.empty());
+    }
+
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java
new file mode 100644
index 00000000000..c1b05a00377
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/process/SpecialTokens.java
@@ -0,0 +1,134 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.process;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+
+/**
+ * A list of special tokens - string that should be treated as word
+ * no matter what they contain. Special tokens are case insensitive.
+ *
+ * @author bratseth
+ */
+public class SpecialTokens {
+
+    private static final SpecialTokens empty = new SpecialTokens("(empty)", List.of());
+
+    private final String name;
+    private final List<Token> tokens;
+    private final int maximumLength;
+
+    public SpecialTokens(String name,  List<Token> tokens) {
+        tokens.stream().peek(token -> token.validate());
+        List<Token> mutableTokens = new ArrayList<>(tokens);
+        Collections.sort(mutableTokens);
+        this.tokens = List.copyOf(mutableTokens);
+        this.name = name;
+        this.maximumLength = tokens.stream().mapToInt(token -> token.token().length()).max().orElse(0);
+    }
+
+    /** Returns the name of this special tokens list */
+    public String name() {
+        return name;
+    }
+
+    /** Returns a sorted immutable list of the special tokens in this */
+    public List<Token> tokens() { return tokens; }
+
+    /**
+     * Returns the special token starting at the start of the given string, or null if no
+     * special token starts at this string
+     *
+     * @param string the string to search for a special token at the start position
+     * @param substring true to allow the special token to be followed by a character which does not
+     *        mark the end of a token
+     */
+    public Token tokenize(String string, boolean substring) {
+        // XXX detonator pattern token.length may be != the length of the
+        // matching data in string, ref caseIndependentLength(String)
+        String input = toLowerCase(string.substring(0, Math.min(string.length(), maximumLength)));
+        for (Iterator<Token> i = tokens.iterator(); i.hasNext();) {
+            Token special = i.next();
+
+            if (input.startsWith(special.token())) {
+                if (string.length() == special.token().length() || substring || tokenEndsAt(special.token().length(), string))
+                    return special;
+            }
+        }
+        return null;
+    }
+
+    private boolean tokenEndsAt(int position,String string) {
+        return !Character.isLetterOrDigit(string.charAt(position));
+    }
+
+    public static SpecialTokens empty() { return empty; }
+
+    /** An immutable special token */
+    public final static class Token implements Comparable<Token> {
+
+        private final String token;
+        private final String replacement;
+
+        /** Creates a special token */
+        public Token(String token) {
+            this(token, null);
+        }
+
+        /** Creates a special token which will be represented by the given replacement token */
+        public Token(String token, String replacement) {
+            this.token = toLowerCase(token);
+            if (replacement == null || replacement.trim().equals(""))
+                this.replacement = this.token;
+            else
+                this.replacement = toLowerCase(replacement);
+        }
+
+        /** Returns the special token */
+        public String token() { return token; }
+
+        /** Returns the token to replace occurrences of this by, which equals token() unless this has a replacement. */
+        public String replacement() { return replacement; }
+
+        @Override
+        public int compareTo(Token other) {
+            if (this.token().length() < other.token().length()) return 1;
+            if (this.token().length() == other.token().length()) return 0;
+            return -1;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (other == this) return true;
+            if ( ! (other instanceof Token)) return false;
+            return Objects.equals(this.token, ((Token)other).token);
+        }
+
+        @Override
+        public int hashCode() { return token.hashCode(); }
+
+        @Override
+        public String toString() {
+            return "token '" + token + "'" + (replacement.equals(token) ? "" : " replacement '" + replacement + "'");
+        }
+
+        private void validate() {
+            // XXX not fool proof length test, should test codepoint by codepoint for mixed case user input? not even that will necessarily be 100% robust...
+            String asLow = toLowerCase(token);
+            // TODO: Put along with the global toLowerCase
+            String asHigh = token.toUpperCase(Locale.ENGLISH);
+            if (asLow.length() != token.length() || asHigh.length() != token.length()) {
+                throw new IllegalArgumentException("Special token '" + token + "' has case sensitive length. " +
+                                                   "Please report this to the Vespa team.");
+            }
+        }
+
+    }
+
+}
diff --git a/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java
new file mode 100644
index 00000000000..fee70e3a407
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/process/SpecialTokensTestCase.java
@@ -0,0 +1,42 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.process;
+
+import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class SpecialTokensTestCase {
+
+    @Test
+    public void testSpecialTokensConfig() {
+        var builder = new SpecialtokensConfig.Builder();
+        var tokenBuilder = new SpecialtokensConfig.Tokenlist.Builder();
+        tokenBuilder.name("default");
+
+        var tokenListBuilder1 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
+        tokenListBuilder1.token("c++");
+        tokenListBuilder1.replace("cpp");
+        tokenBuilder.tokens(tokenListBuilder1);
+
+        var tokenListBuilder2 = new SpecialtokensConfig.Tokenlist.Tokens.Builder();
+        tokenListBuilder2.token("...");
+        tokenBuilder.tokens(tokenListBuilder2);
+
+        builder.tokenlist(tokenBuilder);
+
+        var registry = new SpecialTokenRegistry(builder.build());
+
+        var defaultTokens = registry.getSpecialTokens("default");
+        assertEquals("default", defaultTokens.name());
+        assertEquals(2, defaultTokens.tokens().size());
+        assertEquals("c++", defaultTokens.tokens().get(0).token());
+        assertEquals("cpp", defaultTokens.tokens().get(0).replacement());
+        assertEquals("...", defaultTokens.tokens().get(1).token());
+        assertEquals("...", defaultTokens.tokens().get(1).replacement());
+    }
+
+}