Factor out string filtering method

2 files changed, 87 insertions, 0 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java
index b9e96bdf850..370d079b3ec 100644
--- a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java
+++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java
@@ -16,6 +16,7 @@ import java.util.Set;
  *
  * @author Haakon Humberset
  */
+// TODO: Text utilities should which are still needed should move to Text. This should be deprecated.
 public class StringUtilities {
 
     private static Charset UTF8 = Charset.forName("utf8");
diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java
new file mode 100644
index 00000000000..2b670e5d727
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/text/Text.java
@@ -0,0 +1,86 @@
+package com.yahoo.text;
+
+/**
+ * Text utility functions.
+ * 
+ * @author bratseth
+ */
+public final class Text {
+
+    private static final boolean[] allowedAsciiChars = new boolean[0x80];
+
+    static {
+        allowedAsciiChars[0x0] = false;
+        allowedAsciiChars[0x1] = false;
+        allowedAsciiChars[0x2] = false;
+        allowedAsciiChars[0x3] = false;
+        allowedAsciiChars[0x4] = false;
+        allowedAsciiChars[0x5] = false;
+        allowedAsciiChars[0x6] = false;
+        allowedAsciiChars[0x7] = false;
+        allowedAsciiChars[0x8] = false;
+        allowedAsciiChars[0x9] = true;  //tab
+        allowedAsciiChars[0xA] = true;  //nl
+        allowedAsciiChars[0xB] = false;
+        allowedAsciiChars[0xC] = false;
+        allowedAsciiChars[0xD] = true;  //cr
+        for (int i = 0xE; i < 0x20; i++) {
+            allowedAsciiChars[i] = false;
+        }
+        for (int i = 0x20; i < 0x7F; i++) {
+            allowedAsciiChars[i] = true;  //printable ascii chars
+        }
+        allowedAsciiChars[0x7F] = true;  //del - discouraged, but allowed
+    }
+
+    /** No instantiation */
+    private Text() {}
+
+    /**
+     * Returns whether the given codepoint is a valid text character, potentially suitable for 
+     * purposes such as indexing and display, see http://www.w3.org/TR/2006/REC-xml11-20060816/#charsets
+     */
+    public static boolean isTextCharacter(int codepoint) {
+        // The link above notes that 0x7F-0x84 and 0x86-0x9F are discouraged, but they are still allowed -
+        // see http://www.w3.org/International/questions/qa-controls
+
+        if (codepoint <  0x80)     return allowedAsciiChars[codepoint];
+        if (codepoint <  0xFDD0)   return true;
+        if (codepoint <= 0xFDDF)   return false;
+        if (codepoint <  0x1FFFE)  return true;
+        if (codepoint <= 0x1FFFF)  return false;
+        if (codepoint <  0x2FFFE)  return true;
+        if (codepoint <= 0x2FFFF)  return false;
+        if (codepoint <  0x3FFFE)  return true;
+        if (codepoint <= 0x3FFFF)  return false;
+        if (codepoint <  0x4FFFE)  return true;
+        if (codepoint <= 0x4FFFF)  return false;
+        if (codepoint <  0x5FFFE)  return true;
+        if (codepoint <= 0x5FFFF)  return false;
+        if (codepoint <  0x6FFFE)  return true;
+        if (codepoint <= 0x6FFFF)  return false;
+        if (codepoint <  0x7FFFE)  return true;
+        if (codepoint <= 0x7FFFF)  return false;
+        if (codepoint <  0x8FFFE)  return true;
+        if (codepoint <= 0x8FFFF)  return false;
+        if (codepoint <  0x9FFFE)  return true;
+        if (codepoint <= 0x9FFFF)  return false;
+        if (codepoint <  0xAFFFE)  return true;
+        if (codepoint <= 0xAFFFF)  return false;
+        if (codepoint <  0xBFFFE)  return true;
+        if (codepoint <= 0xBFFFF)  return false;
+        if (codepoint <  0xCFFFE)  return true;
+        if (codepoint <= 0xCFFFF)  return false;
+        if (codepoint <  0xDFFFE)  return true;
+        if (codepoint <= 0xDFFFF)  return false;
+        if (codepoint <  0xEFFFE)  return true;
+        if (codepoint <= 0xEFFFF)  return false;
+        if (codepoint <  0xFFFFE)  return true;
+        if (codepoint <= 0xFFFFF)  return false;
+        if (codepoint <  0x10FFFE) return true;
+        if (codepoint <= 0x10FFFF) return false;
+
+        return true;
+    }    
+    
+}