diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-08-21 17:08:08 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2017-08-21 17:08:08 +0200 |
commit | 1441c8ddee4f84cb6e6aa33b67b7eaf7927ce946 (patch) | |
tree | cd49480b30179e575f448cc40674ec9ac23a9eee /vespajlib | |
parent | 55326aa9f686c6cb2a54d4a03c25f1b113bd10a3 (diff) |
Factor out string filtering method
Diffstat (limited to 'vespajlib')
-rw-r--r-- | vespajlib/src/main/java/com/yahoo/text/StringUtilities.java | 1 | ||||
-rw-r--r-- | vespajlib/src/main/java/com/yahoo/text/Text.java | 86 |
2 files changed, 87 insertions, 0 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java index b9e96bdf850..370d079b3ec 100644 --- a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java +++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java @@ -16,6 +16,7 @@ import java.util.Set; * * @author Haakon Humberset */ +// TODO: Text utilities should which are still needed should move to Text. This should be deprecated. public class StringUtilities { private static Charset UTF8 = Charset.forName("utf8"); diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java new file mode 100644 index 00000000000..2b670e5d727 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -0,0 +1,86 @@ +package com.yahoo.text; + +/** + * Text utility functions. + * + * @author bratseth + */ +public final class Text { + + private static final boolean[] allowedAsciiChars = new boolean[0x80]; + + static { + allowedAsciiChars[0x0] = false; + allowedAsciiChars[0x1] = false; + allowedAsciiChars[0x2] = false; + allowedAsciiChars[0x3] = false; + allowedAsciiChars[0x4] = false; + allowedAsciiChars[0x5] = false; + allowedAsciiChars[0x6] = false; + allowedAsciiChars[0x7] = false; + allowedAsciiChars[0x8] = false; + allowedAsciiChars[0x9] = true; //tab + allowedAsciiChars[0xA] = true; //nl + allowedAsciiChars[0xB] = false; + allowedAsciiChars[0xC] = false; + allowedAsciiChars[0xD] = true; //cr + for (int i = 0xE; i < 0x20; i++) { + allowedAsciiChars[i] = false; + } + for (int i = 0x20; i < 0x7F; i++) { + allowedAsciiChars[i] = true; //printable ascii chars + } + allowedAsciiChars[0x7F] = true; //del - discouraged, but allowed + } + + /** No instantiation */ + private Text() {} + + /** + * Returns whether the given codepoint is a valid text character, potentially suitable for + * purposes such as indexing and display, see http://www.w3.org/TR/2006/REC-xml11-20060816/#charsets + */ + public static boolean isTextCharacter(int codepoint) { + // The link above notes that 0x7F-0x84 and 0x86-0x9F are discouraged, but they are still allowed - + // see http://www.w3.org/International/questions/qa-controls + + if (codepoint < 0x80) return allowedAsciiChars[codepoint]; + if (codepoint < 0xFDD0) return true; + if (codepoint <= 0xFDDF) return false; + if (codepoint < 0x1FFFE) return true; + if (codepoint <= 0x1FFFF) return false; + if (codepoint < 0x2FFFE) return true; + if (codepoint <= 0x2FFFF) return false; + if (codepoint < 0x3FFFE) return true; + if (codepoint <= 0x3FFFF) return false; + if (codepoint < 0x4FFFE) return true; + if (codepoint <= 0x4FFFF) return false; + if (codepoint < 0x5FFFE) return true; + if (codepoint <= 0x5FFFF) return false; + if (codepoint < 0x6FFFE) return true; + if (codepoint <= 0x6FFFF) return false; + if (codepoint < 0x7FFFE) return true; + if (codepoint <= 0x7FFFF) return false; + if (codepoint < 0x8FFFE) return true; + if (codepoint <= 0x8FFFF) return false; + if (codepoint < 0x9FFFE) return true; + if (codepoint <= 0x9FFFF) return false; + if (codepoint < 0xAFFFE) return true; + if (codepoint <= 0xAFFFF) return false; + if (codepoint < 0xBFFFE) return true; + if (codepoint <= 0xBFFFF) return false; + if (codepoint < 0xCFFFE) return true; + if (codepoint <= 0xCFFFF) return false; + if (codepoint < 0xDFFFE) return true; + if (codepoint <= 0xDFFFF) return false; + if (codepoint < 0xEFFFE) return true; + if (codepoint <= 0xEFFFF) return false; + if (codepoint < 0xFFFFE) return true; + if (codepoint <= 0xFFFFF) return false; + if (codepoint < 0x10FFFE) return true; + if (codepoint <= 0x10FFFF) return false; + + return true; + } + +} |