Merge pull request #13729 from vespa-engine/arnej/filter-invalid-utf8

filter invalid UTF-8 (including encoded surrogates) to make protobuf …

2 files changed, 35 insertions, 6 deletions

diff --git a/vespalib/src/vespa/vespalib/text/utf8.cpp b/vespalib/src/vespa/vespalib/text/utf8.cpp
index 660178aaaa1..58b587d45b5 100644
--- a/vespalib/src/vespa/vespalib/text/utf8.cpp
+++ b/vespalib/src/vespa/vespalib/text/utf8.cpp
@@ -175,8 +175,9 @@ Utf8ReaderForZTS::getComplexChar(unsigned char firstbyte, uint32_t fallback)
 }
 
 
-Utf8Writer&
-Utf8Writer::putChar(uint32_t codepoint)
+template <typename Target>
+Utf8Writer<Target>&
+Utf8Writer<Target>::putChar(uint32_t codepoint)
 {
     if (codepoint < 0x80) {
         _target.push_back((char)codepoint);
@@ -229,5 +230,23 @@ Utf8Writer::putChar(uint32_t codepoint)
     return *this;
 }
 
+template class Utf8Writer<vespalib::string>;
+template class Utf8Writer<std::string>;
 
-} // namespace vespalib
+template <typename T>
+T Utf8::filter_invalid_sequences(const T& input)
+{
+    T retval;
+    Utf8Reader reader(input.c_str(), input.size());
+    Utf8Writer writer(retval);
+    while (reader.hasMore()) {
+        uint32_t ch = reader.getChar();
+        writer.putChar(ch);
+    }
+    return retval;
+}
+
+template vespalib::string Utf8::filter_invalid_sequences(const vespalib::string&);
+template std::string Utf8::filter_invalid_sequences(const std::string&);
+
+} // namespace
diff --git a/vespalib/src/vespa/vespalib/text/utf8.h b/vespalib/src/vespa/vespalib/text/utf8.h
index 0c75203fbbe..e65aaee9708 100644
--- a/vespalib/src/vespa/vespalib/text/utf8.h
+++ b/vespalib/src/vespa/vespalib/text/utf8.h
@@ -28,6 +28,15 @@ public:
     };
 
     /**
+     * Filter a string (std::string or vespalib::string)
+     * and replace any invalid UTF8 sequences with the
+     * standard replacement char U+FFFD; note that any
+     * UTF-8 encoded surrogates are also considered invalid.
+     **/
+    template <typename T>
+    static T filter_invalid_sequences(const T& input);
+
+    /**
      * check if a byte is valid as the first byte of an UTF-8 character.
      * @param c the byte to be checked
      * @return true if a valid UTF-8 character can start with this byte
@@ -155,7 +164,7 @@ protected:
         first_high_surrogate = 0xD800,
         last_high_surrogate = 0xDBFF,
         first_low_surrogate = 0xDC00,
-        last_low_surrogate = 0xDCFF
+        last_low_surrogate = 0xDFFF
     };
 };
 
@@ -321,9 +330,10 @@ public:
 /**
  * @brief Writer class that appends UTF-8 characters to a string
  **/
+template <typename Target>
 class Utf8Writer : public Utf8
 {
-    string &_target;
+    Target &_target;
 public:
     /**
      * construct a writer appending to the given string
@@ -331,7 +341,7 @@ public:
      * that the writer will append to.  Must be writable
      * and must be kept alive while the writer is active.
      **/
-    Utf8Writer(string &target) : _target(target) {}
+    Utf8Writer(Target &target) : _target(target) {}
 
     /**
      * append the given character to the target string.