Merge pull request #13729 from vespa-engine/arnej/filter-invalid-utf8

filter invalid UTF-8 (including encoded surrogates) to make protobuf …

1 files changed, 16 insertions, 0 deletions

diff --git a/logd/src/tests/proto_converter/proto_converter_test.cpp b/logd/src/tests/proto_converter/proto_converter_test.cpp
index aa0b00e34d6..702752e8482 100644
--- a/logd/src/tests/proto_converter/proto_converter_test.cpp
+++ b/logd/src/tests/proto_converter/proto_converter_test.cpp
@@ -84,5 +84,21 @@ TEST_F(LogRequestTest, log_messages_are_converted_to_request)
                                    ProtoLogLevel::LogMessage_Level_EVENT, "bar_payload", proto.log_messages(1));
 }
 
+// UTF-8 encoding of \U+FFFD
+#define FFFD "\xEF\xBF\xBD"
+
+TEST_F(LogRequestTest, invalid_utf8_is_filtered)
+{
+    messages.emplace_back(12345, "foo_host", 3, 5, "foo_service", "foo_component", Logger::info,
+        "valid: \xE2\x82\xAC and \xEF\xBF\xBA; semi-valid: \xED\xA0\xBD\xED\xB8\x80; invalid: \xCC surrogate \xED\xBF\xBF overlong \xC1\x81 end"
+    );
+    convert();
+    EXPECT_EQ(1, proto.log_messages_size());
+    expect_proto_log_message_equal(12345, "foo_host", 3, 5, "foo_service", "foo_component",
+        ProtoLogLevel::LogMessage_Level_INFO,
+        "valid: \xE2\x82\xAC and \xEF\xBF\xBA; semi-valid: " FFFD FFFD "; invalid: " FFFD " surrogate " FFFD " overlong " FFFD FFFD " end",
+        proto.log_messages(0));
+}
+
 GTEST_MAIN_RUN_ALL_TESTS()