diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2019-06-06 11:06:55 +0200 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2019-06-06 13:01:43 +0200 |
commit | f03acc6b83b346a81c0effcf505d6dea3febdb16 (patch) | |
tree | 34ac1ce7276ba05c3413a921efd7396fb4dd0c2e | |
parent | 2d10eff40215c7b11ae3b01a2fba45ad8135cc61 (diff) |
Do not use chunked output for Base64-encoded raw fields in JSON output
Previous code would always insert at least one linebreak in the output.
Replace Apache Commons encoder with explicit basic `java.util.Base64`
encoder to make us less dependent on magic constructor args.
Explicitly test that we still can decode chunked _input_ to ensure
we do not break roundtrip serialization ability for old outputs.
4 files changed, 27 insertions, 13 deletions
diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 5fc9ee0b2f1..eafc6d214d3 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -32,10 +32,10 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.objects.FieldBase; -import org.apache.commons.codec.binary.Base64; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Base64; import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -45,7 +45,7 @@ import java.util.Set; * @author Vegard Sjonfjell */ public class JsonSerializationHelper { - private final static Base64 base64Encoder = new Base64(); + private final static Base64.Encoder base64Encoder = Base64.getEncoder(); // Important: _basic_ format static class JsonSerializationException extends RuntimeException { public JsonSerializationException(Exception base) { diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index 454ad72f344..57946741335 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -485,7 +485,7 @@ public class DocumentUpdateJsonSerializerTest { " 'update': 'DOCUMENT_ID',", " 'fields': {", " 'raw_field': {", - " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw==\\r\\n'", + " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw=='", " }", " }", "}" diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 9df7d1f91c1..f8ee23e86ba 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -53,7 +53,6 @@ import com.yahoo.tensor.MappedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; -import org.apache.commons.codec.binary.Base64; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -64,8 +63,8 @@ import org.mockito.internal.matchers.Contains; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -603,10 +602,24 @@ public class JsonReaderTestCase { @Test public void testRaw() throws IOException { - String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest")))); + String base64 = new String(new JsonStringEncoder().quoteAsString( + Base64.getEncoder().encodeToString(Utf8.toBytes("smoketest")))); + String s = fieldStringFromBase64RawContent(base64); + assertEquals("smoketest", s); + } + + @Test + public void can_read_legacy_chunked_base64_raw_field_encoding() throws IOException { + String expected = "this is a string with an impressive length. it's long enough to reach the end of the line, wow!"; + String base64withDelims = "dGhpcyBpcyBhIHN0cmluZyB3aXRoIGFuIGltcHJlc3NpdmUgbGVuZ3RoLiBpdCdzIGxvbmcgZW5v\\r\\n" + + "dWdoIHRvIHJlYWNoIHRoZSBlbmQgb2YgdGhlIGxpbmUsIHdvdyE=\\r\\n"; + assertEquals(expected, fieldStringFromBase64RawContent(base64withDelims)); + } + + private String fieldStringFromBase64RawContent(String base64data) throws IOException { JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testraw::whee',", " 'fields': {", - " 'actualraw': '" + stuff + "' }}")); + " 'actualraw': '" + base64data + "' }}")); DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); @@ -615,8 +628,7 @@ public class JsonReaderTestCase { FieldValue f = doc.getFieldValue(doc.getField("actualraw")); assertSame(Raw.class, f.getClass()); Raw s = (Raw) f; - ByteBuffer b = s.getByteBuffer(); - assertEquals("smoketest", Utf8.toString(b)); + return Utf8.toString(s.getByteBuffer()); } @Test diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 0ab00b4e2bc..fcc67136526 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -26,7 +26,6 @@ import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.json.readers.VespaJsonDocumentReader; import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; -import org.apache.commons.codec.binary.Base64; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -273,10 +272,13 @@ public class JsonWriterTestCase { } @Test - public final void rawTest() throws IOException { + public final void raw_fields_are_emitted_as_basic_base64() throws IOException { + // "string long enough to emit more than 76 base64 characters and which should certainly not be newline-delimited!" String payload = new String( - new JsonStringEncoder().quoteAsString(new Base64() - .encodeToString(Utf8.toBytes("smoketest")))); + new JsonStringEncoder().quoteAsString( + "c3RyaW5nIGxvbmcgZW5vdWdoIHRvIGVtaXQgbW9yZSB0aGFuIDc2IGJhc2U2NCBjaGFyYWN0ZXJzIGFuZC" + + "B3aGljaCBzaG91bGQgY2VydGFpbmx5IG5vdCBiZSBuZXdsaW5lLWRlbGltaXRlZCE=")); + String docId = "id:unittest:testraw::whee"; String fields = "{ \"actualraw\": \"" + payload + "\"" + " }"; |