Do not use chunked output for Base64-encoded raw fields in JSON output

Previous code would always insert at least one linebreak in the output.
Replace Apache Commons encoder with explicit basic `java.util.Base64`
encoder to make us less dependent on magic constructor args.

Explicitly test that we still can decode chunked _input_ to ensure
we do not break roundtrip serialization ability for old outputs.

4 files changed, 27 insertions, 13 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
index 5fc9ee0b2f1..eafc6d214d3 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java
@@ -32,10 +32,10 @@ import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorAddress;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.vespa.objects.FieldBase;
-import org.apache.commons.codec.binary.Base64;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Base64;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
@@ -45,7 +45,7 @@ import java.util.Set;
  * @author Vegard Sjonfjell
  */
 public class JsonSerializationHelper {
-    private final static Base64 base64Encoder = new Base64();
+    private final static Base64.Encoder base64Encoder = Base64.getEncoder(); // Important: _basic_ format
 
     static class JsonSerializationException extends RuntimeException {
         public JsonSerializationException(Exception base) {
diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
index 454ad72f344..57946741335 100644
--- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
+++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java
@@ -485,7 +485,7 @@ public class DocumentUpdateJsonSerializerTest {
                         "    'update': 'DOCUMENT_ID',",
                         "    'fields': {",
                         "        'raw_field': {",
-                        "            'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw==\\r\\n'",
+                        "            'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw=='",
                         "        }",
                         "    }",
                         "}"
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 9df7d1f91c1..f8ee23e86ba 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -53,7 +53,6 @@ import com.yahoo.tensor.MappedTensor;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.text.Utf8;
-import org.apache.commons.codec.binary.Base64;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -64,8 +63,8 @@ import org.mockito.internal.matchers.Contains;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -603,10 +602,24 @@ public class JsonReaderTestCase {
 
     @Test
     public void testRaw() throws IOException {
-        String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest"))));
+        String base64 = new String(new JsonStringEncoder().quoteAsString(
+                Base64.getEncoder().encodeToString(Utf8.toBytes("smoketest"))));
+        String s = fieldStringFromBase64RawContent(base64);
+        assertEquals("smoketest", s);
+    }
+
+    @Test
+    public void can_read_legacy_chunked_base64_raw_field_encoding() throws IOException {
+        String expected = "this is a string with an impressive length. it's long enough to reach the end of the line, wow!";
+        String base64withDelims = "dGhpcyBpcyBhIHN0cmluZyB3aXRoIGFuIGltcHJlc3NpdmUgbGVuZ3RoLiBpdCdzIGxvbmcgZW5v\\r\\n" +
+                "dWdoIHRvIHJlYWNoIHRoZSBlbmQgb2YgdGhlIGxpbmUsIHdvdyE=\\r\\n";
+        assertEquals(expected, fieldStringFromBase64RawContent(base64withDelims));
+    }
+
+    private String fieldStringFromBase64RawContent(String base64data) throws IOException {
         JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testraw::whee',",
                         "  'fields': {",
-                        "    'actualraw': '" + stuff + "' }}"));
+                        "    'actualraw': '" + base64data + "' }}"));
         DocumentParseInfo parseInfo = r.parseDocument().get();
         DocumentType docType = r.readDocumentType(parseInfo.documentId);
         DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId));
@@ -615,8 +628,7 @@ public class JsonReaderTestCase {
         FieldValue f = doc.getFieldValue(doc.getField("actualraw"));
         assertSame(Raw.class, f.getClass());
         Raw s = (Raw) f;
-        ByteBuffer b = s.getByteBuffer();
-        assertEquals("smoketest", Utf8.toString(b));
+        return Utf8.toString(s.getByteBuffer());
     }
 
     @Test
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
index 0ab00b4e2bc..fcc67136526 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -26,7 +26,6 @@ import com.yahoo.document.json.readers.DocumentParseInfo;
 import com.yahoo.document.json.readers.VespaJsonDocumentReader;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.text.Utf8;
-import org.apache.commons.codec.binary.Base64;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -273,10 +272,13 @@ public class JsonWriterTestCase {
     }
 
     @Test
-    public final void rawTest() throws IOException {
+    public final void raw_fields_are_emitted_as_basic_base64() throws IOException {
+        // "string long enough to emit more than 76 base64 characters and which should certainly not be newline-delimited!"
         String payload = new String(
-                new JsonStringEncoder().quoteAsString(new Base64()
-                        .encodeToString(Utf8.toBytes("smoketest"))));
+                new JsonStringEncoder().quoteAsString(
+                        "c3RyaW5nIGxvbmcgZW5vdWdoIHRvIGVtaXQgbW9yZSB0aGFuIDc2IGJhc2U2NCBjaGFyYWN0ZXJzIGFuZC" +
+                              "B3aGljaCBzaG91bGQgY2VydGFpbmx5IG5vdCBiZSBuZXdsaW5lLWRlbGltaXRlZCE="));
+
         String docId = "id:unittest:testraw::whee";
 
         String fields = "{ \"actualraw\": \"" + payload + "\"" + " }";