Merge pull request #8309 from vespa-engine/lesters/skip-null-fields-in-document-put

Skip fields with null values when parsing document puts

3 files changed, 62 insertions, 2 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
index 779d706ac3c..824a3073515 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.json.readers;
 
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Preconditions;
 import com.yahoo.document.DataType;
 import com.yahoo.document.datatypes.CollectionFieldValue;
 import com.yahoo.document.datatypes.FieldValue;
@@ -14,6 +16,7 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;
 public class ArrayReader {
     public static void fillArrayUpdate(TokenBuffer buffer, int initNesting, DataType valueType, List<FieldValue> arrayContents) {
         while (buffer.nesting() >= initNesting) {
+            Preconditions.checkArgument(buffer.currentToken() != JsonToken.VALUE_NULL, "Illegal null value for array entry");
             arrayContents.add(readSingleValue(buffer, valueType));
             buffer.next();
         }
@@ -25,6 +28,7 @@ public class ArrayReader {
         expectArrayStart(buffer.currentToken());
         buffer.next();
         while (buffer.nesting() >= initNesting) {
+            Preconditions.checkArgument(buffer.currentToken() != JsonToken.VALUE_NULL, "Illegal null value for array entry");
             parent.add(readSingleValue(buffer, valueType));
             buffer.next();
         }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
index 3d2d08842dc..cab55903b76 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document.json.readers;
 
+import com.fasterxml.jackson.core.JsonToken;
 import com.yahoo.document.Field;
 import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.StructuredFieldValue;
@@ -18,8 +19,11 @@ public class StructReader {
         while (buffer.nesting() >= initNesting) {
             Field f = getField(buffer, parent);
             try {
-                FieldValue v = readSingleValue(buffer, f.getDataType());
-                parent.setFieldValue(f, v);
+                // skip fields set to null
+                if (buffer.currentToken() != JsonToken.VALUE_NULL) {
+                    FieldValue v = readSingleValue(buffer, f.getDataType());
+                    parent.setFieldValue(f, v);
+                }
                 buffer.next();
             } catch (IllegalArgumentException e) {
                 throw new JsonReaderException(f, e);
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 9afebc16a3b..1ef671d90c0 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -167,6 +167,16 @@ public class JsonReaderTestCase {
             x.addField(new Field("integerfield", DataType.INT));
             types.registerDocumentType(x);
         }
+        {
+            DocumentType x = new DocumentType("testnull");
+            x.addField(new Field("intfield", DataType.INT));
+            x.addField(new Field("stringfield", DataType.STRING));
+            x.addField(new Field("arrayfield", new ArrayDataType(DataType.STRING)));
+            x.addField(new Field("weightedsetfield", new WeightedSetDataType(DataType.STRING, true, true)));
+            x.addField(new Field("mapfield",  new MapDataType(DataType.STRING, DataType.STRING)));
+            x.addField(new Field("tensorfield", new TensorDataType(new TensorType.Builder().indexed("x").build())));
+            types.registerDocumentType(x);
+        }
     }
 
     @After
@@ -1166,6 +1176,48 @@ public class JsonReaderTestCase {
         }
     }
 
+    @Test
+    public void testNullValues() {
+        JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testnull::doc1',",
+                "  'fields': {",
+                "    'intfield': null,",
+                "    'stringfield': null,",
+                "    'arrayfield': null,",
+                "    'weightedsetfield': null,",
+                "    'mapfield': null,",
+                "    'tensorfield': null",
+                "  }",
+                "}"));
+        DocumentPut put = (DocumentPut) r.readSingleDocument(DocumentParser.SupportedOperation.PUT,
+                "id:unittest:testnull::doc1");
+        Document doc = put.getDocument();
+        assertFieldValueNull(doc, "intfield");
+        assertFieldValueNull(doc, "stringfield");
+        assertFieldValueNull(doc, "arrayfield");
+        assertFieldValueNull(doc, "weightedsetfield");
+        assertFieldValueNull(doc, "mapfield");
+        assertFieldValueNull(doc, "tensorfield");
+    }
+
+    @Test(expected=JsonReaderException.class)
+    public void testNullArrayElement() {
+        JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testnull::doc1',",
+                "  'fields': {",
+                "    'arrayfield': [ null ]",
+                "  }",
+                "}"));
+        r.readSingleDocument(DocumentParser.SupportedOperation.PUT, "id:unittest:testnull::doc1");
+        fail();
+    }
+
+    private void assertFieldValueNull(Document doc, String fieldName) {
+        Field field = doc.getField(fieldName);
+        assertNotNull(field);
+        FieldValue fieldValue = doc.getFieldValue(field);
+        assertNull(fieldValue);
+    }
+
+
     static ByteArrayInputStream jsonToInputStream(String json) {
         return new ByteArrayInputStream(Utf8.toBytes(json));
     }