Don't skip too far

4 files changed, 43 insertions, 19 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
index 58ceeec4f4c..6b2bdbd53d8 100644
--- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
+++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java
@@ -17,18 +17,6 @@ import com.google.common.base.Preconditions;
  */
 public class TokenBuffer {
 
-    public static final class Token {
-        public final JsonToken token;
-        public final String name;
-        public final String text;
-
-        Token(JsonToken token, String name, String text) {
-            this.token = token;
-            this.name = name;
-            this.text = text;
-        }
-    }
-
     private final Deque<Token> buffer;
     private int nesting = 0;
 
@@ -181,4 +169,27 @@ public class TokenBuffer {
         } while ( nesting() > initialNesting + relativeNesting);
     }
 
+    public Deque<Token> rest() {
+        return buffer;
+    }
+
+    public static final class Token {
+
+        public final JsonToken token;
+        public final String name;
+        public final String text;
+
+        Token(JsonToken token, String name, String text) {
+            this.token = token;
+            this.name = name;
+            this.text = text;
+        }
+
+        @Override
+        public String toString() {
+            return "Token(" + token + ", " + name + ", " + text + ")";
+        }
+
+    }
+
 }
diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
index 6cbc1c1e0b1..27c18ea2f69 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java
@@ -7,7 +7,6 @@ import com.yahoo.document.PositionDataType;
 import com.yahoo.document.datatypes.CollectionFieldValue;
 import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.MapFieldValue;
-import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.StructuredFieldValue;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.document.datatypes.WeightedSet;
diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
index b944d273a72..d489c03eb9d 100644
--- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
+++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java
@@ -30,8 +30,7 @@ public class StructReader {
                     throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" +
                                                        parent.getDataType().getDataTypeName() +
                                                        "', which has the fields: " + parent.getDataType().getFields());
-
-                buffer.skipToRelativeNesting(0);
+                buffer.skipToRelativeNesting(1);
                 fullyApplied = false;
                 continue;
             }
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 97422217857..41d607b0d8e 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -108,6 +108,7 @@ public class JsonReaderTestCase {
             x.addField(new Field("field2", DataType.STRING));
             x.addField(new Field("int1", DataType.INT));
             x.addField(new Field("flag", DataType.BOOL));
+            x.addField(new Field("tensor1", DataType.getTensor(TensorType.fromSpec("tensor(x{})"))));
             types.registerDocumentType(x);
         }
         {
@@ -990,7 +991,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public void nonExistingFieldCausesException()  throws IOException{
+    public void nonExistingFieldCausesException() throws IOException {
         JsonReader r = createReader(inputJson(
                 "{ 'put': 'id:unittest:smoke::whee',",
                 "  'fields': {",
@@ -1009,7 +1010,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public void nonExistingFieldsCanBeIgnoredInPut()  throws IOException{
+    public void nonExistingFieldsCanBeIgnoredInPut() throws IOException {
         JsonReader r = createReader(inputJson(
                 "{ ",
                 "  'put': 'id:unittest:smoke::doc1',",
@@ -1023,7 +1024,11 @@ public class JsonReaderTestCase {
                 "      }",
                 "    },",
                 "    'field2': 'value2',",
-                "    'nonexisting3': 'ignored value'",
+                "    'nonexisting3': {",
+                "      'cells': [{'address': {'x': 'x1'}, 'value': 1.0}]",
+                "    },",
+                "    'tensor1': {'cells': {'x1': 1.0}},",
+                "    'nonexisting4': 'ignored value'",
                 "  }",
                 "}"));
         DocumentParseInfo parseInfo = r.parseDocument().get();
@@ -1036,6 +1041,8 @@ public class JsonReaderTestCase {
         assertNull(put.getDocument().getField("nonexisting2"));
         assertEquals("value2", put.getDocument().getFieldValue("field2").toString());
         assertNull(put.getDocument().getField("nonexisting3"));
+        assertEquals(Tensor.from("tensor(x{}):{{x:x1}:1.0}"), put.getDocument().getFieldValue("tensor1").getWrappedValue());
+        assertNull(put.getDocument().getField("nonexisting4"));
     }
 
     @Test
@@ -1055,7 +1062,13 @@ public class JsonReaderTestCase {
                 "      }",
                 "    },",
                 "    'field2': { 'assign': 'value2' },",
-                "    'nonexisting3': { 'assign': 'ignored value' }",
+                "    'nonexisting3': {",
+                "      'assign' : {",
+                "        'cells': [{'address': {'x': 'x1'}, 'value': 1.0}]",
+                "      }",
+                "    },",
+                "    'tensor1': {'assign': { 'cells': {'x1': 1.0} } },",
+                "    'nonexisting4': { 'assign': 'ignored value' }",
                 "  }",
                 "}"));
         DocumentParseInfo parseInfo = r.parseDocument().get();
@@ -1068,6 +1081,8 @@ public class JsonReaderTestCase {
         assertNull(update.getFieldUpdate("nonexisting2"));
         assertEquals("value2", update.getFieldUpdate("field2").getValueUpdates().get(0).getValue().getWrappedValue().toString());
         assertNull(update.getFieldUpdate("nonexisting3"));
+        assertEquals(Tensor.from("tensor(x{}):{{x:x1}:1.0}"), update.getFieldUpdate("tensor1").getValueUpdates().get(0).getValue().getWrappedValue());
+        assertNull(update.getFieldUpdate("nonexisting4"));
     }
 
     @Test