From af5ac0cf75da78148b80ad49ec82fa3cd0ccb526 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 9 Jan 2023 11:52:03 +0100 Subject: Don't skip too far --- .../java/com/yahoo/document/json/TokenBuffer.java | 35 ++++++++++++++-------- .../document/json/readers/CompositeReader.java | 1 - .../yahoo/document/json/readers/StructReader.java | 3 +- .../yahoo/document/json/JsonReaderTestCase.java | 23 +++++++++++--- 4 files changed, 43 insertions(+), 19 deletions(-) (limited to 'document') diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 58ceeec4f4c..6b2bdbd53d8 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -17,18 +17,6 @@ import com.google.common.base.Preconditions; */ public class TokenBuffer { - public static final class Token { - public final JsonToken token; - public final String name; - public final String text; - - Token(JsonToken token, String name, String text) { - this.token = token; - this.name = name; - this.text = text; - } - } - private final Deque buffer; private int nesting = 0; @@ -181,4 +169,27 @@ public class TokenBuffer { } while ( nesting() > initialNesting + relativeNesting); } + public Deque rest() { + return buffer; + } + + public static final class Token { + + public final JsonToken token; + public final String name; + public final String text; + + Token(JsonToken token, String name, String text) { + this.token = token; + this.name = name; + this.text = text; + } + + @Override + public String toString() { + return "Token(" + token + ", " + name + ", " + text + ")"; + } + + } + } diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java index 6cbc1c1e0b1..27c18ea2f69 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java @@ -7,7 +7,6 @@ import com.yahoo.document.PositionDataType; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.MapFieldValue; -import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.StructuredFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java index b944d273a72..d489c03eb9d 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java @@ -30,8 +30,7 @@ public class StructReader { throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" + parent.getDataType().getDataTypeName() + "', which has the fields: " + parent.getDataType().getFields()); - - buffer.skipToRelativeNesting(0); + buffer.skipToRelativeNesting(1); fullyApplied = false; continue; } diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 97422217857..41d607b0d8e 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -108,6 +108,7 @@ public class JsonReaderTestCase { x.addField(new Field("field2", DataType.STRING)); x.addField(new Field("int1", DataType.INT)); x.addField(new Field("flag", DataType.BOOL)); + x.addField(new Field("tensor1", DataType.getTensor(TensorType.fromSpec("tensor(x{})")))); types.registerDocumentType(x); } { @@ -990,7 +991,7 @@ public class JsonReaderTestCase { } @Test - public void nonExistingFieldCausesException() throws IOException{ + public void nonExistingFieldCausesException() throws IOException { JsonReader r = createReader(inputJson( "{ 'put': 'id:unittest:smoke::whee',", " 'fields': {", @@ -1009,7 +1010,7 @@ public class JsonReaderTestCase { } @Test - public void nonExistingFieldsCanBeIgnoredInPut() throws IOException{ + public void nonExistingFieldsCanBeIgnoredInPut() throws IOException { JsonReader r = createReader(inputJson( "{ ", " 'put': 'id:unittest:smoke::doc1',", @@ -1023,7 +1024,11 @@ public class JsonReaderTestCase { " }", " },", " 'field2': 'value2',", - " 'nonexisting3': 'ignored value'", + " 'nonexisting3': {", + " 'cells': [{'address': {'x': 'x1'}, 'value': 1.0}]", + " },", + " 'tensor1': {'cells': {'x1': 1.0}},", + " 'nonexisting4': 'ignored value'", " }", "}")); DocumentParseInfo parseInfo = r.parseDocument().get(); @@ -1036,6 +1041,8 @@ public class JsonReaderTestCase { assertNull(put.getDocument().getField("nonexisting2")); assertEquals("value2", put.getDocument().getFieldValue("field2").toString()); assertNull(put.getDocument().getField("nonexisting3")); + assertEquals(Tensor.from("tensor(x{}):{{x:x1}:1.0}"), put.getDocument().getFieldValue("tensor1").getWrappedValue()); + assertNull(put.getDocument().getField("nonexisting4")); } @Test @@ -1055,7 +1062,13 @@ public class JsonReaderTestCase { " }", " },", " 'field2': { 'assign': 'value2' },", - " 'nonexisting3': { 'assign': 'ignored value' }", + " 'nonexisting3': {", + " 'assign' : {", + " 'cells': [{'address': {'x': 'x1'}, 'value': 1.0}]", + " }", + " },", + " 'tensor1': {'assign': { 'cells': {'x1': 1.0} } },", + " 'nonexisting4': { 'assign': 'ignored value' }", " }", "}")); DocumentParseInfo parseInfo = r.parseDocument().get(); @@ -1068,6 +1081,8 @@ public class JsonReaderTestCase { assertNull(update.getFieldUpdate("nonexisting2")); assertEquals("value2", update.getFieldUpdate("field2").getValueUpdates().get(0).getValue().getWrappedValue().toString()); assertNull(update.getFieldUpdate("nonexisting3")); + assertEquals(Tensor.from("tensor(x{}):{{x:x1}:1.0}"), update.getFieldUpdate("tensor1").getValueUpdates().get(0).getValue().getWrappedValue()); + assertNull(update.getFieldUpdate("nonexisting4")); } @Test -- cgit v1.2.3