From 769811b442f47ac9f13638b0d3ee0b0ecbf06b47 Mon Sep 17 00:00:00 2001 From: jonmv Date: Fri, 26 Jan 2024 09:49:12 +0100 Subject: Handle other fields in streaming document JSON parsing --- .../java/com/yahoo/document/json/JsonReader.java | 66 +++++++++++++++------- .../document/json/document/DocumentParser.java | 2 +- .../json/readers/VespaJsonDocumentReader.java | 2 +- .../yahoo/document/json/JsonReaderTestCase.java | 51 +++++++++++++++++ 4 files changed, 100 insertions(+), 21 deletions(-) (limited to 'document') diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 08d1fe688ed..adf876262fa 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -6,8 +6,10 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.json.document.DocumentParser; import com.yahoo.document.json.readers.DocumentParseInfo; @@ -18,6 +20,8 @@ import java.io.InputStream; import java.util.Optional; import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED; +import static com.yahoo.document.json.document.DocumentParser.CONDITION; +import static com.yahoo.document.json.document.DocumentParser.CREATE_IF_NON_EXISTENT; import static com.yahoo.document.json.document.DocumentParser.FIELDS; import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; @@ -80,7 +84,7 @@ public class JsonReader { } /** - * Reads a JSON which is expected to contain only the "fields" object of a document, + * Reads a JSON which is expected to contain a single document operation, * and where other parameters, like the document ID and operation type, are supplied by other means. * * @param operationType the type of operation (update or put) @@ -97,28 +101,52 @@ public class JsonReader { if (JsonToken.START_OBJECT != parser.nextValue()) throw new IllegalArgumentException("expected start of root object, got " + parser.currentToken()); - parser.nextValue(); - if ( ! FIELDS.equals(parser.getCurrentName())) - throw new IllegalArgumentException("expected field \"fields\", but got " + parser.getCurrentName()); + Boolean create = null; + String condition = null; + ParsedDocumentOperation operation = null; + while (JsonToken.END_OBJECT != parser.nextValue()) { + switch (parser.getCurrentName()) { + case FIELDS -> { + documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser); + VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); + operation = vespaJsonDocumentReader.createDocumentOperation( + getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo); + + if ( ! documentParseInfo.fieldsBuffer.isEmpty()) + throw new IllegalArgumentException("expected all content to be consumed by document parsing, but " + + documentParseInfo.fieldsBuffer.nesting() + " levels remain"); + + } + case CONDITION -> { + if ( ! JsonToken.VALUE_STRING.equals(parser.currentToken()) && ! JsonToken.VALUE_NULL.equals(parser.currentToken())) + throw new IllegalArgumentException("expected string value for condition, got " + parser.currentToken()); + + condition = parser.getValueAsString(); + } + case CREATE_IF_NON_EXISTENT -> { + create = parser.getBooleanValue(); // Throws if not boolean. + } + default -> { + // We ignore stray fields, but need to ensure structural balance in doing do. + if (parser.currentToken().isStructStart()) parser.skipChildren(); + } + } + } - if (JsonToken.START_OBJECT != parser.currentToken()) - throw new IllegalArgumentException("expected start of \"fields\" object, got " + parser.currentToken()); + if (null != parser.nextToken()) + throw new IllegalArgumentException("expected end of input, got " + parser.currentToken()); - documentParseInfo.fieldsBuffer = new LazyTokenBuffer(parser); - VespaJsonDocumentReader vespaJsonDocumentReader = new VespaJsonDocumentReader(typeManager.getIgnoreUndefinedFields()); - ParsedDocumentOperation operation = vespaJsonDocumentReader.createDocumentOperation( - getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager), documentParseInfo); + assert null != operation: "VespaDocumentReader should throw on missing fields"; - if ( ! documentParseInfo.fieldsBuffer.isEmpty()) - throw new IllegalArgumentException("expected all content to be consumed by document parsing, but " + - documentParseInfo.fieldsBuffer.nesting() + " levels remain"); + if (null != create) { + switch (operationType) { + case PUT -> ((DocumentPut) operation.operation()).setCreateIfNonExistent(create); + case UPDATE -> ((DocumentUpdate) operation.operation()).setCreateIfNonExistent(create); + case REMOVE -> throw new IllegalArgumentException(CREATE_IF_NON_EXISTENT + " is not supported for remove operations"); + } + } - if (JsonToken.END_OBJECT != parser.currentToken()) - throw new IllegalArgumentException("expected end of \"fields\" object, got " + parser.currentToken()); - if (JsonToken.END_OBJECT != parser.nextToken()) - throw new IllegalArgumentException("expected end of root object, got " + parser.currentToken()); - if (null != parser.nextToken()) - throw new IllegalArgumentException("expected end of input, got " + parser.currentToken()); + operation.operation().setCondition(TestAndSetCondition.fromConditionString(Optional.ofNullable(condition))); return operation; } diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java index aef7e1cffe2..77e11dcf2a8 100644 --- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -20,7 +20,7 @@ public class DocumentParser { private static final String UPDATE = "update"; private static final String PUT = "put"; private static final String ID = "id"; - private static final String CONDITION = "condition"; + public static final String CONDITION = "condition"; public static final String CREATE_IF_NON_EXISTENT = "create"; public static final String FIELDS = "fields"; public static final String REMOVE = "remove"; diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index c7303d31ea2..067dabdbdab 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -230,7 +230,7 @@ public class VespaJsonDocumentReader { private static boolean isFieldPath(String field) { - return field.matches("^.*?[.\\[\\{].*$"); + return field.matches("^.*?[.\\[{].*$"); } private static void verifyEndState(TokenBuffer buffer, JsonToken expectedFinalToken) { diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 080528fea77..12ccba62005 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -20,6 +20,7 @@ import com.yahoo.document.MapDataType; import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; +import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.BoolFieldValue; @@ -220,6 +221,56 @@ public class JsonReaderTestCase { return new JsonReader(types, input, parserFactory); } + @Test + public void readSingleDocumentsPutStreaming() throws IOException { + String json = """ + { + "remove": "id:unittest:smoke::ignored", + "ignored-extra-array": [{ "foo": null }, { }], + "ignored-extra-object": { "foo": [null, { }], "bar": { } }, + "fields": { + "something": "smoketest", + "flag": true, + "nalle": "bamse" + }, + "id": "id:unittest:smoke::ignored", + "create": false, + "condition": "true" + } + """; + ParsedDocumentOperation operation = createReader(json).readSingleDocumentStreaming(DocumentOperationType.PUT,"id:unittest:smoke::doc1"); + DocumentPut put = ((DocumentPut) operation.operation()); + assertFalse(put.getCreateIfNonExistent()); + assertEquals("true", put.getCondition().getSelection()); + smokeTestDoc(put.getDocument()); + } + + @Test + public void readSingleDocumentsUpdateStreaming() throws IOException { + String json = """ + { + "remove": "id:unittest:smoke::ignored", + "ignored-extra-array": [{ "foo": null }, { }], + "ignored-extra-object": { "foo": [null, { }], "bar": { } }, + "fields": { + "something": { "assign": "smoketest" }, + "flag": { "assign": true }, + "nalle": { "assign": "bamse" } + }, + "id": "id:unittest:smoke::ignored", + "create": true, + "condition": "false" + } + """; + ParsedDocumentOperation operation = createReader(json).readSingleDocumentStreaming(DocumentOperationType.UPDATE,"id:unittest:smoke::doc1"); + Document doc = new Document(types.getDocumentType("smoke"), new DocumentId("id:unittest:smoke::doc1")); + DocumentUpdate update = ((DocumentUpdate) operation.operation()); + update.applyTo(doc); + smokeTestDoc(doc); + assertTrue(update.getCreateIfNonExistent()); + assertEquals("false", update.getCondition().getSelection()); + } + @Test public void readSingleDocumentPut() throws IOException { Document doc = docFromJson(""" -- cgit v1.2.3