From 43a8b52eb70f6de9db64ee36a720a97ff1410e17 Mon Sep 17 00:00:00 2001 From: Haakon Dybdahl Date: Fri, 10 Feb 2017 09:12:15 +0100 Subject: Merge the two document parsers into one. --- .../java/com/yahoo/document/json/JsonReader.java | 15 +- .../java/com/yahoo/document/json/TokenBuffer.java | 2 +- .../document/json/document/DocumentParser.java | 183 ++++++++++----------- 3 files changed, 99 insertions(+), 101 deletions(-) (limited to 'document') diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 7034ff56dfb..91a07aa6c5f 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -25,7 +25,6 @@ import java.io.InputStream; import java.util.Optional; import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED; -import static com.yahoo.document.json.document.DocumentParser.parseDocumentsFields; import static com.yahoo.document.json.readers.AddRemoveCreator.createAdds; import static com.yahoo.document.json.readers.AddRemoveCreator.createRemoves; import static com.yahoo.document.json.readers.CompositeReader.populateComposite; @@ -41,14 +40,14 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate * valid JSON representation of a feed. * * @author Steinar Knutsen - * @since 5.1.25 + * @author dybis */ @Beta public class JsonReader { - // Only used for testing. public Optional parseDocument() throws IOException { - return DocumentParser.parseDocument(parser); + DocumentParser documentParser = new DocumentParser(parser); + return documentParser.parse(Optional.empty()); } private static final String UPDATE_REMOVE = "remove"; @@ -81,9 +80,10 @@ public class JsonReader { */ public DocumentOperation readSingleDocument(DocumentParser.SupportedOperation operationType, String docIdString) { DocumentId docId = new DocumentId(docIdString); - DocumentParseInfo documentParseInfo = null; + final DocumentParseInfo documentParseInfo; try { - documentParseInfo = parseDocumentsFields(parser, docId); + DocumentParser documentParser = new DocumentParser(parser); + documentParseInfo = documentParser.parse(Optional.of(docId)).get(); } catch (IOException e) { state = END_OF_FEED; throw new RuntimeException(e); @@ -107,8 +107,9 @@ public class JsonReader { break; } Optional documentParseInfo; + DocumentParser documentParser = new DocumentParser(parser); try { - documentParseInfo = DocumentParser.parseDocument(parser); + documentParseInfo = documentParser.parse(Optional.empty()); } catch (IOException r) { // Jackson is not able to recover from structural parse errors state = END_OF_FEED; diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 9f6a0ba3641..b551b4c6473 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -64,7 +64,7 @@ public class TokenBuffer { return buffer.peekFirst().text; } - int size() { + public int size() { return buffer.size(); } diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java index 8053246a266..b2713a1bc79 100644 --- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -9,8 +9,11 @@ import com.yahoo.document.json.readers.DocumentParseInfo; import java.io.IOException; import java.util.Optional; -import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; - +/** + * Parses a document operation. + * + * @author dybis + */ public class DocumentParser { public enum SupportedOperation { PUT, UPDATE, REMOVE @@ -22,56 +25,100 @@ public class DocumentParser { public static final String CREATE_IF_NON_EXISTENT = "create"; public static final String FIELDS = "fields"; public static final String REMOVE = "remove"; + private final JsonParser parser; + private long indentLevel; + + public DocumentParser(JsonParser parser) { + this.parser = parser; + } + + public Optional parse(Optional documentIdArg) throws IOException { + indentLevel = 0; + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + if (documentIdArg.isPresent()) { + documentParseInfo.documentId = documentIdArg.get(); + } + do { + parseOneItem(documentParseInfo, documentIdArg.isPresent() /* doc id set externally */); + } while (indentLevel > 0L); - public static Optional parseDocument(JsonParser parser) throws IOException { + if (documentParseInfo.documentId != null) { + return Optional.of(documentParseInfo); + } + return Optional.empty(); + } + + private void parseOneItem(DocumentParseInfo documentParseInfo, boolean docIdAndOperationIsSetExternally) throws IOException { // we should now be at the start of a feed operation or at the end of the feed - JsonToken token = parser.nextValue(); - if (token == JsonToken.END_ARRAY) { - return Optional.empty(); // end of feed + JsonToken currentToken = parser.nextValue(); + processIndent(currentToken); + if (parser.getCurrentName() == null) { + return; + } + if (indentLevel == 1L) { + handleIdentLevelOne(documentParseInfo, currentToken, docIdAndOperationIsSetExternally); + } else if (indentLevel == 2L) { + handleIdentLevelTwo(documentParseInfo, currentToken); } - expectObjectStart(token); + } - DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + private void processIndent(JsonToken currentToken) throws IOException { + if (currentToken == null) { + throw new IllegalArgumentException("Could not read document, no document?"); + } + switch (currentToken) { + case START_OBJECT: + indentLevel++; + break; + case END_OBJECT: + indentLevel--; + return; + case START_ARRAY: + indentLevel+=10000L; + break; + case END_ARRAY: + indentLevel-=10000L; + break; + } + } - while (true) { + private void handleIdentLevelOne( + DocumentParseInfo documentParseInfo, JsonToken currentToken, boolean docIdAndOperationIsSetExternally) + throws IOException { + if (currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) { try { - token = parser.nextValue(); - if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) && - CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { - documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE); - continue; - } - if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { - documentParseInfo.condition = Optional.of(parser.getText()); - continue; - } - if (token == JsonToken.START_OBJECT) { - try { - if (!FIELDS.equals(parser.getCurrentName())) { - throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName()); - } - } catch (IOException e) { - // TODO more specific wrapping - throw new RuntimeException(e); - } - documentParseInfo.fieldsBuffer.bufferObject(token, parser); - continue; + if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); + return; } - if (token == JsonToken.END_OBJECT) { - if (documentParseInfo.documentId == null) { - throw new RuntimeException("Did not find document operation"); - } - return Optional.of(documentParseInfo); - } - if (token == JsonToken.VALUE_STRING) { - documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); - documentParseInfo.documentId = new DocumentId(parser.getText()); - continue; - } - throw new RuntimeException("Expected document start or document operation."); } catch (IOException e) { - throw new IllegalStateException(e); + throw new RuntimeException("Got IO exception while parsing document", e); + } + } + if ((currentToken == JsonToken.VALUE_TRUE || currentToken == JsonToken.VALUE_FALSE) && + CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.of(currentToken == JsonToken.VALUE_TRUE); + } else if (currentToken == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { + documentParseInfo.condition = Optional.of(parser.getText()); + } else if (currentToken == JsonToken.VALUE_STRING) { + // Value is expected to be set in the header not in the document. Ignore any unknown field + // as well. + if (! docIdAndOperationIsSetExternally) { + documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); + documentParseInfo.documentId = new DocumentId(parser.getText()); + } + } + } + + private void handleIdentLevelTwo(DocumentParseInfo documentParseInfo, JsonToken currentToken) { + try { + // "Fields" opens a dictionary and is therefore on level two which might be surprising. + if (currentToken == JsonToken.START_OBJECT && FIELDS.equals(parser.getCurrentName())) { + documentParseInfo.fieldsBuffer.bufferObject(currentToken, parser); + indentLevel--; } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); } } @@ -90,54 +137,4 @@ public class DocumentParser { "\"remove\" and \"update\" are supported."); } } - - public static DocumentParseInfo parseDocumentsFields(JsonParser parser, DocumentId documentId) throws IOException { - long indentLevel = 0; - DocumentParseInfo documentParseInfo = new DocumentParseInfo(); - documentParseInfo.documentId = documentId; - while (true) { - // we should now be at the start of a feed operation or at the end of the feed - JsonToken t = parser.nextValue(); - if (t == null) { - throw new IllegalArgumentException("Could not read document, no document?"); - } - switch (t) { - case START_OBJECT: - indentLevel++; - break; - case END_OBJECT: - indentLevel--; - break; - case START_ARRAY: - indentLevel+=10000L; - break; - case END_ARRAY: - indentLevel-=10000L; - break; - } - if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) { - try { - if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { - documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); - continue; - } - } catch (IOException e) { - throw new RuntimeException("Got IO exception while parsing document", e); - } - } - if (indentLevel == 2L && t == JsonToken.START_OBJECT) { - - try { - if (!FIELDS.equals(parser.getCurrentName())) { - continue; - } - } catch (IOException e) { - throw new RuntimeException("Got IO exception while parsing document", e); - } - documentParseInfo.fieldsBuffer.bufferObject(t, parser); - break; - } - } - return documentParseInfo; - } } -- cgit v1.2.3