Set end of feed on errors.

4 files changed, 51 insertions, 47 deletions

diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java
index 8c48be01799..7034ff56dfb 100644
--- a/document/src/main/java/com/yahoo/document/json/JsonReader.java
+++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java
@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.Optional;
 
+import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED;
 import static com.yahoo.document.json.document.DocumentParser.parseDocumentsFields;
 import static com.yahoo.document.json.readers.AddRemoveCreator.createAdds;
 import static com.yahoo.document.json.readers.AddRemoveCreator.createRemoves;
@@ -46,7 +47,7 @@ import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate
 public class JsonReader {
 
     // Only used for testing.
-    public Optional<DocumentParseInfo> parseDocument() {
+    public Optional<DocumentParseInfo> parseDocument() throws IOException {
         return DocumentParser.parseDocument(parser);
     }
 
@@ -67,7 +68,7 @@ public class JsonReader {
         try {
             parser = parserFactory.createParser(input);
         } catch (IOException e) {
-            state = ReaderState.END_OF_FEED;
+            state = END_OF_FEED;
             throw new RuntimeException(e);
         }
     }
@@ -80,7 +81,13 @@ public class JsonReader {
      */
     public DocumentOperation readSingleDocument(DocumentParser.SupportedOperation operationType, String docIdString) {
         DocumentId docId = new DocumentId(docIdString);
-        DocumentParseInfo documentParseInfo = parseDocumentsFields(parser, docId);
+        DocumentParseInfo documentParseInfo = null;
+        try {
+            documentParseInfo = parseDocumentsFields(parser, docId);
+        } catch (IOException e) {
+            state = END_OF_FEED;
+            throw new RuntimeException(e);
+        }
         documentParseInfo.operationType = operationType;
         DocumentOperation operation = createDocumentOperation(documentParseInfo.fieldsBuffer, documentParseInfo);
         operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition));
@@ -99,11 +106,16 @@ public class JsonReader {
             case READING:
                 break;
         }
-
-        Optional<DocumentParseInfo> documentParseInfo = DocumentParser.parseDocument(parser);
-
+        Optional<DocumentParseInfo> documentParseInfo;
+        try {
+            documentParseInfo = DocumentParser.parseDocument(parser);
+        } catch (IOException r) {
+            // Jackson is not able to recover from structural parse errors
+            state = END_OF_FEED;
+            throw new RuntimeException(r);
+        }
         if (! documentParseInfo.isPresent()) {
-            state = ReaderState.END_OF_FEED;
+            state = END_OF_FEED;
             return null;
         }
         DocumentOperation operation = createDocumentOperation(documentParseInfo.get().fieldsBuffer, documentParseInfo.get());
@@ -223,13 +235,12 @@ public class JsonReader {
         return docType;
     }
 
-    public static JsonToken nextToken(JsonParser parser) {
+    public JsonToken nextToken(JsonParser parser) {
         try {
             return parser.nextValue();
         } catch (IOException e) {
             // Jackson is not able to recover from structural parse errors
-            // TODO Do we really need to set state on exception?
-            // state = ReaderState.END_OF_FEED;
+            state = END_OF_FEED;
             throw new RuntimeException(e);
         }
     }
diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
index e464100a59d..8053246a266 100644
--- a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
+++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java
@@ -23,9 +23,9 @@ public class DocumentParser {
     public static final String FIELDS = "fields";
     public static final String REMOVE = "remove";
 
-    public static Optional<DocumentParseInfo> parseDocument(JsonParser parser) {
+    public static Optional<DocumentParseInfo> parseDocument(JsonParser parser) throws IOException {
         // we should now be at the start of a feed operation or at the end of the feed
-        JsonToken token = nextToken(parser);
+        JsonToken token = parser.nextValue();
         if (token == JsonToken.END_ARRAY) {
             return Optional.empty(); // end of feed
         }
@@ -35,7 +35,7 @@ public class DocumentParser {
 
         while (true) {
             try {
-                token = nextToken(parser);
+                token = parser.nextValue();
                 if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) &&
                         CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) {
                     documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE);
@@ -91,13 +91,13 @@ public class DocumentParser {
         }
     }
 
-    public static DocumentParseInfo parseDocumentsFields(JsonParser parser, DocumentId documentId) {
+    public static DocumentParseInfo parseDocumentsFields(JsonParser parser, DocumentId documentId) throws IOException {
         long indentLevel = 0;
         DocumentParseInfo documentParseInfo = new DocumentParseInfo();
         documentParseInfo.documentId = documentId;
         while (true) {
             // we should now be at the start of a feed operation or at the end of the feed
-            JsonToken t = nextToken(parser);
+            JsonToken t = parser.nextValue();
             if (t == null) {
                 throw new IllegalArgumentException("Could not read document, no document?");
             }
@@ -140,12 +140,4 @@ public class DocumentParser {
         }
         return documentParseInfo;
     }
-
-    private static JsonToken nextToken(JsonParser parser) {
-        try {
-            return parser.nextValue();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
 }
diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
index 30cb8db708c..756e3fe3aa9 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java
@@ -54,6 +54,7 @@ import org.junit.rules.ExpectedException;
 import org.mockito.internal.matchers.Contains;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
@@ -206,7 +207,7 @@ public class JsonReaderTestCase {
 
 
     @Test
-    public final void smokeTest() {
+    public final void smokeTest() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
                         + " \"fields\": { \"something\": \"smoketest\","
@@ -220,7 +221,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void docIdLookaheadTest() {
+    public final void docIdLookaheadTest() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{"
                         + " \"fields\": { \"something\": \"smoketest\","
@@ -237,7 +238,7 @@ public class JsonReaderTestCase {
 
 
     @Test
-    public final void emptyDocTest() {
+    public final void emptyDocTest() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
                         + " \"fields\": {}}"));
@@ -250,7 +251,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testStruct() {
+    public final void testStruct() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:mirrors::whee\","
                         + " \"fields\": { "
@@ -270,7 +271,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testUpdateArray() {
+    public final void testUpdateArray() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\","
                         + " \"fields\": { " + "\"actualarray\": {"
@@ -286,7 +287,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testUpdateWeighted() {
+    public final void testUpdateWeighted() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
                         + " \"fields\": { " + "\"actualset\": {"
@@ -315,7 +316,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testUpdateMatch() {
+    public final void testUpdateMatch() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
                         + " \"fields\": { " + "\"actualset\": {"
@@ -347,7 +348,7 @@ public class JsonReaderTestCase {
 
     @SuppressWarnings({ "cast", "unchecked", "rawtypes" })
     @Test
-    public final void testArithmeticOperators() {
+    public final void testArithmeticOperators() throws IOException {
         Tuple2[] operations = new Tuple2[] {
                 new Tuple2<String, Operator>(UPDATE_DECREMENT,
                         ArithmeticValueUpdate.Operator.SUB),
@@ -391,7 +392,7 @@ public class JsonReaderTestCase {
 
     @SuppressWarnings("rawtypes")
     @Test
-    public final void testArrayIndexing() {
+    public final void testArrayIndexing() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\","
                         + " \"fields\": { " + "\"actualarray\": {"
@@ -430,7 +431,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testWeightedSet() {
+    public final void testWeightedSet() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:testset::whee\","
                         + " \"fields\": { \"actualset\": {"
@@ -451,7 +452,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testArray() {
+    public final void testArray() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:testarray::whee\","
                         + " \"fields\": { \"actualarray\": ["
@@ -472,7 +473,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testMap() {
+    public final void testMap() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:testmap::whee\","
                         + " \"fields\": { \"actualmap\": ["
@@ -493,7 +494,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testPositionPositive() {
+    public final void testPositionPositive() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\","
                         + " \"fields\": { \"singlepos\": \"N63.429722;E10.393333\" }}"));
@@ -510,7 +511,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testPositionNegative() {
+    public final void testPositionNegative() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\","
                         + " \"fields\": { \"singlepos\": \"W46.63;S23.55\" }}"));
@@ -527,7 +528,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testRaw() {
+    public final void testRaw() throws IOException {
         String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest"))));
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:testraw::whee\","
@@ -549,7 +550,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testMapStringToArrayOfInt() {
+    public final void testMapStringToArrayOfInt() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:testMapStringToArrayOfInt::whee\","
                         + " \"fields\": { \"actualMapStringToArrayOfInt\": ["
@@ -572,7 +573,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testAssignToString() {
+    public final void testAssignToString() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\","
                         + " \"fields\": { \"something\": {"
@@ -589,7 +590,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testAssignToArray() {
+    public final void testAssignToArray() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"update\": \"id:unittest:testMapStringToArrayOfInt::whee\","
                         + " \"fields\": { \"actualMapStringToArrayOfInt\": {"
@@ -613,7 +614,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void testAssignToWeightedSet() {
+    public final void testAssignToWeightedSet() throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\","
                         + " \"fields\": { " + "\"actualset\": {"
@@ -816,7 +817,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void misspelledFieldTest() {
+    public final void misspelledFieldTest()  throws IOException{
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\","
                         + " \"fields\": { \"smething\": \"smoketest\","
@@ -845,7 +846,7 @@ public class JsonReaderTestCase {
     }
 
     @Test
-    public final void idAsAliasForPutTest() {
+    public final void idAsAliasForPutTest()  throws IOException{
         InputStream rawDoc = new ByteArrayInputStream(
                 Utf8.toBytes("{\"id\": \"id:unittest:smoke::whee\","
                         + " \"fields\": { \"something\": \"smoketest\","
diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
index 2ef6fad965e..76d18e71a51 100644
--- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
+++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java
@@ -291,7 +291,7 @@ public class JsonWriterTestCase {
         assertEquals(populateMap(inputMap), populateMap(generatedMap));
     }
 
-    private Document readDocumentFromJson(String docId, String fields) {
+    private Document readDocumentFromJson(String docId, String fields) throws IOException {
         InputStream rawDoc = new ByteArrayInputStream(asFeed(docId, fields));
 
 
@@ -393,7 +393,7 @@ public class JsonWriterTestCase {
     }
 
     @Test
-    public void non_empty_reference_field_results_in_reference_value_with_doc_id_present() {
+    public void non_empty_reference_field_results_in_reference_value_with_doc_id_present() throws IOException {
         final Document doc = readDocumentFromJson("id:unittest:testrefs::helloworld",
                 "{ \"ref_field\": \"id:unittest:smoke::and_mirrors_too\" }");
         ReferenceFieldValue ref = (ReferenceFieldValue)doc.getFieldValue("ref_field");
@@ -408,7 +408,7 @@ public class JsonWriterTestCase {
     }
 
     @Test
-    public void empty_reference_field_results_in_reference_value_without_doc_id_present() {
+    public void empty_reference_field_results_in_reference_value_without_doc_id_present() throws IOException {
         final Document doc = readDocumentFromJson("id:unittest:testrefs::helloworld", "{ \"ref_field\": \"\" }");
         ReferenceFieldValue ref = (ReferenceFieldValue)doc.getFieldValue("ref_field");
         assertFalse(ref.getDocumentId().isPresent());