diff options
author | Valerij Fredriksen <valerij92@gmail.com> | 2018-10-30 22:10:07 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerij92@gmail.com> | 2018-10-30 22:10:07 +0100 |
commit | 878a506472a2617390f8d117b51267f0ee44cfce (patch) | |
tree | 764abbf0bbcf483aab3c8016871066b516f71532 /vespa-http-client | |
parent | ee101cda2bd471d66540bcb9f129dd2b0fa6d113 (diff) |
Throw if document ID is missing
Diffstat (limited to 'vespa-http-client')
-rw-r--r-- | vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java | 35 | ||||
-rw-r--r-- | vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java | 7 |
2 files changed, 30 insertions, 12 deletions
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java index f5db2af216d..d10480eb183 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java @@ -5,9 +5,11 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.yahoo.vespa.http.client.FeedClient; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.concurrent.atomic.AtomicInteger; @@ -34,26 +36,34 @@ public class JsonReader { * @param numSent counter to be incremented for every document streamed. */ public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { - try { - final InputStreamJsonElementBuffer jsonElementBuffer = new InputStreamJsonElementBuffer(inputStream); + try (final InputStreamJsonElementBuffer jsonElementBuffer = new InputStreamJsonElementBuffer(inputStream)) { final JsonFactory jfactory = new JsonFactory().disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES); final JsonParser jParser = jfactory.createParser(jsonElementBuffer); while (true) { + int documentStart = (int) jParser.getCurrentLocation().getCharOffset(); String docId = parseOneDocument(jParser); if (docId == null) { - break; + int documentEnd = (int) jParser.getCurrentLocation().getCharOffset(); + int documentLength = documentEnd - documentStart; + int maxTruncatedLength = 500; + StringBuilder stringBuilder = new StringBuilder(maxTruncatedLength + 3); + for (int i = 0; i < Math.min(documentLength, maxTruncatedLength); i++) + stringBuilder.append(jsonElementBuffer.circular.get(documentStart + i)); + + if (documentLength > maxTruncatedLength) + stringBuilder.append("..."); + + throw new IllegalArgumentException("Document is missing ID: '" + stringBuilder.toString() + "'"); } CharSequence data = jsonElementBuffer.getJsonAsArray(jParser.getCurrentLocation().getCharOffset()); - if (data == null) { - continue; - } feedClient.stream(docId, data); numSent.incrementAndGet(); } - jsonElementBuffer.close(); + } catch (EOFException ignored) { + // No more documents } catch (IOException ioe) { System.err.println(ioe.getMessage()); - throw new RuntimeException(ioe); + throw new UncheckedIOException(ioe); } } @@ -85,7 +95,7 @@ public class JsonReader { /** * This is for throwing away [ and spaces in front of a json object, and find the position of {. * Not for parsing much text. - * @return posisiton for { + * @return position for { */ public int findNextObjectStart() { int readerPos = 0; @@ -145,9 +155,6 @@ public class JsonReader { */ public CharSequence getJsonAsArray(long parserPosition) throws IOException { final int charSize = (int)parserPosition - processedChars; - if (charSize<2) { - return null; - } final int endPosOfJson = circular.findLastObjectEnd(charSize); final int startPosOfJson = circular.findNextObjectStart(); processedChars += charSize; @@ -192,6 +199,7 @@ public class JsonReader { private static String parseOneDocument(JsonParser jParser) throws IOException { int objectLevel = 0; String documentId = null; + boolean foundObject = false; boolean valueIsDocumentId = false; while (jParser.nextToken() != null) { final String tokenAsText = jParser.getText(); @@ -204,6 +212,7 @@ public class JsonReader { } switch(jParser.getCurrentToken()) { case START_OBJECT: + foundObject = true; objectLevel++; break; case END_OBJECT: @@ -224,6 +233,8 @@ public class JsonReader { default: // No operation on all other tags. } } + if (!foundObject) + throw new EOFException("No more documents"); return null; } } diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java index f1c4a46dffc..d8dfae00b85 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java @@ -205,6 +205,13 @@ public class JsonReaderTest { JsonReader.read(inputStream, session, numSent); } + @Test(expected = IllegalArgumentException.class) + public void throwsOnMissingId() { + InputStream inputStream = new ByteArrayInputStream( + inputJson("[{'fields':{ 'something': 'smoketest', 'nalle': 'bamse' }}]").getBytes(StandardCharsets.UTF_8)); + JsonReader.read(inputStream, session, numSent); + } + @Test public void testFullDocument() throws Exception { InputStream inputStream = new ByteArrayInputStream(( |