avoid reading extra byte after json value

3 files changed, 42 insertions, 0 deletions

diff --git a/vespalib/src/tests/slime/slime_json_format_test.cpp b/vespalib/src/tests/slime/slime_json_format_test.cpp
index 52e293f4a5e..313c7bd41c3 100644
--- a/vespalib/src/tests/slime/slime_json_format_test.cpp
+++ b/vespalib/src/tests/slime/slime_json_format_test.cpp
@@ -1,10 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/data/input.h>
+#include <vespa/vespalib/data/memory_input.h>
 #include <iostream>
 #include <fstream>
 
 using namespace vespalib::slime::convenience;
+using vespalib::Input;
+using vespalib::MemoryInput;
 
 std::string make_json(const Slime &slime, bool compact) {
     vespalib::SimpleBuffer buf;
@@ -64,6 +68,12 @@ std::string normalize(const std::string &json) {
     return make_json(slime, true);
 }
 
+std::string normalize(Input &input) {
+    Slime slime;
+    EXPECT_GREATER(vespalib::slime::JsonFormat::decode(input, slime), 0u);
+    return make_json(slime, true);
+}
+
 bool check_valid(const std::string &json) {
     Slime slime;
     return (vespalib::slime::JsonFormat::decode(json, slime) > 0);
@@ -359,4 +369,19 @@ TEST_F("decode bytes not null-terminated", Slime) {
     EXPECT_TRUE(parse_json_bytes(mem, f));
 }
 
+TEST("require that multiple adjacent values can be decoded from a single input") {
+    vespalib::string data("true{}false[]null\"foo\"'bar'1.5null");
+    MemoryInput input(data);
+    EXPECT_EQUAL(std::string("true"), normalize(input));
+    EXPECT_EQUAL(std::string("{}"), normalize(input));
+    EXPECT_EQUAL(std::string("false"), normalize(input));
+    EXPECT_EQUAL(std::string("[]"), normalize(input));
+    EXPECT_EQUAL(std::string("null"), normalize(input));
+    EXPECT_EQUAL(std::string("\"foo\""), normalize(input));
+    EXPECT_EQUAL(std::string("\"bar\""), normalize(input));
+    EXPECT_EQUAL(std::string("1.5"), normalize(input));
+    EXPECT_EQUAL(std::string("null"), normalize(input));
+    EXPECT_EQUAL(input.obtain().size, 0u);
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/vespa/vespalib/data/input_reader.h b/vespalib/src/vespa/vespalib/data/input_reader.h
index b3dfa0d4cd7..29f6b88b506 100644
--- a/vespalib/src/vespa/vespalib/data/input_reader.h
+++ b/vespalib/src/vespa/vespalib/data/input_reader.h
@@ -74,6 +74,22 @@ public:
     }
 
     /**
+     * Try to unread a single byte. This will work for data that is
+     * read, but not yet evicted. Note that after eof is found (the
+     * obtain function returns 0), unreading will not be possible.
+     *
+     * @return whether unreading could be performed
+     **/
+    bool try_unread() {
+        if (__builtin_expect(_pos > 0, true)) {
+            --_pos;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    /**
      * Read a continous sequence of bytes. Bytes within an input chunk
      * will be referenced directly. Reads crossing chunk boundries
      * will result in a gathering copy into a temporary buffer owned
diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp
index d6126667d68..6eb314141c1 100644
--- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp
+++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp
@@ -489,6 +489,7 @@ JsonFormat::decode(Input &input, Slime &slime)
     InputReader reader(input);
     JsonDecoder decoder(reader);
     decoder.decodeValue(slime);
+    reader.try_unread();
     if (reader.failed()) {
         slime.wrap("partial_result");
         slime.get().setLong("offending_offset", reader.get_offset());