From a46ddef2850cbce4af3a499a4ef2da3f9720e17d Mon Sep 17 00:00:00 2001 From: Arne H Juul Date: Tue, 12 Oct 2021 06:07:06 +0000 Subject: add "data" format as JSON extension * when parsing JSON, allow binary data to be input as a hexdump prefixed with the letter "x"; this means that { foo: xFF0011 } will decode to an object where the "foo" field is of type DATA and contains the bytes 255, 0, and 17. --- .../src/tests/slime/slime_json_format_test.cpp | 39 +++++++++++- .../src/vespa/vespalib/data/slime/json_format.cpp | 71 ++++++++++++++++------ 2 files changed, 90 insertions(+), 20 deletions(-) (limited to 'vespalib/src') diff --git a/vespalib/src/tests/slime/slime_json_format_test.cpp b/vespalib/src/tests/slime/slime_json_format_test.cpp index 9e1040b36ad..05dca9112d6 100644 --- a/vespalib/src/tests/slime/slime_json_format_test.cpp +++ b/vespalib/src/tests/slime/slime_json_format_test.cpp @@ -278,6 +278,36 @@ TEST("decode string") { EXPECT_EQUAL(std::string("\xf4\x8f\xbf\xbf"), json_string("\\udbff\\udfff")); } +TEST_F("decode data", Slime) { + EXPECT_TRUE(parse_json("x", f)); + EXPECT_EQUAL(vespalib::slime::DATA::ID, f.get().type().getId()); + Memory m = f.get().asData(); + EXPECT_EQUAL(0u, m.size); + + EXPECT_TRUE(parse_json("x0000", f)); + EXPECT_EQUAL(vespalib::slime::DATA::ID, f.get().type().getId()); + m = f.get().asData(); + EXPECT_EQUAL(2u, m.size); + EXPECT_EQUAL(0, m.data[0]); + EXPECT_EQUAL(0, m.data[1]); + + EXPECT_TRUE(parse_json("x1234567890abcdefABCDEF", f)); + EXPECT_EQUAL(vespalib::slime::DATA::ID, f.get().type().getId()); + m = f.get().asData(); + EXPECT_EQUAL(11u, m.size); + EXPECT_EQUAL((char)0x12, m.data[0]); + EXPECT_EQUAL((char)0x34, m.data[1]); + EXPECT_EQUAL((char)0x56, m.data[2]); + EXPECT_EQUAL((char)0x78, m.data[3]); + EXPECT_EQUAL((char)0x90, m.data[4]); + EXPECT_EQUAL((char)0xAB, m.data[5]); + EXPECT_EQUAL((char)0xCD, m.data[6]); + EXPECT_EQUAL((char)0xEF, m.data[7]); + EXPECT_EQUAL((char)0xAB, m.data[8]); + EXPECT_EQUAL((char)0xCD, m.data[9]); + EXPECT_EQUAL((char)0xEF, m.data[10]); +} + TEST_F("decode empty array", Slime) { EXPECT_TRUE(parse_json("[]", f)); EXPECT_EQUAL(vespalib::slime::ARRAY::ID, f.get().type().getId()); @@ -301,13 +331,18 @@ TEST_F("decode array", Slime) { } TEST_F("decode object", Slime) { - EXPECT_TRUE(parse_json("{\"a\":123,\"b\":0.5,\"c\":\"foo\",\"d\":true}", f)); + EXPECT_TRUE(parse_json("{\"a\":123,\"b\":0.5,\"c\":\"foo\",\"d\":true,\"e\":xff0011}", f)); EXPECT_EQUAL(vespalib::slime::OBJECT::ID, f.get().type().getId()); - EXPECT_EQUAL(4u, f.get().children()); + EXPECT_EQUAL(5u, f.get().children()); EXPECT_EQUAL(123.0, f.get()["a"].asDouble()); EXPECT_EQUAL(0.5, f.get()["b"].asDouble()); EXPECT_EQUAL(std::string("foo"), f.get()["c"].asString().make_string()); EXPECT_EQUAL(true, f.get()["d"].asBool()); + Memory m = f.get()["e"].asData(); + EXPECT_EQUAL(3u, m.size); + EXPECT_EQUAL((char)255, m.data[0]); + EXPECT_EQUAL((char)0, m.data[1]); + EXPECT_EQUAL((char)17, m.data[2]); } TEST_F("decode nesting", Slime) { diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp index b0c66d4cb88..cf01aedb8df 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp @@ -17,6 +17,29 @@ namespace vespalib::slime { namespace { +inline int fromHexDigit(char digit) { + switch(digit) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': case 'A': return 0xA; + case 'b': case 'B': return 0xB; + case 'c': case 'C': return 0xC; + case 'd': case 'D': return 0xD; + case 'e': case 'E': return 0xE; + case 'f': case 'F': return 0xF; + default: + return -1; + } +} + template struct JsonEncoder : public ArrayTraverser, public ObjectTraverser @@ -223,6 +246,7 @@ struct JsonDecoder { void decodeObject(Inserter &inserter); void decodeArray(Inserter &inserter); void decodeNumber(Inserter &inserter); + void decodeData(Inserter &inserter); void decodeValue(Inserter &inserter) { skipWhiteSpace(); switch (c) { @@ -232,6 +256,7 @@ struct JsonDecoder { case 't': expect("true"); inserter.insertBool(true); return; case 'f': expect("false"); inserter.insertBool(false); return; case 'n': expect("null"); inserter.insertNix(); return; + case 'x': return decodeData(inserter); case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return decodeNumber(inserter); } @@ -249,27 +274,12 @@ JsonDecoder::readHexValue(uint32_t len) { uint32_t ret = 0; for (uint32_t i = 0; i < len; ++i) { - switch (c) { - case '0': ret = (ret << 4) | 0; break; - case '1': ret = (ret << 4) | 1; break; - case '2': ret = (ret << 4) | 2; break; - case '3': ret = (ret << 4) | 3; break; - case '4': ret = (ret << 4) | 4; break; - case '5': ret = (ret << 4) | 5; break; - case '6': ret = (ret << 4) | 6; break; - case '7': ret = (ret << 4) | 7; break; - case '8': ret = (ret << 4) | 8; break; - case '9': ret = (ret << 4) | 9; break; - case 'a': case 'A': ret = (ret << 4) | 0xa; break; - case 'b': case 'B': ret = (ret << 4) | 0xb; break; - case 'c': case 'C': ret = (ret << 4) | 0xc; break; - case 'd': case 'D': ret = (ret << 4) | 0xd; break; - case 'e': case 'E': ret = (ret << 4) | 0xe; break; - case 'f': case 'F': ret = (ret << 4) | 0xf; break; - default: + int d = fromHexDigit(c); + if (d < 0) { in.fail("invalid hex character"); return 0; } + ret = (ret << 4) | d; next(); } return ret; @@ -413,6 +423,31 @@ JsonDecoder::decodeArray(Inserter &inserter) expect("]"); } +void JsonDecoder::decodeData(Inserter &inserter) { + vespalib::string data; + value.clear(); + for (;;) { + next(); + value.push_back(c); + int hi = fromHexDigit(c); + if (hi < 0) { + inserter.insertData(data); + return; + } + next(); + value.push_back(c); + int lo = fromHexDigit(c); + if (lo < 0) { + std::stringstream ss; + ss << "error inserting data '" << value << "'. invalid hex dump."; + in.fail(ss.str()); + return; + } + char byte = (hi << 4) | lo; + data.push_back(byte); + } +} + static int insertNumber(Inserter &inserter, bool isLong, const vespalib::string &value, char **endp); void -- cgit v1.2.3