add "data" format as JSON extension

* when parsing JSON, allow binary data to be input as a
  hexdump prefixed with the letter "x"; this means that
  { foo: xFF0011 }
  will decode to an object where the "foo" field is of type DATA
  and contains the bytes 255, 0, and 17.

2 files changed, 90 insertions, 20 deletions

diff --git a/vespalib/src/tests/slime/slime_json_format_test.cpp b/vespalib/src/tests/slime/slime_json_format_test.cpp
index 9e1040b36ad..05dca9112d6 100644
--- a/vespalib/src/tests/slime/slime_json_format_test.cpp
+++ b/vespalib/src/tests/slime/slime_json_format_test.cpp
@@ -278,6 +278,36 @@ TEST("decode string") {
     EXPECT_EQUAL(std::string("\xf4\x8f\xbf\xbf"), json_string("\\udbff\\udfff"));
 }
 
+TEST_F("decode data", Slime) {
+    EXPECT_TRUE(parse_json("x", f));
+    EXPECT_EQUAL(vespalib::slime::DATA::ID, f.get().type().getId());
+    Memory m = f.get().asData();
+    EXPECT_EQUAL(0u, m.size);
+
+    EXPECT_TRUE(parse_json("x0000", f));
+    EXPECT_EQUAL(vespalib::slime::DATA::ID, f.get().type().getId());
+    m = f.get().asData();
+    EXPECT_EQUAL(2u, m.size);
+    EXPECT_EQUAL(0, m.data[0]);
+    EXPECT_EQUAL(0, m.data[1]);
+
+    EXPECT_TRUE(parse_json("x1234567890abcdefABCDEF", f));
+    EXPECT_EQUAL(vespalib::slime::DATA::ID, f.get().type().getId());
+    m = f.get().asData();
+    EXPECT_EQUAL(11u, m.size);
+    EXPECT_EQUAL((char)0x12, m.data[0]);
+    EXPECT_EQUAL((char)0x34, m.data[1]);
+    EXPECT_EQUAL((char)0x56, m.data[2]);
+    EXPECT_EQUAL((char)0x78, m.data[3]);
+    EXPECT_EQUAL((char)0x90, m.data[4]);
+    EXPECT_EQUAL((char)0xAB, m.data[5]);
+    EXPECT_EQUAL((char)0xCD, m.data[6]);
+    EXPECT_EQUAL((char)0xEF, m.data[7]);
+    EXPECT_EQUAL((char)0xAB, m.data[8]);
+    EXPECT_EQUAL((char)0xCD, m.data[9]);
+    EXPECT_EQUAL((char)0xEF, m.data[10]);
+}
+
 TEST_F("decode empty array", Slime) {
     EXPECT_TRUE(parse_json("[]", f));
     EXPECT_EQUAL(vespalib::slime::ARRAY::ID, f.get().type().getId());
@@ -301,13 +331,18 @@ TEST_F("decode array", Slime) {
 }
 
 TEST_F("decode object", Slime) {
-    EXPECT_TRUE(parse_json("{\"a\":123,\"b\":0.5,\"c\":\"foo\",\"d\":true}", f));
+    EXPECT_TRUE(parse_json("{\"a\":123,\"b\":0.5,\"c\":\"foo\",\"d\":true,\"e\":xff0011}", f));
     EXPECT_EQUAL(vespalib::slime::OBJECT::ID, f.get().type().getId());
-    EXPECT_EQUAL(4u, f.get().children());
+    EXPECT_EQUAL(5u, f.get().children());
     EXPECT_EQUAL(123.0, f.get()["a"].asDouble());
     EXPECT_EQUAL(0.5, f.get()["b"].asDouble());
     EXPECT_EQUAL(std::string("foo"), f.get()["c"].asString().make_string());
     EXPECT_EQUAL(true, f.get()["d"].asBool());
+    Memory m = f.get()["e"].asData();
+    EXPECT_EQUAL(3u, m.size);
+    EXPECT_EQUAL((char)255, m.data[0]);
+    EXPECT_EQUAL((char)0,   m.data[1]);
+    EXPECT_EQUAL((char)17,  m.data[2]);
 }
 
 TEST_F("decode nesting", Slime) {
diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp
index b0c66d4cb88..cf01aedb8df 100644
--- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp
+++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp
@@ -17,6 +17,29 @@ namespace vespalib::slime {
 
 namespace {
 
+inline int fromHexDigit(char digit) {
+    switch(digit) {
+    case '0': return 0;
+    case '1': return 1;
+    case '2': return 2;
+    case '3': return 3; 
+    case '4': return 4; 
+    case '5': return 5; 
+    case '6': return 6; 
+    case '7': return 7; 
+    case '8': return 8; 
+    case '9': return 9; 
+    case 'a': case 'A': return 0xA; 
+    case 'b': case 'B': return 0xB; 
+    case 'c': case 'C': return 0xC; 
+    case 'd': case 'D': return 0xD; 
+    case 'e': case 'E': return 0xE; 
+    case 'f': case 'F': return 0xF; 
+    default:
+        return -1;
+    }
+}
+
 template <bool COMPACT>
 struct JsonEncoder : public ArrayTraverser,
                      public ObjectTraverser
@@ -223,6 +246,7 @@ struct JsonDecoder {
     void decodeObject(Inserter &inserter);
     void decodeArray(Inserter &inserter);
     void decodeNumber(Inserter &inserter);
+    void decodeData(Inserter &inserter);
     void decodeValue(Inserter &inserter) {
         skipWhiteSpace();
         switch (c) {
@@ -232,6 +256,7 @@ struct JsonDecoder {
         case 't': expect("true"); inserter.insertBool(true); return;
         case 'f': expect("false"); inserter.insertBool(false); return;
         case 'n': expect("null"); inserter.insertNix(); return;
+        case 'x': return decodeData(inserter);
         case '-': case '0': case '1': case '2': case '3': case '4': case '5':
         case '6': case '7': case '8': case '9': return decodeNumber(inserter);
         }
@@ -249,27 +274,12 @@ JsonDecoder::readHexValue(uint32_t len)
 {
     uint32_t ret = 0;
     for (uint32_t i = 0; i < len; ++i) {
-        switch (c) {
-        case '0': ret = (ret << 4) | 0; break;
-        case '1': ret = (ret << 4) | 1; break;
-        case '2': ret = (ret << 4) | 2; break;
-        case '3': ret = (ret << 4) | 3; break;
-        case '4': ret = (ret << 4) | 4; break;
-        case '5': ret = (ret << 4) | 5; break;
-        case '6': ret = (ret << 4) | 6; break;
-        case '7': ret = (ret << 4) | 7; break;
-        case '8': ret = (ret << 4) | 8; break;
-        case '9': ret = (ret << 4) | 9; break;
-        case 'a': case 'A': ret = (ret << 4) | 0xa; break;
-        case 'b': case 'B': ret = (ret << 4) | 0xb; break;
-        case 'c': case 'C': ret = (ret << 4) | 0xc; break;
-        case 'd': case 'D': ret = (ret << 4) | 0xd; break;
-        case 'e': case 'E': ret = (ret << 4) | 0xe; break;
-        case 'f': case 'F': ret = (ret << 4) | 0xf; break;
-        default:
+        int d = fromHexDigit(c);
+        if (d < 0) {
             in.fail("invalid hex character");
             return 0;
         }
+        ret = (ret << 4) | d;
         next();
     }
     return ret;
@@ -413,6 +423,31 @@ JsonDecoder::decodeArray(Inserter &inserter)
     expect("]");
 }
 
+void JsonDecoder::decodeData(Inserter &inserter) {
+    vespalib::string data;
+    value.clear();
+    for (;;) {
+        next();
+        value.push_back(c);
+        int hi = fromHexDigit(c);
+        if (hi < 0) {
+            inserter.insertData(data);
+            return;
+        }
+        next();
+        value.push_back(c);
+        int lo = fromHexDigit(c);
+        if (lo < 0) {
+            std::stringstream ss;
+            ss << "error inserting data '" << value << "'. invalid hex dump.";
+            in.fail(ss.str());
+            return;
+        }
+        char byte = (hi << 4) | lo;
+        data.push_back(byte);
+    }
+}
+
 static int insertNumber(Inserter &inserter, bool isLong, const vespalib::string &value, char **endp);
 
 void