allow a string (with a hex dump of binary representation) as cell values

1 files changed, 125 insertions, 0 deletions

diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
index 3ca20661587..011c4b1fe12 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
@@ -82,6 +82,131 @@ public class JsonFormatTestCase {
     }
 
     @Test
+    public void testInt8VectorInHexForm() {
+        Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor<int8>(x[2],y[3])"));
+        builder.cell().label("x", 0).label("y", 0).value(2.0);
+        builder.cell().label("x", 0).label("y", 1).value(127.0);
+        builder.cell().label("x", 0).label("y", 2).value(-1.0);
+        builder.cell().label("x", 1).label("y", 0).value(-128.0);
+        builder.cell().label("x", 1).label("y", 1).value(0.0);
+        builder.cell().label("x", 1).label("y", 2).value(42.0);
+        Tensor expected = builder.build();
+        String denseJson = "{\"values\":\"027FFF80002A\"}";
+        Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8));
+        assertEquals(expected, decoded);
+    }
+
+    @Test
+    public void testInt8VectorInvalidHex() {
+        var type = TensorType.fromSpec("tensor<int8>(x[2])");
+        String denseJson = "{\"values\":\"abXc\"}";
+        try {
+            Tensor decoded = JsonFormat.decode(type, denseJson.getBytes(StandardCharsets.UTF_8));
+            fail("did not get exception as expected, decoded as: "+decoded);
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "Invalid digit 'X' at index 2 in input abXc");
+        }
+    }
+
+    @Test
+    public void testMixedInt8TensorWithHexForm() {
+        Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor<int8>(x{},y[3])"));
+        builder.cell().label("x", 0).label("y", 0).value(2.0);
+        builder.cell().label("x", 0).label("y", 1).value(3.0);
+        builder.cell().label("x", 0).label("y", 2).value(4.0);
+        builder.cell().label("x", 1).label("y", 0).value(5.0);
+        builder.cell().label("x", 1).label("y", 1).value(6.0);
+        builder.cell().label("x", 1).label("y", 2).value(7.0);
+        Tensor expected = builder.build();
+        String mixedJson = "{\"blocks\":[" +
+                           "{\"address\":{\"x\":\"0\"},\"values\":\"020304\"}," +
+                           "{\"address\":{\"x\":\"1\"},\"values\":\"050607\"}" +
+                           "]}";
+        Tensor decoded = JsonFormat.decode(expected.type(), mixedJson.getBytes(StandardCharsets.UTF_8));
+        assertEquals(expected, decoded);
+    }
+
+    @Test
+    public void testBFloat16VectorInHexForm() {
+        var builder = Tensor.Builder.of(TensorType.fromSpec("tensor<bfloat16>(x[3],y[4])"));
+        builder.cell().label("x", 0).label("y", 0).value(42.0);
+        builder.cell().label("x", 0).label("y", 1).value(1048576.0);
+        builder.cell().label("x", 0).label("y", 2).value(0.00000095367431640625);
+        builder.cell().label("x", 0).label("y", 3).value(-255.00);
+
+        builder.cell().label("x", 1).label("y", 0).value(0.0);
+        builder.cell().label("x", 1).label("y", 1).value(-0.0);
+        builder.cell().label("x", 1).label("y", 2).value(Float.MIN_NORMAL);
+        builder.cell().label("x", 1).label("y", 3).value(0x1.feP+127);
+
+        builder.cell().label("x", 2).label("y", 0).value(Float.POSITIVE_INFINITY);
+        builder.cell().label("x", 2).label("y", 1).value(Float.NEGATIVE_INFINITY);
+        builder.cell().label("x", 2).label("y", 2).value(Float.NaN);
+        builder.cell().label("x", 2).label("y", 3).value(-Float.NaN);
+        Tensor expected = builder.build();
+
+        String denseJson = "{\"values\":\"422849803580c37f0000800000807f7f7f80ff807fc0ffc0\"}";
+        Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8));
+        assertEquals(expected, decoded);
+    }
+
+    @Test
+    public void testFloatVectorInHexForm() {
+        var builder = Tensor.Builder.of(TensorType.fromSpec("tensor<float>(x[3],y[4])"));
+        builder.cell().label("x", 0).label("y", 0).value(42.0);
+        builder.cell().label("x", 0).label("y", 1).value(1048577.0);
+        builder.cell().label("x", 0).label("y", 2).value(0.00000095367431640625);
+        builder.cell().label("x", 0).label("y", 3).value(-255.00);
+
+        builder.cell().label("x", 1).label("y", 0).value(0.0);
+        builder.cell().label("x", 1).label("y", 1).value(-0.0);
+        builder.cell().label("x", 1).label("y", 2).value(Float.MIN_VALUE);
+        builder.cell().label("x", 1).label("y", 3).value(Float.MAX_VALUE);
+
+        builder.cell().label("x", 2).label("y", 0).value(Float.POSITIVE_INFINITY);
+        builder.cell().label("x", 2).label("y", 1).value(Float.NEGATIVE_INFINITY);
+        builder.cell().label("x", 2).label("y", 2).value(Float.NaN);
+        builder.cell().label("x", 2).label("y", 3).value(-Float.NaN);
+        Tensor expected = builder.build();
+
+        String denseJson = "{\"values\":\""
+            +"42280000"+"49800008"+"35800000"+"c37f0000"
+            +"00000000"+"80000000"+"00000001"+"7f7fffff"
+            +"7f800000"+"ff800000"+"7fc00000"+"ffc00000"
+            +"\"}";
+        Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8));
+        assertEquals(expected, decoded);
+    }
+
+    @Test
+    public void testDoubleVectorInHexForm() {
+        var builder = Tensor.Builder.of(TensorType.fromSpec("tensor<double>(x[3],y[4])"));
+        builder.cell().label("x", 0).label("y", 0).value(42.0);
+        builder.cell().label("x", 0).label("y", 1).value(1048577.0);
+        builder.cell().label("x", 0).label("y", 2).value(0.00000095367431640625);
+        builder.cell().label("x", 0).label("y", 3).value(-255.00);
+
+        builder.cell().label("x", 1).label("y", 0).value(0.0);
+        builder.cell().label("x", 1).label("y", 1).value(-0.0);
+        builder.cell().label("x", 1).label("y", 2).value(Double.MIN_VALUE);
+        builder.cell().label("x", 1).label("y", 3).value(Double.MAX_VALUE);
+
+        builder.cell().label("x", 2).label("y", 0).value(Double.POSITIVE_INFINITY);
+        builder.cell().label("x", 2).label("y", 1).value(Double.NEGATIVE_INFINITY);
+        builder.cell().label("x", 2).label("y", 2).value(Double.NaN);
+        builder.cell().label("x", 2).label("y", 3).value(-Double.NaN);
+        Tensor expected = builder.build();
+
+        String denseJson = "{\"values\":\""
+            +"4045000000000000"+"4130000100000000"+"3eb0000000000000"+"c06fe00000000000"
+            +"0000000000000000"+"8000000000000000"+"0000000000000001"+"7fefffffffffffff"
+            +"7ff0000000000000"+"fff0000000000000"+"7ff8000000000000"+"fff8000000000000"
+            +"\"}";
+        Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8));
+        assertEquals(expected, decoded);
+    }
+
+    @Test
     public void testMixedTensorInMixedForm() {
         Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y[3])"));
         builder.cell().label("x", 0).label("y", 0).value(2.0);