diff options
author | Arne Juul <arnej@verizonmedia.com> | 2021-04-29 11:12:38 +0000 |
---|---|---|
committer | Arne Juul <arnej@verizonmedia.com> | 2021-04-29 11:12:38 +0000 |
commit | 66e49b1201f9574bd42b208a56969db87716f2db (patch) | |
tree | 17f71449562b09096537346b52ce210e48bbe33e /vespajlib/src/test | |
parent | 126d4f78c4464c79f3365c433890119306693102 (diff) |
allow a string (with a hex dump of binary representation) as cell values
Diffstat (limited to 'vespajlib/src/test')
-rw-r--r-- | vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java | 125 |
1 files changed, 125 insertions, 0 deletions
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java index 3ca20661587..011c4b1fe12 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java @@ -82,6 +82,131 @@ public class JsonFormatTestCase { } @Test + public void testInt8VectorInHexForm() { + Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor<int8>(x[2],y[3])")); + builder.cell().label("x", 0).label("y", 0).value(2.0); + builder.cell().label("x", 0).label("y", 1).value(127.0); + builder.cell().label("x", 0).label("y", 2).value(-1.0); + builder.cell().label("x", 1).label("y", 0).value(-128.0); + builder.cell().label("x", 1).label("y", 1).value(0.0); + builder.cell().label("x", 1).label("y", 2).value(42.0); + Tensor expected = builder.build(); + String denseJson = "{\"values\":\"027FFF80002A\"}"; + Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); + } + + @Test + public void testInt8VectorInvalidHex() { + var type = TensorType.fromSpec("tensor<int8>(x[2])"); + String denseJson = "{\"values\":\"abXc\"}"; + try { + Tensor decoded = JsonFormat.decode(type, denseJson.getBytes(StandardCharsets.UTF_8)); + fail("did not get exception as expected, decoded as: "+decoded); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Invalid digit 'X' at index 2 in input abXc"); + } + } + + @Test + public void testMixedInt8TensorWithHexForm() { + Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor<int8>(x{},y[3])")); + builder.cell().label("x", 0).label("y", 0).value(2.0); + builder.cell().label("x", 0).label("y", 1).value(3.0); + builder.cell().label("x", 0).label("y", 2).value(4.0); + builder.cell().label("x", 1).label("y", 0).value(5.0); + builder.cell().label("x", 1).label("y", 1).value(6.0); + builder.cell().label("x", 1).label("y", 2).value(7.0); + Tensor expected = builder.build(); + String mixedJson = "{\"blocks\":[" + + "{\"address\":{\"x\":\"0\"},\"values\":\"020304\"}," + + "{\"address\":{\"x\":\"1\"},\"values\":\"050607\"}" + + "]}"; + Tensor decoded = JsonFormat.decode(expected.type(), mixedJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); + } + + @Test + public void testBFloat16VectorInHexForm() { + var builder = Tensor.Builder.of(TensorType.fromSpec("tensor<bfloat16>(x[3],y[4])")); + builder.cell().label("x", 0).label("y", 0).value(42.0); + builder.cell().label("x", 0).label("y", 1).value(1048576.0); + builder.cell().label("x", 0).label("y", 2).value(0.00000095367431640625); + builder.cell().label("x", 0).label("y", 3).value(-255.00); + + builder.cell().label("x", 1).label("y", 0).value(0.0); + builder.cell().label("x", 1).label("y", 1).value(-0.0); + builder.cell().label("x", 1).label("y", 2).value(Float.MIN_NORMAL); + builder.cell().label("x", 1).label("y", 3).value(0x1.feP+127); + + builder.cell().label("x", 2).label("y", 0).value(Float.POSITIVE_INFINITY); + builder.cell().label("x", 2).label("y", 1).value(Float.NEGATIVE_INFINITY); + builder.cell().label("x", 2).label("y", 2).value(Float.NaN); + builder.cell().label("x", 2).label("y", 3).value(-Float.NaN); + Tensor expected = builder.build(); + + String denseJson = "{\"values\":\"422849803580c37f0000800000807f7f7f80ff807fc0ffc0\"}"; + Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); + } + + @Test + public void testFloatVectorInHexForm() { + var builder = Tensor.Builder.of(TensorType.fromSpec("tensor<float>(x[3],y[4])")); + builder.cell().label("x", 0).label("y", 0).value(42.0); + builder.cell().label("x", 0).label("y", 1).value(1048577.0); + builder.cell().label("x", 0).label("y", 2).value(0.00000095367431640625); + builder.cell().label("x", 0).label("y", 3).value(-255.00); + + builder.cell().label("x", 1).label("y", 0).value(0.0); + builder.cell().label("x", 1).label("y", 1).value(-0.0); + builder.cell().label("x", 1).label("y", 2).value(Float.MIN_VALUE); + builder.cell().label("x", 1).label("y", 3).value(Float.MAX_VALUE); + + builder.cell().label("x", 2).label("y", 0).value(Float.POSITIVE_INFINITY); + builder.cell().label("x", 2).label("y", 1).value(Float.NEGATIVE_INFINITY); + builder.cell().label("x", 2).label("y", 2).value(Float.NaN); + builder.cell().label("x", 2).label("y", 3).value(-Float.NaN); + Tensor expected = builder.build(); + + String denseJson = "{\"values\":\"" + +"42280000"+"49800008"+"35800000"+"c37f0000" + +"00000000"+"80000000"+"00000001"+"7f7fffff" + +"7f800000"+"ff800000"+"7fc00000"+"ffc00000" + +"\"}"; + Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); + } + + @Test + public void testDoubleVectorInHexForm() { + var builder = Tensor.Builder.of(TensorType.fromSpec("tensor<double>(x[3],y[4])")); + builder.cell().label("x", 0).label("y", 0).value(42.0); + builder.cell().label("x", 0).label("y", 1).value(1048577.0); + builder.cell().label("x", 0).label("y", 2).value(0.00000095367431640625); + builder.cell().label("x", 0).label("y", 3).value(-255.00); + + builder.cell().label("x", 1).label("y", 0).value(0.0); + builder.cell().label("x", 1).label("y", 1).value(-0.0); + builder.cell().label("x", 1).label("y", 2).value(Double.MIN_VALUE); + builder.cell().label("x", 1).label("y", 3).value(Double.MAX_VALUE); + + builder.cell().label("x", 2).label("y", 0).value(Double.POSITIVE_INFINITY); + builder.cell().label("x", 2).label("y", 1).value(Double.NEGATIVE_INFINITY); + builder.cell().label("x", 2).label("y", 2).value(Double.NaN); + builder.cell().label("x", 2).label("y", 3).value(-Double.NaN); + Tensor expected = builder.build(); + + String denseJson = "{\"values\":\"" + +"4045000000000000"+"4130000100000000"+"3eb0000000000000"+"c06fe00000000000" + +"0000000000000000"+"8000000000000000"+"0000000000000001"+"7fefffffffffffff" + +"7ff0000000000000"+"fff0000000000000"+"7ff8000000000000"+"fff8000000000000" + +"\"}"; + Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); + } + + @Test public void testMixedTensorInMixedForm() { Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y[3])")); builder.cell().label("x", 0).label("y", 0).value(2.0); |