diff options
Diffstat (limited to 'vespajlib')
4 files changed, 329 insertions, 95 deletions
diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 8a2e68a8d8c..7f4a19b029d 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3079,7 +3079,8 @@ "methods" : [ "public static java.lang.String encode(java.util.Map)", "public static java.lang.String escape(java.lang.String)", - "public static boolean equals(java.lang.String, java.lang.String)" + "public static boolean equals(java.lang.String, java.lang.String)", + "public static java.lang.String canonical(java.lang.String)" ], "fields" : [ ] }, diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index 0e8fbc30bb6..b7e6e67ce73 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -37,52 +37,78 @@ import java.util.stream.Collectors; */ public class JsonFormat { - /** Serializes the given tensor value into JSON format */ - public static byte[] encode(Tensor tensor) { + /** + * Serializes the given tensor value into JSON format. + * + * @param tensor the tensor to serialize + * @param shortForm whether to encode in a short type-dependent format + * @param directValues whether to encode values directly, or wrapped in am object containing "type" and "cells" + */ + public static byte[] encode(Tensor tensor, boolean shortForm, boolean directValues) { Slime slime = new Slime(); - Cursor root = slime.setObject(); - encodeCells(tensor, root); + Cursor root = null; + if ( ! directValues) { + root = slime.setObject(); + root.setString("type", tensor.type().toString()); + } + + if (shortForm) { + if (tensor instanceof IndexedTensor denseTensor) { + // Encode as nested lists if indexed tensor + Cursor parent = root == null ? slime.setArray() : root.setArray("values"); + encodeValues(denseTensor, parent, new long[denseTensor.dimensionSizes().dimensions()], 0); + } else if (tensor instanceof MappedTensor && tensor.type().dimensions().size() == 1) { + // Short form for a single mapped dimension + Cursor parent = root == null ? slime.setObject() : root.setObject("cells"); + encodeSingleDimensionCells((MappedTensor) tensor, parent); + } else if (tensor instanceof MixedTensor && + tensor.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped)) { + // Short form for a mixed tensor + boolean singleMapped = tensor.type().dimensions().stream().filter(TensorType.Dimension::isMapped).count() == 1; + Cursor parent = root == null ? ( singleMapped ? slime.setObject() : slime.setArray() ) + : ( singleMapped ? root.setObject("blocks") : root.setArray("blocks")); + encodeBlocks((MixedTensor) tensor, parent); + } else { + // default to standard cell address output + Cursor parent = root == null ? slime.setArray() : root.setArray("cells"); + encodeCells(tensor, parent); + } + + return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + } + else { + Cursor parent = root == null ? slime.setArray() : root.setArray("cells"); + encodeCells(tensor, parent); + } return com.yahoo.slime.JsonFormat.toJsonBytes(slime); } - /** Serializes the given tensor type and value into JSON format */ + /** Serializes the given tensor value into JSON format, in long format, wrapped in an object containing "cells" only. */ + public static byte[] encode(Tensor tensor) { + return encode(tensor, false, false); + } + + /** + * Serializes the given tensor type and value into JSON format. + * + * @deprecated use #encode(#Tensor, boolean, boolean) + */ + @Deprecated // TODO: Remove on Vespa 9 public static byte[] encodeWithType(Tensor tensor) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("type", tensor.type().toString()); - encodeCells(tensor, root); - return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + return encode(tensor, false, false); } - /** Serializes the given tensor type and value into a short-form JSON format */ + /** + * Serializes the given tensor type and value into a short-form JSON format. + * + * @deprecated use #encode(#Tensor, boolean, boolean) + */ + @Deprecated // TODO: Remove on Vespa 9 public static byte[] encodeShortForm(Tensor tensor) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("type", tensor.type().toString()); - - if (tensor instanceof IndexedTensor denseTensor) { - // Encode as nested lists if indexed tensor - encodeValues(denseTensor, root.setArray("values"), new long[denseTensor.dimensionSizes().dimensions()], 0); - } - else if (tensor instanceof MappedTensor && tensor.type().dimensions().size() == 1) { - // Short form for a single mapped dimension - encodeSingleDimensionCells((MappedTensor) tensor, root); - } - else if (tensor instanceof MixedTensor && - tensor.type().dimensions().stream().filter(TensorType.Dimension::isMapped).count() >= 1) { - // Short form for a mixed tensor - encodeBlocks((MixedTensor) tensor, root); - } - else { - // No other short forms exist: default to standard cell address output - encodeCells(tensor, root); - } - - return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + return encode(tensor, true, false); } - private static void encodeCells(Tensor tensor, Cursor rootObject) { - Cursor cellsArray = rootObject.setArray("cells"); + private static void encodeCells(Tensor tensor, Cursor cellsArray) { for (Iterator<Tensor.Cell> i = tensor.cellIterator(); i.hasNext(); ) { Tensor.Cell cell = i.next(); Cursor cellObject = cellsArray.addObject(); @@ -91,8 +117,7 @@ public class JsonFormat { } } - private static void encodeSingleDimensionCells(MappedTensor tensor, Cursor cursor) { - Cursor cells = cursor.setObject("cells"); + private static void encodeSingleDimensionCells(MappedTensor tensor, Cursor cells) { if (tensor.type().dimensions().size() > 1) throw new IllegalStateException("JSON encode of mapped tensor can only contain a single dimension"); tensor.cells().forEach((k,v) -> cells.setDouble(k.label(0), v)); @@ -124,7 +149,6 @@ public class JsonFormat { if (mappedDimensions.size() < 1) { throw new IllegalArgumentException("Should be ensured by caller"); } - cursor = (mappedDimensions.size() == 1) ? cursor.setObject("blocks") : cursor.setArray("blocks"); // Create tensor type for mapped dimensions subtype TensorType mappedSubType = new TensorType.Builder(mappedDimensions).build(); @@ -216,48 +240,52 @@ public class JsonFormat { } private static void decodeValues(Inspector values, Tensor.Builder builder) { + decodeValues(values, builder, new MutableInteger(0)); + } + + private static void decodeValues(Inspector values, Tensor.Builder builder, MutableInteger index) { if ( ! (builder instanceof IndexedTensor.BoundBuilder indexedBuilder)) - throw new IllegalArgumentException("The 'values' field can only be used with dense tensors. " + - "Use 'cells' or 'blocks' instead"); + throw new IllegalArgumentException("An array of values can only be used with a dense tensor. Use a map instead"); if (values.type() == Type.STRING) { double[] decoded = decodeHexString(values.asString(), builder.type().valueType()); if (decoded.length == 0) - throw new IllegalArgumentException("The 'values' string does not contain any values"); + throw new IllegalArgumentException("The values string does not contain any values"); for (int i = 0; i < decoded.length; i++) { indexedBuilder.cellByDirectIndex(i, decoded[i]); } return; } if (values.type() != Type.ARRAY) - throw new IllegalArgumentException("Excepted 'values' to contain an array, not " + values.type()); + throw new IllegalArgumentException("Excepted values to be an array, not " + values.type()); if (values.entries() == 0) - throw new IllegalArgumentException("The 'values' array does not contain any values"); + throw new IllegalArgumentException("The values array does not contain any values"); - MutableInteger index = new MutableInteger(0); values.traverse((ArrayTraverser) (__, value) -> { - if (value.type() != Type.LONG && value.type() != Type.DOUBLE) { - throw new IllegalArgumentException("Excepted the values array to contain numbers, not " + value.type()); - } - indexedBuilder.cellByDirectIndex(index.next(), value.asDouble()); + if (value.type() == Type.ARRAY) + decodeValues(value, builder, index); + else if (value.type() == Type.LONG || value.type() == Type.DOUBLE) + indexedBuilder.cellByDirectIndex(index.next(), value.asDouble()); + else + throw new IllegalArgumentException("Excepted the values array to contain numbers or nested arrays, not " + value.type()); }); } private static void decodeBlocks(Inspector values, Tensor.Builder builder) { if ( ! (builder instanceof MixedTensor.BoundBuilder mixedBuilder)) - throw new IllegalArgumentException("The 'blocks' field can only be used with mixed tensors with bound dimensions. " + - "Use 'cells' or 'values' instead"); + throw new IllegalArgumentException("Blocks of values can only be used with mixed (sparse and dense) tensors." + + "Use an array of cell values instead."); if (values.type() == Type.ARRAY) values.traverse((ArrayTraverser) (__, value) -> decodeBlock(value, mixedBuilder)); else if (values.type() == Type.OBJECT) values.traverse((ObjectTraverser) (key, value) -> decodeSingleDimensionBlock(key, value, mixedBuilder)); else - throw new IllegalArgumentException("Excepted 'blocks' to contain an array or object, not " + values.type()); + throw new IllegalArgumentException("Excepted the block to contain an array or object, not " + values.type()); } private static void decodeBlock(Inspector block, MixedTensor.BoundBuilder mixedBuilder) { if (block.type() != Type.OBJECT) - throw new IllegalArgumentException("Expected an item in a 'blocks' array to be an object, not " + block.type()); + throw new IllegalArgumentException("Expected an item in a blocks array to be an object, not " + block.type()); mixedBuilder.block(decodeAddress(block.field("address"), mixedBuilder.type().mappedSubtype()), decodeValues(block.field("values"), mixedBuilder)); } @@ -267,7 +295,9 @@ public class JsonFormat { boolean hasIndexed = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isIndexed); boolean hasMapped = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped); - if ( ! hasMapped) + if (isArrayOfObjects(root)) + decodeCells(root, builder); + else if ( ! hasMapped) decodeValues(root, builder); else if (hasMapped && hasIndexed) decodeBlocks(root, builder); @@ -275,9 +305,17 @@ public class JsonFormat { decodeCells(root, builder); } + private static boolean isArrayOfObjects(Inspector inspector) { + if (inspector.type() != Type.ARRAY) return false; + if (inspector.entries() == 0) return false; + Inspector firstItem = inspector.entry(0); + if (firstItem.type() == Type.ARRAY) return isArrayOfObjects(firstItem); + return firstItem.type() == Type.OBJECT; + } + private static void decodeSingleDimensionBlock(String key, Inspector value, MixedTensor.BoundBuilder mixedBuilder) { if (value.type() != Type.ARRAY) - throw new IllegalArgumentException("Expected an item in a 'blocks' array to be an array, not " + value.type()); + throw new IllegalArgumentException("Expected an item in a blocks array to be an array, not " + value.type()); mixedBuilder.block(asAddress(key, mixedBuilder.type().mappedSubtype()), decodeValues(value, mixedBuilder)); } @@ -361,19 +399,19 @@ public class JsonFormat { double[] values = new double[(int)mixedBuilder.denseSubspaceSize()]; if (valuesField.type() == Type.ARRAY) { if (valuesField.entries() == 0) { - throw new IllegalArgumentException("The 'block' value array does not contain any values"); + throw new IllegalArgumentException("The block value array does not contain any values"); } valuesField.traverse((ArrayTraverser) (index, value) -> values[index] = decodeNumeric(value)); } else if (valuesField.type() == Type.STRING) { double[] decoded = decodeHexString(valuesField.asString(), mixedBuilder.type().valueType()); if (decoded.length == 0) { - throw new IllegalArgumentException("The 'block' value string does not contain any values"); + throw new IllegalArgumentException("The block value string does not contain any values"); } for (int i = 0; i < decoded.length; i++) { values[i] = decoded[i]; } } else { - throw new IllegalArgumentException("Expected a block to contain a 'values' array"); + throw new IllegalArgumentException("Expected a block to contain an array of values"); } return values; } diff --git a/vespajlib/src/main/java/com/yahoo/text/JSON.java b/vespajlib/src/main/java/com/yahoo/text/JSON.java index 6f8ef9a289f..8ef66b745cc 100644 --- a/vespajlib/src/main/java/com/yahoo/text/JSON.java +++ b/vespajlib/src/main/java/com/yahoo/text/JSON.java @@ -75,4 +75,8 @@ public final class JSON { return leftSlime.equalTo(rightSlime); } + public static String canonical(String jsonString) { + return SlimeUtils.jsonToSlimeOrThrow(jsonString).toString(); + } + } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java index 6a6bb3c6781..7c7391ff895 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java @@ -3,7 +3,9 @@ package com.yahoo.tensor.serialization; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.text.JSON; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import java.nio.charset.StandardCharsets; @@ -20,7 +22,9 @@ public class JsonFormatTestCase { public void testDirectValue() { assertDecoded("tensor(x{}):{a:2, b:3}", "{'a':2.0, 'b':3.0}"); assertDecoded("tensor(x{}):{a:2, b:3}", "{'a':2.0, 'b':3.0}"); - assertDecoded("tensor(x[2]):[2, 3]]", "[2.0, 3.0]"); + assertDecoded("tensor(x[2]):[1.0, 2.0]]", "[1, 2]"); + assertDecoded("tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]", "[1, 2, 3, 4, 5, 6]"); + assertDecoded("tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]", "[[1, 2, 3], [4, 5, 6]]"); assertDecoded("tensor(x{},y[2]):{a:[2, 3], b:[4, 5]}", "{'a':[2, 3], 'b':[4, 5]}"); assertDecoded("tensor(x{},y{}):{{x:a,y:0}:2, {x:b,y:1}:3}", "[{'address':{'x':'a','y':'0'},'value':2}, {'address':{'x':'b','y':'1'},'value':3}]"); @@ -32,35 +36,21 @@ public class JsonFormatTestCase { assertDecoded("tensor(x{}):{cells:2, b:3}", "{'cells':2.0, 'b':3.0}"); assertDecoded("tensor(x{}):{values:2, b:3}", "{'values':2.0, 'b':3.0}"); assertDecoded("tensor(x{}):{block:2, b:3}", "{'block':2.0, 'b':3.0}"); + assertDecoded("tensor(x{}):{type:2, b:3}", "{'type':2.0, 'b':3.0}"); // Multi-valued assertDecoded("tensor(x{},y[2]):{cells:[2, 3], b:[4, 5]}", "{'cells':[2, 3], 'b':[4, 5]}"); assertDecoded("tensor(x{},y[2]):{values:[2, 3], b:[4, 5]}", "{'values':[2, 3], 'b':[4, 5]}"); assertDecoded("tensor(x{},y[2]):{block:[2, 3], b:[4, 5]}", "{'block':[2, 3], 'b':[4, 5]}"); - } - - @Test - public void testSparseTensor() { - Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})")); - builder.cell().label("x", "a").label("y", "b").value(2.0); - builder.cell().label("x", "c").label("y", "d").value(3.0); - Tensor tensor = builder.build(); - byte[] json = JsonFormat.encode(tensor); - assertEquals("{\"cells\":[" + - "{\"address\":{\"x\":\"a\",\"y\":\"b\"},\"value\":2.0}," + - "{\"address\":{\"x\":\"c\",\"y\":\"d\"},\"value\":3.0}" + - "]}", - new String(json, StandardCharsets.UTF_8)); - Tensor decoded = JsonFormat.decode(tensor.type(), json); - assertEquals(tensor, decoded); + assertDecoded("tensor(x{},y[2]):{type:[2, 3], b:[4, 5]}", "{'type':[2, 3], 'b':[4, 5]}"); } @Test public void testEmptySparseTensor() { Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})")); Tensor tensor = builder.build(); - byte[] json = JsonFormat.encode(tensor); - assertEquals("{\"cells\":[]}", + byte[] json = JsonFormat.encode(tensor, false, false); + assertEquals("{\"type\":\"tensor(x{},y{})\",\"cells\":[]}", new String(json, StandardCharsets.UTF_8)); Tensor decoded = JsonFormat.decode(tensor.type(), json); assertEquals(tensor, decoded); @@ -86,6 +76,45 @@ public class JsonFormatTestCase { } @Test + public void testEmptyTensor() { + Tensor tensor = Tensor.Builder.of(TensorType.empty).build(); + + String shortJson = """ + { + "type":"tensor()", + "values":[0.0] + } + """; + byte[] shortEncoded = JsonFormat.encode(tensor, true, false); + assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded)); + + String longJson = """ + { + "type":"tensor()", + "cells":[{"address":{},"value":0.0}] + } + """; + byte[] longEncoded = JsonFormat.encode(tensor, false, false); + assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded)); + + String shortDirectJson = """ + [0.0] + """; + byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true); + assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded)); + + String longDirectJson = """ + [{"address":{},"value":0.0}] + """; + byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true); + assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded)); + } + + @Test public void testDenseTensor() { Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[2])")); builder.cell().label("x", 0).label("y", 0).value(2.0); @@ -93,31 +122,183 @@ public class JsonFormatTestCase { builder.cell().label("x", 1).label("y", 0).value(5.0); builder.cell().label("x", 1).label("y", 1).value(7.0); Tensor tensor = builder.build(); - byte[] json = JsonFormat.encode(tensor); - assertEquals("{\"cells\":[" + - "{\"address\":{\"x\":\"0\",\"y\":\"0\"},\"value\":2.0}," + - "{\"address\":{\"x\":\"0\",\"y\":\"1\"},\"value\":3.0}," + - "{\"address\":{\"x\":\"1\",\"y\":\"0\"},\"value\":5.0}," + - "{\"address\":{\"x\":\"1\",\"y\":\"1\"},\"value\":7.0}" + - "]}", - new String(json, StandardCharsets.UTF_8)); - Tensor decoded = JsonFormat.decode(tensor.type(), json); - assertEquals(tensor, decoded); + + String shortJson = """ + { + "type":"tensor(x[2],y[2])", + "values":[[2.0,3.0],[5.0,7.0]] + } + """; + byte[] shortEncoded = JsonFormat.encode(tensor, true, false); + assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded)); + + String longJson = """ + { + "type":"tensor(x[2],y[2])", + "cells":[ + {"address":{"x":"0","y":"0"},"value":2.0}, + {"address":{"x":"0","y":"1"},"value":3.0}, + {"address":{"x":"1","y":"0"},"value":5.0}, + {"address":{"x":"1","y":"1"},"value":7.0} + ] + } + """; + byte[] longEncoded = JsonFormat.encode(tensor, false, false); + assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded)); + + String shortDirectJson = """ + [[2.0, 3.0], [5.0, 7.0]] + """; + byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true); + assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded)); + + String longDirectJson = """ + [ + {"address":{"x":"0","y":"0"},"value":2.0}, + {"address":{"x":"0","y":"1"},"value":3.0}, + {"address":{"x":"1","y":"0"},"value":5.0}, + {"address":{"x":"1","y":"1"},"value":7.0} + ] + """; + byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true); + assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded)); + } + + @Test + public void testMixedTensor() { + Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y[2])")); + builder.cell().label("x", "a").label("y", 0).value(2.0); + builder.cell().label("x", "a").label("y", 1).value(3.0); + builder.cell().label("x", "b").label("y", 0).value(5.0); + builder.cell().label("x", "b").label("y", 1).value(7.0); + Tensor tensor = builder.build(); + + String shortJson = """ + { + "type":"tensor(x{},y[2])", + "blocks":{"a":[2.0,3.0],"b":[5.0,7.0]} + } + """; + byte[] shortEncoded = JsonFormat.encode(tensor, true, false); + assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded)); + + String longJson = """ + { + "type":"tensor(x{},y[2])", + "cells":[ + {"address":{"x":"a","y":"0"},"value":2.0}, + {"address":{"x":"a","y":"1"},"value":3.0}, + {"address":{"x":"b","y":"0"},"value":5.0}, + {"address":{"x":"b","y":"1"},"value":7.0} + ] + } + """; + byte[] longEncoded = JsonFormat.encode(tensor, false, false); + assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded)); + + String shortDirectJson = """ + {"a":[2.0,3.0],"b":[5.0,7.0]} + """; + byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true); + assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded)); + + String longDirectJson = """ + [ + {"address":{"x":"a","y":"0"},"value":2.0}, + {"address":{"x":"a","y":"1"},"value":3.0}, + {"address":{"x":"b","y":"0"},"value":5.0}, + {"address":{"x":"b","y":"1"},"value":7.0} + ] + """; + byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true); + assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded)); + } + + @Test + public void testSparseTensor() { + Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})")); + builder.cell().label("x", "a").label("y", 0).value(2.0); + builder.cell().label("x", "a").label("y", 1).value(3.0); + builder.cell().label("x", "b").label("y", 0).value(5.0); + builder.cell().label("x", "b").label("y", 1).value(7.0); + Tensor tensor = builder.build(); + + String shortJson = """ + { + "type":"tensor(x{},y{})", + "cells": [ + {"address":{"x":"a","y":"0"},"value":2.0}, + {"address":{"x":"a","y":"1"},"value":3.0}, + {"address":{"x":"b","y":"0"},"value":5.0}, + {"address":{"x":"b","y":"1"},"value":7.0} + ] + } + """; + byte[] shortEncoded = JsonFormat.encode(tensor, true, false); + assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded)); + + String longJson = """ + { + "type":"tensor(x{},y{})", + "cells":[ + {"address":{"x":"a","y":"0"},"value":2.0}, + {"address":{"x":"a","y":"1"},"value":3.0}, + {"address":{"x":"b","y":"0"},"value":5.0}, + {"address":{"x":"b","y":"1"},"value":7.0} + ] + } + """; + byte[] longEncoded = JsonFormat.encode(tensor, false, false); + assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded)); + + String shortDirectJson = """ + [ + {"address":{"x":"a","y":"0"},"value":2.0}, + {"address":{"x":"a","y":"1"},"value":3.0}, + {"address":{"x":"b","y":"0"},"value":5.0}, + {"address":{"x":"b","y":"1"},"value":7.0} + ] + """; + byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true); + assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded)); + + String longDirectJson = """ + [ + {"address":{"x":"a","y":"0"},"value":2.0}, + {"address":{"x":"a","y":"1"},"value":3.0}, + {"address":{"x":"b","y":"0"},"value":5.0}, + {"address":{"x":"b","y":"1"},"value":7.0} + ] + """; + byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true); + assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8)); + assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded)); } @Test public void testDisallowedEmptyDenseTensor() { TensorType type = TensorType.fromSpec("tensor(x[3])"); - assertDecodeFails(type, "{\"values\":[]}", "The 'values' array does not contain any values"); - assertDecodeFails(type, "{\"values\":\"\"}", "The 'values' string does not contain any values"); + assertDecodeFails(type, "{\"values\":[]}", "The values array does not contain any values"); + assertDecodeFails(type, "{\"values\":\"\"}", "The values string does not contain any values"); } @Test public void testDisallowedEmptyMixedTensor() { TensorType type = TensorType.fromSpec("tensor(x{},y[3])"); - assertDecodeFails(type, "{\"blocks\":{ \"a\": [] } }", "The 'block' value array does not contain any values"); + assertDecodeFails(type, "{\"blocks\":{ \"a\": [] } }", "The block value array does not contain any values"); assertDecodeFails(type, "{\"blocks\":[ {\"address\":{\"x\":\"a\"}, \"values\": [] } ] }", - "The 'block' value array does not contain any values"); + "The block value array does not contain any values"); } @Test @@ -204,9 +385,14 @@ public class JsonFormatTestCase { builder.cell().label("x", 1).label("y", 1).value(0.0); builder.cell().label("x", 1).label("y", 2).value(42.0); Tensor expected = builder.build(); + String denseJson = "{\"values\":\"027FFF80002A\"}"; Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); assertEquals(expected, decoded); + + denseJson = "\"027FFF80002A\""; + decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); } @Test @@ -231,6 +417,7 @@ public class JsonFormatTestCase { builder.cell().label("x", 1).label("y", 1).value(6.0); builder.cell().label("x", 1).label("y", 2).value(7.0); Tensor expected = builder.build(); + String mixedJson = "{\"blocks\":[" + "{\"address\":{\"x\":\"0\"},\"values\":\"020304\"}," + "{\"address\":{\"x\":\"1\"},\"values\":\"050607\"}" + @@ -373,7 +560,7 @@ public class JsonFormatTestCase { } private void assertEncodeDecode(Tensor tensor) { - Tensor decoded = JsonFormat.decode(tensor.type(), JsonFormat.encodeWithType(tensor)); + Tensor decoded = JsonFormat.decode(tensor.type(), JsonFormat.encode(tensor, false, false)); assertEquals(tensor, decoded); assertEquals(tensor.type(), decoded.type()); } @@ -401,7 +588,7 @@ public class JsonFormatTestCase { } private void assertEncodeShortForm(Tensor tensor, String expected) { - byte[] json = JsonFormat.encodeShortForm(tensor); + byte[] json = JsonFormat.encode(tensor, true, false); assertEquals(expected, new String(json, StandardCharsets.UTF_8)); } @@ -418,8 +605,12 @@ public class JsonFormatTestCase { Tensor decoded = JsonFormat.decode(type, format.getBytes(StandardCharsets.UTF_8)); fail("Did not get exception as expected, decoded as: " + decoded); } catch (IllegalArgumentException e) { - assertEquals(e.getMessage(), msg); + assertEquals(msg, e.getMessage()); } } + private void assertEqualJson(String expected, String generated) { + Assertions.assertEquals(JSON.canonical(expected), JSON.canonical(generated)); + } + } |