Test direct rendering

3 files changed, 257 insertions, 59 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java
index 68997c82d3e..b7e6e67ce73 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java
@@ -46,13 +46,13 @@ public class JsonFormat {
      */
     public static byte[] encode(Tensor tensor, boolean shortForm, boolean directValues) {
         Slime slime = new Slime();
-        if (shortForm) {
-            Cursor root = null;
-            if ( ! directValues) {
-                root = slime.setObject();
-                root.setString("type", tensor.type().toString());
-            }
+        Cursor root = null;
+        if ( ! directValues) {
+            root = slime.setObject();
+            root.setString("type", tensor.type().toString());
+        }
 
+        if (shortForm) {
             if (tensor instanceof IndexedTensor denseTensor) {
                 // Encode as nested lists if indexed tensor
                 Cursor parent = root == null ? slime.setArray() : root.setArray("values");
@@ -77,9 +77,8 @@ public class JsonFormat {
             return com.yahoo.slime.JsonFormat.toJsonBytes(slime);
         }
         else {
-            Cursor root = slime.setObject();
-            root.setString("type", tensor.type().toString());
-            encodeCells(tensor, root.setArray("cells"));
+            Cursor parent = root == null ? slime.setArray() : root.setArray("cells");
+            encodeCells(tensor, parent);
         }
         return com.yahoo.slime.JsonFormat.toJsonBytes(slime);
     }
@@ -241,48 +240,52 @@ public class JsonFormat {
     }
 
     private static void decodeValues(Inspector values, Tensor.Builder builder) {
+        decodeValues(values, builder, new MutableInteger(0));
+    }
+
+    private static void decodeValues(Inspector values, Tensor.Builder builder, MutableInteger index) {
         if ( ! (builder instanceof IndexedTensor.BoundBuilder indexedBuilder))
-            throw new IllegalArgumentException("The 'values' field can only be used with dense tensors. " +
-                                               "Use 'cells' or 'blocks' instead");
+            throw new IllegalArgumentException("An array of values can only be used with a dense tensor. Use a map instead");
         if (values.type() == Type.STRING) {
             double[] decoded = decodeHexString(values.asString(), builder.type().valueType());
             if (decoded.length == 0)
-                throw new IllegalArgumentException("The 'values' string does not contain any values");
+                throw new IllegalArgumentException("The values string does not contain any values");
             for (int i = 0; i < decoded.length; i++) {
                 indexedBuilder.cellByDirectIndex(i, decoded[i]);
             }
             return;
         }
         if (values.type() != Type.ARRAY)
-            throw new IllegalArgumentException("Excepted 'values' to contain an array, not " + values.type());
+            throw new IllegalArgumentException("Excepted values to be an array, not " + values.type());
         if (values.entries() == 0)
-            throw new IllegalArgumentException("The 'values' array does not contain any values");
+            throw new IllegalArgumentException("The values array does not contain any values");
 
-        MutableInteger index = new MutableInteger(0);
         values.traverse((ArrayTraverser) (__, value) -> {
-            if (value.type() != Type.LONG && value.type() != Type.DOUBLE) {
-                throw new IllegalArgumentException("Excepted the values array to contain numbers, not " + value.type());
-            }
-            indexedBuilder.cellByDirectIndex(index.next(), value.asDouble());
+            if (value.type() == Type.ARRAY)
+                decodeValues(value, builder, index);
+            else if (value.type() == Type.LONG || value.type() == Type.DOUBLE)
+                indexedBuilder.cellByDirectIndex(index.next(), value.asDouble());
+            else
+                throw new IllegalArgumentException("Excepted the values array to contain numbers or nested arrays, not " + value.type());
         });
     }
 
     private static void decodeBlocks(Inspector values, Tensor.Builder builder) {
         if ( ! (builder instanceof MixedTensor.BoundBuilder mixedBuilder))
-            throw new IllegalArgumentException("The 'blocks' field can only be used with mixed tensors with bound dimensions. " +
-                                               "Use 'cells' or 'values' instead");
+            throw new IllegalArgumentException("Blocks of values can only be used with mixed (sparse and dense) tensors." +
+                                               "Use an array of cell values instead.");
 
         if (values.type() == Type.ARRAY)
             values.traverse((ArrayTraverser) (__, value) -> decodeBlock(value, mixedBuilder));
         else if (values.type() == Type.OBJECT)
             values.traverse((ObjectTraverser) (key, value) -> decodeSingleDimensionBlock(key, value, mixedBuilder));
         else
-            throw new IllegalArgumentException("Excepted 'blocks' to contain an array or object, not " + values.type());
+            throw new IllegalArgumentException("Excepted the block to contain an array or object, not " + values.type());
     }
 
     private static void decodeBlock(Inspector block, MixedTensor.BoundBuilder mixedBuilder) {
         if (block.type() != Type.OBJECT)
-            throw new IllegalArgumentException("Expected an item in a 'blocks' array to be an object, not " + block.type());
+            throw new IllegalArgumentException("Expected an item in a blocks array to be an object, not " + block.type());
         mixedBuilder.block(decodeAddress(block.field("address"), mixedBuilder.type().mappedSubtype()),
                            decodeValues(block.field("values"), mixedBuilder));
     }
@@ -292,7 +295,9 @@ public class JsonFormat {
         boolean hasIndexed = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isIndexed);
         boolean hasMapped = builder.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped);
 
-        if ( ! hasMapped)
+        if (isArrayOfObjects(root))
+            decodeCells(root, builder);
+        else if ( ! hasMapped)
             decodeValues(root, builder);
         else if (hasMapped && hasIndexed)
             decodeBlocks(root, builder);
@@ -300,9 +305,17 @@ public class JsonFormat {
             decodeCells(root, builder);
     }
 
+    private static boolean isArrayOfObjects(Inspector inspector) {
+        if (inspector.type() != Type.ARRAY) return false;
+        if (inspector.entries() == 0) return false;
+        Inspector firstItem = inspector.entry(0);
+        if (firstItem.type() == Type.ARRAY) return isArrayOfObjects(firstItem);
+        return firstItem.type() == Type.OBJECT;
+    }
+
     private static void decodeSingleDimensionBlock(String key, Inspector value, MixedTensor.BoundBuilder mixedBuilder) {
         if (value.type() != Type.ARRAY)
-            throw new IllegalArgumentException("Expected an item in a 'blocks' array to be an array, not " + value.type());
+            throw new IllegalArgumentException("Expected an item in a blocks array to be an array, not " + value.type());
         mixedBuilder.block(asAddress(key, mixedBuilder.type().mappedSubtype()),
                            decodeValues(value, mixedBuilder));
     }
@@ -386,19 +399,19 @@ public class JsonFormat {
         double[] values = new double[(int)mixedBuilder.denseSubspaceSize()];
         if (valuesField.type() == Type.ARRAY) {
             if (valuesField.entries() == 0) {
-                throw new IllegalArgumentException("The 'block' value array does not contain any values");
+                throw new IllegalArgumentException("The block value array does not contain any values");
             }
             valuesField.traverse((ArrayTraverser) (index, value) -> values[index] = decodeNumeric(value));
         } else if (valuesField.type() == Type.STRING) {
             double[] decoded = decodeHexString(valuesField.asString(), mixedBuilder.type().valueType());
             if (decoded.length == 0) {
-                throw new IllegalArgumentException("The 'block' value string does not contain any values");
+                throw new IllegalArgumentException("The block value string does not contain any values");
             }
             for (int i = 0; i < decoded.length; i++) {
                 values[i] = decoded[i];
             }
         } else {
-            throw new IllegalArgumentException("Expected a block to contain a 'values' array");
+            throw new IllegalArgumentException("Expected a block to contain an array of values");
         }
         return values;
     }
diff --git a/vespajlib/src/main/java/com/yahoo/text/JSON.java b/vespajlib/src/main/java/com/yahoo/text/JSON.java
index 6f8ef9a289f..8ef66b745cc 100644
--- a/vespajlib/src/main/java/com/yahoo/text/JSON.java
+++ b/vespajlib/src/main/java/com/yahoo/text/JSON.java
@@ -75,4 +75,8 @@ public final class JSON {
         return leftSlime.equalTo(rightSlime);
     }
 
+    public static String canonical(String jsonString) {
+        return SlimeUtils.jsonToSlimeOrThrow(jsonString).toString();
+    }
+
 }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
index 4692cf87d59..7f9705d33bd 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
@@ -3,7 +3,9 @@ package com.yahoo.tensor.serialization;
 
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
+import com.yahoo.text.JSON;
 import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
 
 import java.nio.charset.StandardCharsets;
 
@@ -42,22 +44,6 @@ public class JsonFormatTestCase {
     }
 
     @Test
-    public void testSparseTensor() {
-        Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})"));
-        builder.cell().label("x", "a").label("y", "b").value(2.0);
-        builder.cell().label("x", "c").label("y", "d").value(3.0);
-        Tensor tensor = builder.build();
-        byte[] json = JsonFormat.encode(tensor, false, false);
-        assertEquals("{\"type\":\"tensor(x{},y{})\",\"cells\":[" +
-                     "{\"address\":{\"x\":\"a\",\"y\":\"b\"},\"value\":2.0}," +
-                     "{\"address\":{\"x\":\"c\",\"y\":\"d\"},\"value\":3.0}" +
-                     "]}",
-                     new String(json, StandardCharsets.UTF_8));
-        Tensor decoded = JsonFormat.decode(tensor.type(), json);
-        assertEquals(tensor, decoded);
-    }
-
-    @Test
     public void testEmptySparseTensor() {
         Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})"));
         Tensor tensor = builder.build();
@@ -88,6 +74,45 @@ public class JsonFormatTestCase {
     }
 
     @Test
+    public void testEmptyTensor() {
+        Tensor tensor = Tensor.Builder.of(TensorType.empty).build();
+
+        String shortJson = """
+                {
+                  "type":"tensor()",
+                  "values":[0.0]
+                }
+                """;
+        byte[] shortEncoded = JsonFormat.encode(tensor, true, false);
+        assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded));
+
+        String longJson = """
+                {
+                  "type":"tensor()",
+                  "cells":[{"address":{},"value":0.0}]
+                }
+                """;
+        byte[] longEncoded = JsonFormat.encode(tensor, false, false);
+        assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded));
+
+        String shortDirectJson = """
+                [0.0]
+                """;
+        byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true);
+        assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded));
+
+        String longDirectJson = """
+                [{"address":{},"value":0.0}]
+                """;
+        byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true);
+        assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded));
+    }
+
+    @Test
     public void testDenseTensor() {
         Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x[2],y[2])"));
         builder.cell().label("x", 0).label("y", 0).value(2.0);
@@ -95,31 +120,183 @@ public class JsonFormatTestCase {
         builder.cell().label("x", 1).label("y", 0).value(5.0);
         builder.cell().label("x", 1).label("y", 1).value(7.0);
         Tensor tensor = builder.build();
-        byte[] json = JsonFormat.encode(tensor, false, false);
-        assertEquals("{\"type\":\"tensor(x[2],y[2])\",\"cells\":[" +
-                     "{\"address\":{\"x\":\"0\",\"y\":\"0\"},\"value\":2.0}," +
-                     "{\"address\":{\"x\":\"0\",\"y\":\"1\"},\"value\":3.0}," +
-                     "{\"address\":{\"x\":\"1\",\"y\":\"0\"},\"value\":5.0}," +
-                     "{\"address\":{\"x\":\"1\",\"y\":\"1\"},\"value\":7.0}" +
-                     "]}",
-                     new String(json, StandardCharsets.UTF_8));
-        Tensor decoded = JsonFormat.decode(tensor.type(), json);
-        assertEquals(tensor, decoded);
+
+        String shortJson = """
+                {
+                  "type":"tensor(x[2],y[2])",
+                  "values":[[2.0,3.0],[5.0,7.0]]
+                }
+                """;
+        byte[] shortEncoded = JsonFormat.encode(tensor, true, false);
+        assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded));
+
+        String longJson = """
+                {
+                  "type":"tensor(x[2],y[2])",
+                  "cells":[
+                    {"address":{"x":"0","y":"0"},"value":2.0},
+                    {"address":{"x":"0","y":"1"},"value":3.0},
+                    {"address":{"x":"1","y":"0"},"value":5.0},
+                    {"address":{"x":"1","y":"1"},"value":7.0}
+                  ]
+                }
+                """;
+        byte[] longEncoded = JsonFormat.encode(tensor, false, false);
+        assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded));
+
+        String shortDirectJson = """
+                [[2.0, 3.0], [5.0, 7.0]]
+                """;
+        byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true);
+        assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded));
+
+        String longDirectJson = """
+                [
+                    {"address":{"x":"0","y":"0"},"value":2.0},
+                    {"address":{"x":"0","y":"1"},"value":3.0},
+                    {"address":{"x":"1","y":"0"},"value":5.0},
+                    {"address":{"x":"1","y":"1"},"value":7.0}
+                ]
+                """;
+        byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true);
+        assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded));
+    }
+
+    @Test
+    public void testMixedTensor() {
+        Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y[2])"));
+        builder.cell().label("x", "a").label("y", 0).value(2.0);
+        builder.cell().label("x", "a").label("y", 1).value(3.0);
+        builder.cell().label("x", "b").label("y", 0).value(5.0);
+        builder.cell().label("x", "b").label("y", 1).value(7.0);
+        Tensor tensor = builder.build();
+
+        String shortJson = """
+                {
+                  "type":"tensor(x{},y[2])",
+                  "blocks":{"a":[2.0,3.0],"b":[5.0,7.0]}
+                }
+                """;
+        byte[] shortEncoded = JsonFormat.encode(tensor, true, false);
+        assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded));
+
+        String longJson = """
+                {
+                  "type":"tensor(x{},y[2])",
+                  "cells":[
+                    {"address":{"x":"a","y":"0"},"value":2.0},
+                    {"address":{"x":"a","y":"1"},"value":3.0},
+                    {"address":{"x":"b","y":"0"},"value":5.0},
+                    {"address":{"x":"b","y":"1"},"value":7.0}
+                  ]
+                }
+                """;
+        byte[] longEncoded = JsonFormat.encode(tensor, false, false);
+        assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded));
+
+        String shortDirectJson = """
+                {"a":[2.0,3.0],"b":[5.0,7.0]}
+                """;
+        byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true);
+        assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded));
+
+        String longDirectJson = """
+                [
+                    {"address":{"x":"a","y":"0"},"value":2.0},
+                    {"address":{"x":"a","y":"1"},"value":3.0},
+                    {"address":{"x":"b","y":"0"},"value":5.0},
+                    {"address":{"x":"b","y":"1"},"value":7.0}
+                ]
+                """;
+        byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true);
+        assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded));
+    }
+
+    @Test
+    public void testSparseTensor() {
+        Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})"));
+        builder.cell().label("x", "a").label("y", 0).value(2.0);
+        builder.cell().label("x", "a").label("y", 1).value(3.0);
+        builder.cell().label("x", "b").label("y", 0).value(5.0);
+        builder.cell().label("x", "b").label("y", 1).value(7.0);
+        Tensor tensor = builder.build();
+
+        String shortJson = """
+                {
+                  "type":"tensor(x{},y{})",
+                   "cells": [
+                     {"address":{"x":"a","y":"0"},"value":2.0},
+                     {"address":{"x":"a","y":"1"},"value":3.0},
+                     {"address":{"x":"b","y":"0"},"value":5.0},
+                     {"address":{"x":"b","y":"1"},"value":7.0}
+                   ]
+                }
+                """;
+        byte[] shortEncoded = JsonFormat.encode(tensor, true, false);
+        assertEqualJson(shortJson, new String(shortEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortEncoded));
+
+        String longJson = """
+                {
+                  "type":"tensor(x{},y{})",
+                  "cells":[
+                    {"address":{"x":"a","y":"0"},"value":2.0},
+                    {"address":{"x":"a","y":"1"},"value":3.0},
+                    {"address":{"x":"b","y":"0"},"value":5.0},
+                    {"address":{"x":"b","y":"1"},"value":7.0}
+                  ]
+                }
+                """;
+        byte[] longEncoded = JsonFormat.encode(tensor, false, false);
+        assertEqualJson(longJson, new String(longEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longEncoded));
+
+        String shortDirectJson = """
+                [
+                  {"address":{"x":"a","y":"0"},"value":2.0},
+                  {"address":{"x":"a","y":"1"},"value":3.0},
+                  {"address":{"x":"b","y":"0"},"value":5.0},
+                  {"address":{"x":"b","y":"1"},"value":7.0}
+                ]
+                """;
+        byte[] shortDirectEncoded = JsonFormat.encode(tensor, true, true);
+        assertEqualJson(shortDirectJson, new String(shortDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), shortDirectEncoded));
+
+        String longDirectJson = """
+                [
+                    {"address":{"x":"a","y":"0"},"value":2.0},
+                    {"address":{"x":"a","y":"1"},"value":3.0},
+                    {"address":{"x":"b","y":"0"},"value":5.0},
+                    {"address":{"x":"b","y":"1"},"value":7.0}
+                ]
+                """;
+        byte[] longDirectEncoded = JsonFormat.encode(tensor, false, true);
+        assertEqualJson(longDirectJson, new String(longDirectEncoded, StandardCharsets.UTF_8));
+        assertEquals(tensor, JsonFormat.decode(tensor.type(), longDirectEncoded));
     }
 
     @Test
     public void testDisallowedEmptyDenseTensor() {
         TensorType type = TensorType.fromSpec("tensor(x[3])");
-        assertDecodeFails(type, "{\"values\":[]}", "The 'values' array does not contain any values");
-        assertDecodeFails(type, "{\"values\":\"\"}", "The 'values' string does not contain any values");
+        assertDecodeFails(type, "{\"values\":[]}", "The values array does not contain any values");
+        assertDecodeFails(type, "{\"values\":\"\"}", "The values string does not contain any values");
     }
 
     @Test
     public void testDisallowedEmptyMixedTensor() {
         TensorType type = TensorType.fromSpec("tensor(x{},y[3])");
-        assertDecodeFails(type, "{\"blocks\":{ \"a\": [] } }", "The 'block' value array does not contain any values");
+        assertDecodeFails(type, "{\"blocks\":{ \"a\": [] } }", "The block value array does not contain any values");
         assertDecodeFails(type, "{\"blocks\":[ {\"address\":{\"x\":\"a\"}, \"values\": [] } ] }",
-                "The 'block' value array does not contain any values");
+                "The block value array does not contain any values");
     }
 
     @Test
@@ -426,8 +603,12 @@ public class JsonFormatTestCase {
             Tensor decoded = JsonFormat.decode(type, format.getBytes(StandardCharsets.UTF_8));
             fail("Did not get exception as expected, decoded as: " + decoded);
         } catch (IllegalArgumentException e) {
-            assertEquals(e.getMessage(), msg);
+            assertEquals(msg, e.getMessage());
         }
     }
 
+    private void assertEqualJson(String expected, String generated) {
+        Assertions.assertEquals(JSON.canonical(expected), JSON.canonical(generated));
+    }
+
 }