handle JSON null as NaN

2 files changed, 38 insertions, 6 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java
index 204c0331e3a..8cf36cbd038 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java
@@ -234,7 +234,7 @@ public class JsonFormat {
         TensorAddress address = decodeAddress(cell.field("address"), builder.type());
 
         Inspector value = cell.field("value");
-        if (value.type() == Type.STRING || value.type() == Type.LONG || value.type() == Type.DOUBLE) {
+        if (value.valid()) {
             builder.cell(address, decodeNumeric(value));
         } else {
             throw new IllegalArgumentException("Excepted a cell to contain a numeric value called 'value'");
@@ -269,7 +269,7 @@ public class JsonFormat {
         values.traverse((ArrayTraverser) (__, value) -> {
             if (value.type() == Type.ARRAY)
                 decodeNestedValues(value, builder, index);
-            else if (value.type() == Type.LONG || value.type() == Type.DOUBLE || value.type() == Type.STRING)
+            else if (value.type() == Type.LONG || value.type() == Type.DOUBLE || value.type() == Type.STRING || value.type() == Type.NIX)
                 indexedBuilder.cellByDirectIndex(index.next(), decodeNumeric(value));
             else
                 throw new IllegalArgumentException("Excepted the values array to contain numbers or nested arrays, not " + value.type());
@@ -446,14 +446,17 @@ public class JsonFormat {
         return new TensorAddress.Builder(type).add(type.dimensions().get(0).name(), label).build();
     }
 
-
     private static double decodeNumeric(Inspector numericField) {
+        if (numericField.type() == Type.DOUBLE || numericField.type() == Type.LONG) {
+            return numericField.asDouble();
+        }
         if (numericField.type() == Type.STRING) {
             return decodeNumberString(numericField.asString());
         }
-        if (numericField.type() != Type.LONG && numericField.type() != Type.DOUBLE)
-            throw new IllegalArgumentException("Excepted a number, not " + numericField.type());
-        return numericField.asDouble();
+        if (numericField.type() == Type.NIX) {
+            return Double.NaN;
+        }
+        throw new IllegalArgumentException("Excepted a number, not " + numericField.type());
     }
 
     public static double decodeNumberString(String input) {
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
index 66d3a0e824d..52f44f0ac42 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java
@@ -687,6 +687,35 @@ public class JsonFormatTestCase {
         assertEquals(0xFFF8000000000000L, Double.doubleToRawLongBits(JsonFormat.decodeNumberString("-NaN")));
     }
 
+    @Test
+    public void testWithNanVariants() {
+        TensorType x3 = TensorType.fromSpec("tensor(x[3])");
+        String json = "{\"cells\":[" +
+                      "{\"address\":{\"x\":\"0\"},\"value\":\"nan\"}," +
+                      "{\"address\":{\"x\":\"1\"},\"value\":null}," +
+                      "{\"address\":{\"x\":\"2\"},\"value\":\"+NaN\"}" +
+                      "]}";
+        var t = JsonFormat.decode(x3, json.getBytes(StandardCharsets.UTF_8));
+        checkThreeNans(t);
+        json = "['nan', null, '+NaN']";
+        t = JsonFormat.decode(x3, json.getBytes(StandardCharsets.UTF_8));
+        checkThreeNans(t);
+        json = "{'type':'tensor(x[3])','values':['nan', null, '+NaN']}";
+        t = JsonFormat.decode(x3, json.getBytes(StandardCharsets.UTF_8));
+        checkThreeNans(t);
+    }
+
+    private void checkThreeNans(Tensor t) {
+        final Double nan = Double.NaN;
+        int cnt = 0;
+        for (var iter = t.cellIterator(); iter.hasNext(); ) {
+            var cell = iter.next();
+            assertEquals(nan, cell.getValue());
+            ++cnt;
+        }
+        assertEquals(3, cnt);
+    }
+
     private void assertEncodeShortForm(String tensor, String expected) {
         assertEncodeShortForm(Tensor.from(tensor), expected);
     }