Revert "Revert "Add (disabled) dense tensor binary format""

6 files changed, 100 insertions, 75 deletions

diff --git a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java
index 3f7f02c6c00..e150b1cf24f 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java
@@ -1,5 +1,6 @@
 package com.yahoo.tensor;
 
+import junit.framework.TestCase;
 import org.junit.Test;
 
 import java.util.HashMap;
@@ -7,6 +8,7 @@ import java.util.Iterator;
 import java.util.Map;
 
 import static junit.framework.TestCase.assertTrue;
+import static junit.framework.TestCase.fail;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -23,16 +25,12 @@ public class IndexedTensorTestCase {
     @Test
     public void testEmpty() {
         Tensor empty = Tensor.Builder.of(TensorType.empty).build();
-        assertTrue(empty instanceof IndexedTensor);
-        assertTrue(empty.isEmpty());
-        assertEquals("{}", empty.toString());
+        assertEquals(1, empty.size());
+        assertEquals((double)0.0, (double)empty.valueIterator().next(), 0.00000001);
         Tensor emptyFromString = Tensor.from(TensorType.empty, "{}");
-        assertEquals("{}", Tensor.from(TensorType.empty, "{}").toString());
-        assertTrue(emptyFromString.isEmpty());
-        assertTrue(emptyFromString instanceof IndexedTensor);
         assertEquals(empty, emptyFromString);
     }
-    
+
     @Test
     public void testSingleValue() {
         Tensor singleValue = Tensor.Builder.of(TensorType.empty).cell(TensorAddress.empty, 3.5).build();
@@ -45,22 +43,6 @@ public class IndexedTensorTestCase {
     }
     
     @Test
-    public void testSingleValueWithDimensions() {
-        TensorType type = new TensorType.Builder().indexed("x").indexed("y").build();
-        Tensor emptyWithDimensions = Tensor.Builder.of(type).build();
-        assertTrue(emptyWithDimensions instanceof IndexedTensor);
-        assertEquals("tensor(x[],y[]):{}", emptyWithDimensions.toString());
-        Tensor emptyWithDimensionsFromString = Tensor.from("tensor(x[],y[]):{}");
-        assertEquals("tensor(x[],y[]):{}", emptyWithDimensionsFromString.toString());
-        assertTrue(emptyWithDimensionsFromString instanceof IndexedTensor);
-        assertEquals(emptyWithDimensions, emptyWithDimensionsFromString);
-
-        IndexedTensor emptyWithDimensionsIndexed = (IndexedTensor)emptyWithDimensions;
-        assertEquals(0, emptyWithDimensionsIndexed.dimensionSizes().size(0));
-        assertEquals(0, emptyWithDimensionsIndexed.dimensionSizes().size(1));
-    }
-    
-    @Test
     public void testBoundBuilding() {
         TensorType type = new TensorType.Builder().indexed("v", vSize)
                                                   .indexed("w", wSize)
@@ -91,7 +73,7 @@ public class IndexedTensorTestCase {
                         for (int z = 0; z < zSize; z++)
                             builder.cell(value(v, w, x, y, z), v, w, x, y, z);
 
-        IndexedTensor tensor = builder.build();
+        IndexedTensor tensor = (IndexedTensor)builder.build();
 
         // Lookup by index arguments
         for (int v = 0; v < vSize; v++)
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java
index 4c32a80dc11..5c2c3b9db32 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java
@@ -2,6 +2,7 @@
 package com.yahoo.tensor;
 
 import com.google.common.collect.Sets;
+import junit.framework.TestCase;
 import org.junit.Test;
 
 import java.util.Set;
@@ -18,6 +19,20 @@ import static org.junit.Assert.fail;
 public class MappedTensorTestCase {
 
     @Test
+    public void testEmpty() {
+        TensorType type = new TensorType.Builder().mapped("x").build();
+        Tensor empty = Tensor.Builder.of(type).build();
+        TestCase.assertTrue(empty instanceof MappedTensor);
+        TestCase.assertTrue(empty.isEmpty());
+        assertEquals("tensor(x{}):{}", empty.toString());
+        Tensor emptyFromString = Tensor.from(type, "{}");
+        assertEquals("tensor(x{}):{}", Tensor.from("tensor(x{}):{}").toString());
+        TestCase.assertTrue(emptyFromString.isEmpty());
+        TestCase.assertTrue(emptyFromString instanceof MappedTensor);
+        assertEquals(empty, emptyFromString);
+    }
+
+    @Test
     public void testOneDimensionalBuilding() {
         TensorType type = new TensorType.Builder().mapped("x").build();
         Tensor tensor = Tensor.Builder.of(type).
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
index 2f060239eb1..e2baa1d5ac3 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
@@ -27,6 +27,7 @@ public class TensorFunctionBenchmark {
             modelVectors = modelVectors.stream().map(t -> t.multiply(unitVector("k"))).collect(Collectors.toList());
         }
         dotProduct(queryVector, modelVectors, Math.max(iterations/10, 10)); // warmup
+        System.gc();
         long startTime = System.currentTimeMillis();
         dotProduct(queryVector, modelVectors, iterations);
         long totalTime = System.currentTimeMillis() - startTime;
@@ -106,51 +107,41 @@ public class TensorFunctionBenchmark {
 
         // ---------------- Mapped with extra space (sidesteps current special-case optimizations):
         // 410 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(20, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true);
         System.out.printf("Mapped vectors, x space  time per join: %1$8.3f ms\n", time);
         // 770 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(20, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true);
         System.out.printf("Mapped matrix, x space   time per join: %1$8.3f ms\n", time);
 
         // ---------------- Mapped:
         // 2.6 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(5000, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false);
         System.out.printf("Mapped vectors,          time per join: %1$8.3f ms\n", time);
         // 6.8 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(1000, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false);
         System.out.printf("Mapped matrix,           time per join: %1$8.3f ms\n", time);
 
         // ---------------- Indexed (unbound) with extra space (sidesteps current special-case optimizations):
         // 30 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(500, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true);
         System.out.printf("Indexed vectors, x space time per join: %1$8.3f ms\n", time);
         // 27 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true);
         System.out.printf("Indexed matrix, x space  time per join: %1$8.3f ms\n", time);
 
         // ---------------- Indexed unbound:
         // 0.14 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false);
         System.out.printf("Indexed unbound vectors, time per join: %1$8.3f ms\n", time);
         // 0.14 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false);
         System.out.printf("Indexed unbound matrix,  time per join: %1$8.3f ms\n", time);
 
         // ---------------- Indexed bound:
         // 0.14 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, vectors(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false);
         System.out.printf("Indexed bound vectors,   time per join: %1$8.3f ms\n", time);
         // 0.14 ms
-        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, matrix(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false);
         System.out.printf("Indexed bound matrix,    time per join: %1$8.3f ms\n", time);
     }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
index feeba1a7a10..b35220cf013 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
- * Tests Tensor functionality
+ * Tests tensor functionality
  * 
  * @author bratseth
  */
@@ -29,7 +29,8 @@ public class TensorTestCase {
 
     @Test
     public void testStringForm() {
-        assertEquals("{}", Tensor.from("{}").toString());
+        assertEquals("{5.7}", Tensor.from("{5.7}").toString());
+        assertTrue(Tensor.from("{5.7}") instanceof IndexedTensor);
         assertEquals("{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}", Tensor.from("{ {d1:l1,d2:l1}: 5,   {d2:l2, d1:l1}:6.0} ").toString());
         assertEquals("{{d1:l1,d2:l1}:-5.3,{d1:l1,d2:l2}:0.0}", Tensor.from("{ {d1:l1,d2:l1}:-5.3, {d2:l2, d1:l1}:0}").toString());
     }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java
new file mode 100644
index 00000000000..d2b2044f3ed
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.tensor.serialization;
+
+import com.google.common.collect.Sets;
+import com.yahoo.tensor.Tensor;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for the dense binary format.
+ *
+ * @author bratseth
+ */
+public class DenseBinaryFormatTestCase {
+
+    @Test
+    public void testSerialization() {
+        assertSerialization("{-5.37}");
+        assertSerialization("tensor(x[]):{{x:0}:2.0}");
+        assertSerialization("tensor(x[],y[]):{{x:0,y:0}:2.0}");
+        assertSerialization("tensor(x[],y[]):{{x:0,y:0}:2.0, {x:0,y:1}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0}");
+        assertSerialization("tensor(x[1],y[2],z[3]):{{y:0,x:0,z:0}:2.0}");
+    }
+
+    @Test
+    @Ignore // TODO: Activate when encoding in this format is activated
+    public void requireThatSerializationFormatDoNotChange() {
+        byte[] encodedTensor = new byte[]{2, // binary format type
+                                          2, // dimension count
+                                          2, (byte) 'x', (byte) 'y', 2, // dimension xy with size
+                                          1, (byte) 'z', 1, // dimension z with size
+                                          64, 0, 0, 0, 0, 0, 0, 0, // value 1
+                                          64, 8, 0, 0, 0, 0, 0, 0  // value 2
+        };
+        assertEquals(Arrays.toString(encodedTensor),
+                     Arrays.toString(TypedBinaryFormat.encode(Tensor.from("tensor(xy[],z[]):{{xy:0,z:0}:2.0,{xy:1,z:0}:3.0}"))));
+    }
+
+    private void assertSerialization(String tensorString) {
+        assertSerialization(Tensor.from(tensorString));
+    }
+
+    private void assertSerialization(Tensor tensor) {
+        byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
+        Tensor decodedTensor = TypedBinaryFormat.decode(tensor.type(), encodedTensor);
+        assertEquals(tensor, decodedTensor);
+    }
+
+}
+
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java
index ad908101329..283aa90cf65 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java
@@ -13,52 +13,23 @@ import static org.junit.Assert.assertEquals;
 /**
  * Tests for the sparse binary format.
  *
- * TODO: When new formats are added we should refactor this test to test all formats
- *       with the same set of tensor inputs (if feasible).
- *
  * @author geirst
  */
 public class SparseBinaryFormatTestCase {
 
-    private static void assertSerialization(String tensorString) {
-        assertSerialization(Tensor.from(tensorString));
-    }
-
-    private static void assertSerialization(String tensorString, Set<String> dimensions) {
-        Tensor tensor = Tensor.from(tensorString);
-        assertEquals(dimensions, tensor.type().dimensionNames());
-        assertSerialization(tensor);
-    }
-
-    private static void assertSerialization(Tensor tensor) {
-        byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
-        Tensor decodedTensor = TypedBinaryFormat.decode(encodedTensor);
-        assertEquals(tensor, decodedTensor);
-    }
-
     @Test
-    public void testSerializationOfTensorsWithDenseTensorAddresses() {
-        assertSerialization("{}");
-        assertSerialization("{{x:0}:2.0}");
-        assertSerialization("{{x:0}:2.0,{x:1}:3.0}");
-        assertSerialization("{{x:0,y:0}:2.0}");
-        assertSerialization("{{x:0,y:0}:2.0,{x:0,y:1}:3.0}");
-        assertSerialization("{{y:0,x:0}:2.0}");
-        assertSerialization("{{y:0,x:0}:2.0,{y:1,x:0}:3.0}");
-        assertSerialization("{{dimX:labelA,dimY:labelB}:2.0,{dimY:labelC,dimX:labelD}:3.0}");
+    public void testSerialization() {
+        assertSerialization("tensor(x{}):{}");
+        assertSerialization("tensor(x{}):{{x:0}:2.0}");
+        assertSerialization("tensor(dimX{},dimY{}):{{dimX:labelA,dimY:labelB}:2.0,{dimY:labelC,dimX:labelD}:3.0}");
+        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0}");
+        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0,{x:1,y:4}:3.0}");
+        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0}");
+        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0,{y:1,x:0,z:6}:3.0}");
     }
 
     @Test
-    public void testSerializationOfTensorsWithSparseTensorAddresses() {
-        assertSerialization("{{x:0}:2.0, {x:1}:3.0}", Sets.newHashSet("x"));
-        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0}", Sets.newHashSet("x", "y"));
-        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0,{x:1,y:4}:3.0}", Sets.newHashSet("x", "y"));
-        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0}", Sets.newHashSet("x", "y", "z"));
-        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0,{y:1,x:0,z:6}:3.0}", Sets.newHashSet("x", "y", "z"));
-    }
-
-    @Test
-    public void requireThatCompactSerializationFormatDoNotChange() {
+    public void requireThatSerializationFormatDoNotChange() {
         byte[] encodedTensor = new byte[] {1, // binary format type
                 2, // num dimensions
                 2, (byte)'x', (byte)'y', 1, (byte)'z', // dimensions
@@ -66,7 +37,17 @@ public class SparseBinaryFormatTestCase {
                 2, (byte)'a', (byte)'b', 1, (byte)'e', 64, 0, 0, 0, 0, 0, 0, 0, // cell 0
                 2, (byte)'c', (byte)'d', 1, (byte)'e', 64, 8, 0, 0, 0, 0, 0, 0}; // cell 1
         assertEquals(Arrays.toString(encodedTensor),
-                Arrays.toString(TypedBinaryFormat.encode(Tensor.from("{{xy:ab,z:e}:2.0,{xy:cd,z:e}:3.0}"))));
+                Arrays.toString(TypedBinaryFormat.encode(Tensor.from("tensor(xy{},z{}):{{xy:ab,z:e}:2.0,{xy:cd,z:e}:3.0}"))));
+    }
+
+    private void assertSerialization(String tensorString) {
+        assertSerialization(Tensor.from(tensorString));
+    }
+
+    private void assertSerialization(Tensor tensor) {
+        byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
+        Tensor decodedTensor = TypedBinaryFormat.decode(tensor.type(), encodedTensor);
+        assertEquals(tensor, decodedTensor);
     }
 
 }