Revert "Add (disabled) dense tensor binary format"

6 files changed, 75 insertions, 100 deletions

diff --git a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java
index e150b1cf24f..3f7f02c6c00 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/IndexedTensorTestCase.java
@@ -1,6 +1,5 @@
 package com.yahoo.tensor;
 
-import junit.framework.TestCase;
 import org.junit.Test;
 
 import java.util.HashMap;
@@ -8,7 +7,6 @@ import java.util.Iterator;
 import java.util.Map;
 
 import static junit.framework.TestCase.assertTrue;
-import static junit.framework.TestCase.fail;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -25,12 +23,16 @@ public class IndexedTensorTestCase {
     @Test
     public void testEmpty() {
         Tensor empty = Tensor.Builder.of(TensorType.empty).build();
-        assertEquals(1, empty.size());
-        assertEquals((double)0.0, (double)empty.valueIterator().next(), 0.00000001);
+        assertTrue(empty instanceof IndexedTensor);
+        assertTrue(empty.isEmpty());
+        assertEquals("{}", empty.toString());
         Tensor emptyFromString = Tensor.from(TensorType.empty, "{}");
+        assertEquals("{}", Tensor.from(TensorType.empty, "{}").toString());
+        assertTrue(emptyFromString.isEmpty());
+        assertTrue(emptyFromString instanceof IndexedTensor);
         assertEquals(empty, emptyFromString);
     }
-
+    
     @Test
     public void testSingleValue() {
         Tensor singleValue = Tensor.Builder.of(TensorType.empty).cell(TensorAddress.empty, 3.5).build();
@@ -43,6 +45,22 @@ public class IndexedTensorTestCase {
     }
     
     @Test
+    public void testSingleValueWithDimensions() {
+        TensorType type = new TensorType.Builder().indexed("x").indexed("y").build();
+        Tensor emptyWithDimensions = Tensor.Builder.of(type).build();
+        assertTrue(emptyWithDimensions instanceof IndexedTensor);
+        assertEquals("tensor(x[],y[]):{}", emptyWithDimensions.toString());
+        Tensor emptyWithDimensionsFromString = Tensor.from("tensor(x[],y[]):{}");
+        assertEquals("tensor(x[],y[]):{}", emptyWithDimensionsFromString.toString());
+        assertTrue(emptyWithDimensionsFromString instanceof IndexedTensor);
+        assertEquals(emptyWithDimensions, emptyWithDimensionsFromString);
+
+        IndexedTensor emptyWithDimensionsIndexed = (IndexedTensor)emptyWithDimensions;
+        assertEquals(0, emptyWithDimensionsIndexed.dimensionSizes().size(0));
+        assertEquals(0, emptyWithDimensionsIndexed.dimensionSizes().size(1));
+    }
+    
+    @Test
     public void testBoundBuilding() {
         TensorType type = new TensorType.Builder().indexed("v", vSize)
                                                   .indexed("w", wSize)
@@ -73,7 +91,7 @@ public class IndexedTensorTestCase {
                         for (int z = 0; z < zSize; z++)
                             builder.cell(value(v, w, x, y, z), v, w, x, y, z);
 
-        IndexedTensor tensor = (IndexedTensor)builder.build();
+        IndexedTensor tensor = builder.build();
 
         // Lookup by index arguments
         for (int v = 0; v < vSize; v++)
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java
index 5c2c3b9db32..4c32a80dc11 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/MappedTensorTestCase.java
@@ -2,7 +2,6 @@
 package com.yahoo.tensor;
 
 import com.google.common.collect.Sets;
-import junit.framework.TestCase;
 import org.junit.Test;
 
 import java.util.Set;
@@ -19,20 +18,6 @@ import static org.junit.Assert.fail;
 public class MappedTensorTestCase {
 
     @Test
-    public void testEmpty() {
-        TensorType type = new TensorType.Builder().mapped("x").build();
-        Tensor empty = Tensor.Builder.of(type).build();
-        TestCase.assertTrue(empty instanceof MappedTensor);
-        TestCase.assertTrue(empty.isEmpty());
-        assertEquals("tensor(x{}):{}", empty.toString());
-        Tensor emptyFromString = Tensor.from(type, "{}");
-        assertEquals("tensor(x{}):{}", Tensor.from("tensor(x{}):{}").toString());
-        TestCase.assertTrue(emptyFromString.isEmpty());
-        TestCase.assertTrue(emptyFromString instanceof MappedTensor);
-        assertEquals(empty, emptyFromString);
-    }
-
-    @Test
     public void testOneDimensionalBuilding() {
         TensorType type = new TensorType.Builder().mapped("x").build();
         Tensor tensor = Tensor.Builder.of(type).
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
index e2baa1d5ac3..2f060239eb1 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
@@ -27,7 +27,6 @@ public class TensorFunctionBenchmark {
             modelVectors = modelVectors.stream().map(t -> t.multiply(unitVector("k"))).collect(Collectors.toList());
         }
         dotProduct(queryVector, modelVectors, Math.max(iterations/10, 10)); // warmup
-        System.gc();
         long startTime = System.currentTimeMillis();
         dotProduct(queryVector, modelVectors, iterations);
         long totalTime = System.currentTimeMillis() - startTime;
@@ -107,41 +106,51 @@ public class TensorFunctionBenchmark {
 
         // ---------------- Mapped with extra space (sidesteps current special-case optimizations):
         // 410 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(20, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true);
         System.out.printf("Mapped vectors, x space  time per join: %1$8.3f ms\n", time);
         // 770 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(20, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true);
         System.out.printf("Mapped matrix, x space   time per join: %1$8.3f ms\n", time);
 
         // ---------------- Mapped:
         // 2.6 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(5000, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false);
         System.out.printf("Mapped vectors,          time per join: %1$8.3f ms\n", time);
         // 6.8 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(1000, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false);
         System.out.printf("Mapped matrix,           time per join: %1$8.3f ms\n", time);
 
         // ---------------- Indexed (unbound) with extra space (sidesteps current special-case optimizations):
         // 30 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(500, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true);
         System.out.printf("Indexed vectors, x space time per join: %1$8.3f ms\n", time);
         // 27 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true);
         System.out.printf("Indexed matrix, x space  time per join: %1$8.3f ms\n", time);
 
         // ---------------- Indexed unbound:
         // 0.14 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false);
         System.out.printf("Indexed unbound vectors, time per join: %1$8.3f ms\n", time);
         // 0.14 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false);
         System.out.printf("Indexed unbound matrix,  time per join: %1$8.3f ms\n", time);
 
         // ---------------- Indexed bound:
         // 0.14 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, vectors(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false);
         System.out.printf("Indexed bound vectors,   time per join: %1$8.3f ms\n", time);
         // 0.14 ms
+        System.gc();
         time = new TensorFunctionBenchmark().benchmark(50000, matrix(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false);
         System.out.printf("Indexed bound matrix,    time per join: %1$8.3f ms\n", time);
     }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
index b35220cf013..feeba1a7a10 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
@@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
- * Tests tensor functionality
+ * Tests Tensor functionality
  * 
  * @author bratseth
  */
@@ -29,8 +29,7 @@ public class TensorTestCase {
 
     @Test
     public void testStringForm() {
-        assertEquals("{5.7}", Tensor.from("{5.7}").toString());
-        assertTrue(Tensor.from("{5.7}") instanceof IndexedTensor);
+        assertEquals("{}", Tensor.from("{}").toString());
         assertEquals("{{d1:l1,d2:l1}:5.0,{d1:l1,d2:l2}:6.0}", Tensor.from("{ {d1:l1,d2:l1}: 5,   {d2:l2, d1:l1}:6.0} ").toString());
         assertEquals("{{d1:l1,d2:l1}:-5.3,{d1:l1,d2:l2}:0.0}", Tensor.from("{ {d1:l1,d2:l1}:-5.3, {d2:l2, d1:l1}:0}").toString());
     }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java
deleted file mode 100644
index d2b2044f3ed..00000000000
--- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.tensor.serialization;
-
-import com.google.common.collect.Sets;
-import com.yahoo.tensor.Tensor;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests for the dense binary format.
- *
- * @author bratseth
- */
-public class DenseBinaryFormatTestCase {
-
-    @Test
-    public void testSerialization() {
-        assertSerialization("{-5.37}");
-        assertSerialization("tensor(x[]):{{x:0}:2.0}");
-        assertSerialization("tensor(x[],y[]):{{x:0,y:0}:2.0}");
-        assertSerialization("tensor(x[],y[]):{{x:0,y:0}:2.0, {x:0,y:1}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0}");
-        assertSerialization("tensor(x[1],y[2],z[3]):{{y:0,x:0,z:0}:2.0}");
-    }
-
-    @Test
-    @Ignore // TODO: Activate when encoding in this format is activated
-    public void requireThatSerializationFormatDoNotChange() {
-        byte[] encodedTensor = new byte[]{2, // binary format type
-                                          2, // dimension count
-                                          2, (byte) 'x', (byte) 'y', 2, // dimension xy with size
-                                          1, (byte) 'z', 1, // dimension z with size
-                                          64, 0, 0, 0, 0, 0, 0, 0, // value 1
-                                          64, 8, 0, 0, 0, 0, 0, 0  // value 2
-        };
-        assertEquals(Arrays.toString(encodedTensor),
-                     Arrays.toString(TypedBinaryFormat.encode(Tensor.from("tensor(xy[],z[]):{{xy:0,z:0}:2.0,{xy:1,z:0}:3.0}"))));
-    }
-
-    private void assertSerialization(String tensorString) {
-        assertSerialization(Tensor.from(tensorString));
-    }
-
-    private void assertSerialization(Tensor tensor) {
-        byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
-        Tensor decodedTensor = TypedBinaryFormat.decode(tensor.type(), encodedTensor);
-        assertEquals(tensor, decodedTensor);
-    }
-
-}
-
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java
index 283aa90cf65..ad908101329 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java
@@ -13,23 +13,52 @@ import static org.junit.Assert.assertEquals;
 /**
  * Tests for the sparse binary format.
  *
+ * TODO: When new formats are added we should refactor this test to test all formats
+ *       with the same set of tensor inputs (if feasible).
+ *
  * @author geirst
  */
 public class SparseBinaryFormatTestCase {
 
+    private static void assertSerialization(String tensorString) {
+        assertSerialization(Tensor.from(tensorString));
+    }
+
+    private static void assertSerialization(String tensorString, Set<String> dimensions) {
+        Tensor tensor = Tensor.from(tensorString);
+        assertEquals(dimensions, tensor.type().dimensionNames());
+        assertSerialization(tensor);
+    }
+
+    private static void assertSerialization(Tensor tensor) {
+        byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
+        Tensor decodedTensor = TypedBinaryFormat.decode(encodedTensor);
+        assertEquals(tensor, decodedTensor);
+    }
+
     @Test
-    public void testSerialization() {
-        assertSerialization("tensor(x{}):{}");
-        assertSerialization("tensor(x{}):{{x:0}:2.0}");
-        assertSerialization("tensor(dimX{},dimY{}):{{dimX:labelA,dimY:labelB}:2.0,{dimY:labelC,dimX:labelD}:3.0}");
-        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0}");
-        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0,{x:1,y:4}:3.0}");
-        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0}");
-        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0,{y:1,x:0,z:6}:3.0}");
+    public void testSerializationOfTensorsWithDenseTensorAddresses() {
+        assertSerialization("{}");
+        assertSerialization("{{x:0}:2.0}");
+        assertSerialization("{{x:0}:2.0,{x:1}:3.0}");
+        assertSerialization("{{x:0,y:0}:2.0}");
+        assertSerialization("{{x:0,y:0}:2.0,{x:0,y:1}:3.0}");
+        assertSerialization("{{y:0,x:0}:2.0}");
+        assertSerialization("{{y:0,x:0}:2.0,{y:1,x:0}:3.0}");
+        assertSerialization("{{dimX:labelA,dimY:labelB}:2.0,{dimY:labelC,dimX:labelD}:3.0}");
     }
 
     @Test
-    public void requireThatSerializationFormatDoNotChange() {
+    public void testSerializationOfTensorsWithSparseTensorAddresses() {
+        assertSerialization("{{x:0}:2.0, {x:1}:3.0}", Sets.newHashSet("x"));
+        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0}", Sets.newHashSet("x", "y"));
+        assertSerialization("tensor(x{},y{}):{{x:0,y:1}:2.0,{x:1,y:4}:3.0}", Sets.newHashSet("x", "y"));
+        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0}", Sets.newHashSet("x", "y", "z"));
+        assertSerialization("tensor(x{},y{},z{}):{{y:0,x:0,z:3}:2.0,{y:1,x:0,z:6}:3.0}", Sets.newHashSet("x", "y", "z"));
+    }
+
+    @Test
+    public void requireThatCompactSerializationFormatDoNotChange() {
         byte[] encodedTensor = new byte[] {1, // binary format type
                 2, // num dimensions
                 2, (byte)'x', (byte)'y', 1, (byte)'z', // dimensions
@@ -37,17 +66,7 @@ public class SparseBinaryFormatTestCase {
                 2, (byte)'a', (byte)'b', 1, (byte)'e', 64, 0, 0, 0, 0, 0, 0, 0, // cell 0
                 2, (byte)'c', (byte)'d', 1, (byte)'e', 64, 8, 0, 0, 0, 0, 0, 0}; // cell 1
         assertEquals(Arrays.toString(encodedTensor),
-                Arrays.toString(TypedBinaryFormat.encode(Tensor.from("tensor(xy{},z{}):{{xy:ab,z:e}:2.0,{xy:cd,z:e}:3.0}"))));
-    }
-
-    private void assertSerialization(String tensorString) {
-        assertSerialization(Tensor.from(tensorString));
-    }
-
-    private void assertSerialization(Tensor tensor) {
-        byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
-        Tensor decodedTensor = TypedBinaryFormat.decode(tensor.type(), encodedTensor);
-        assertEquals(tensor, decodedTensor);
+                Arrays.toString(TypedBinaryFormat.encode(Tensor.from("{{xy:ab,z:e}:2.0,{xy:cd,z:e}:3.0}"))));
     }
 
 }