Better benchmarking

2 files changed, 75 insertions, 27 deletions

diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
index dc0b3c47c62..989180288ac 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorFunctionBenchmark.java
@@ -8,6 +8,7 @@ import com.yahoo.tensor.functions.Reduce;
 import com.yahoo.tensor.functions.TensorFunction;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * Microbenchmark of tensor operations.
@@ -18,14 +19,24 @@ public class TensorFunctionBenchmark {
 
     private final static Random random = new Random();
     
-    public double benchmark(int iterations, List<Tensor> modelVectors, TensorType.Dimension.Type dimensionType) {
+    public double benchmark(int iterations, List<Tensor> modelVectors, TensorType.Dimension.Type dimensionType, 
+                            boolean extraSpace) {
         Tensor queryVector = vectors(1, 300, dimensionType).get(0);
+        if (extraSpace) {
+            queryVector = queryVector.multiply(unitVector("j"));
+            modelVectors = modelVectors.stream().map(t -> t.multiply(unitVector("k"))).collect(Collectors.toList());
+        }
         dotProduct(queryVector, modelVectors, Math.max(iterations/10, 10)); // warmup
         long startTime = System.currentTimeMillis();
         dotProduct(queryVector, modelVectors, iterations);
         long totalTime = System.currentTimeMillis() - startTime;
         return (double)totalTime / (double)iterations;
     }
+    
+    private Tensor unitVector(String dimension) {
+        return Tensor.Builder.of(new TensorType.Builder().indexed(dimension, 1).build())
+                .cell().label(dimension, 0).value(1).build();
+    }
 
     private double dotProduct(Tensor tensor, List<Tensor> tensors, int iterations) {
         double result = 0;
@@ -53,7 +64,7 @@ public class TensorFunctionBenchmark {
 
     private static List<Tensor> vectors(int vectorCount, int vectorSize, TensorType.Dimension.Type dimensionType) {
         List<Tensor> tensors = new ArrayList<>();
-        TensorType type = new TensorType.Builder().dimension("x", dimensionType).build();
+        TensorType type = vectorType(new TensorType.Builder(), "x", dimensionType, vectorSize);
         for (int i = 0; i < vectorCount; i++) {
             Tensor.Builder builder = Tensor.Builder.of(type);
             for (int j = 0; j < vectorSize; j++) {
@@ -65,8 +76,10 @@ public class TensorFunctionBenchmark {
     }
 
     private static List<Tensor> matrix(int vectorCount, int vectorSize, TensorType.Dimension.Type dimensionType) {
-        TensorType type = new TensorType.Builder().dimension("i", dimensionType).dimension("x", dimensionType).build();
-        Tensor.Builder builder = Tensor.Builder.of(type);
+        TensorType.Builder typeBuilder = new TensorType.Builder();
+        typeBuilder.dimension("i", dimensionType == TensorType.Dimension.Type.indexedBound ? TensorType.Dimension.Type.indexedUnbound : dimensionType);
+        vectorType(typeBuilder, "x", dimensionType, vectorSize);
+        Tensor.Builder builder = Tensor.Builder.of(typeBuilder.build());
         for (int i = 0; i < vectorCount; i++) {
             for (int j = 0; j < vectorSize; j++) {
                 builder.cell()
@@ -78,33 +91,68 @@ public class TensorFunctionBenchmark {
         return Collections.singletonList(builder.build());
     }
 
+    private static TensorType vectorType(TensorType.Builder builder, String name, TensorType.Dimension.Type type, int size) {
+        switch (type) {
+            case mapped: builder.mapped(name); break;
+            case indexedUnbound: builder.indexed(name); break;
+            case indexedBound: builder.indexed(name, size); break;
+            default: throw new IllegalArgumentException("Dimension type " + type + " not supported");
+        }
+        return builder.build();
+    }
+
     public static void main(String[] args) {
         double time;
-        
+
+        // ---------------- Mapped with extra space (sidesteps current special-case optimizations):
+        // Initial: 450 ms
+        time = new TensorFunctionBenchmark().benchmark(20, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true);
+        System.out.printf("Mapped vectors, x space  time per join: %1$8.3f ms\n", time);
+        // Initial: 1000 ms
+        time = new TensorFunctionBenchmark().benchmark(20, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, true);
+        System.out.printf("Mapped matrix, x space   time per join: %1$8.3f ms\n", time);
+
         // ---------------- Mapped:
         // Initial: 150 ms
         // - After adding type: 300 ms
         // - After sorting dimensions: 100 ms
         // - After special-casing single space: 2.4 ms
-        time = new TensorFunctionBenchmark().benchmark(5000, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped);
-        System.out.printf("Mapped vectors,  time per join: %1$8.3f ms\n", time);
+        time = new TensorFunctionBenchmark().benchmark(5000, vectors(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false);
+        System.out.printf("Mapped vectors,          time per join: %1$8.3f ms\n", time);
         // Initial: 760 ms
         // - After special-casing subspace: 15 ms
-        time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped);
-        System.out.printf("Mapped matrix,   time per join: %1$8.3f ms\n", time);
+        time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.mapped), TensorType.Dimension.Type.mapped, false);
+        System.out.printf("Mapped matrix,           time per join: %1$8.3f ms\n", time);
 
-        // ---------------- Indexed:
+        // ---------------- Indexed (unbound) with extra space (sidesteps current special-case optimizations):
+        // Initial: 1900 ms
+        time = new TensorFunctionBenchmark().benchmark(20, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true);
+        System.out.printf("Indexed vectors, x space time per join: %1$8.3f ms\n", time);
+        // Initial: 2200 ms
+        time = new TensorFunctionBenchmark().benchmark(20, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, true);
+        System.out.printf("Indexed matrix, x space  time per join: %1$8.3f ms\n", time);
+
+        // ---------------- Indexed unbound:
         // Initial: 718 ms
         // - After special casing join: 3.6 ms
         // - After special-casing reduce: 0.80 ms
-        // - After create IndexedTensor without builder: 0.41 ms
-        // - After double-array backing: 0.09 ms
-        time = new TensorFunctionBenchmark().benchmark(10000, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound);
-        System.out.printf("Indexed vectors, time per join: %1$8.3f ms\n", time);
+        // - After create IndexedTensor without builder: 0.4 ms
+        // - After double-array backing: 0.1 ms
+        time = new TensorFunctionBenchmark().benchmark(10000, vectors(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false);
+        System.out.printf("Indexed unbound vectors, time per join: %1$8.3f ms\n", time);
         // Initial: 3500 ms
         // - After special-casing subspace: 28 ms
-        time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound);
-        System.out.printf("Indexed matrix,  time per join: %1$8.3f ms\n", time);
+        time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.indexedUnbound), TensorType.Dimension.Type.indexedUnbound, false);
+        System.out.printf("Indexed unbound matrix,  time per join: %1$8.3f ms\n", time);
+
+        // ---------------- Indexed bound:
+        // Initial: 0.1 ms
+        time = new TensorFunctionBenchmark().benchmark(10000, vectors(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false);
+        System.out.printf("Indexed bound vectors,   time per join: %1$8.3f ms\n", time);
+        // Initial: 28 ms
+        time = new TensorFunctionBenchmark().benchmark(500, matrix(100, 300, TensorType.Dimension.Type.indexedBound), TensorType.Dimension.Type.indexedBound, false);
+        System.out.printf("Indexed bound matrix,    time per join: %1$8.3f ms\n", time);
+
     }
 
 }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
index f16c0df90fa..018c923fd4d 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorTestCase.java
@@ -74,7 +74,7 @@ public class TensorTestCase {
         assertEquals(Tensor.from("{ {y:1,x:1}:8, {x:1,y:2}:12 }"), tensor1.add(tensor2).rename(ImmutableList.of("x", "y"),
                                                                                                ImmutableList.of("y", "x")));
     }
-    
+
     /** Test the same computation made in various ways which are implemented with special-cvase optimizations */
     @Test
     public void testOptimizedComputation() {
@@ -135,16 +135,6 @@ public class TensorTestCase {
         return tensors;
     }
     
-    private TensorType vectorType(TensorType.Builder builder, String name, TensorType.Dimension.Type type, int size) {
-        switch (type) {
-            case mapped: builder.mapped(name); break;
-            case indexedUnbound: builder.indexed(name); break;
-            case indexedBound: builder.indexed(name, size); break;
-            default: throw new IllegalArgumentException("Dimension type " + type + " not supported");
-        }
-        return builder.build();
-    }
-    
     /** 
      * Create a matrix of vectors (in dimension i) where each vector has the dimension x.
      * This matrix contains the same vectors as returned by createVectors, in a single list element for convenience.
@@ -166,4 +156,14 @@ public class TensorTestCase {
         return Collections.singletonList(builder.build());
     }
 
+    private TensorType vectorType(TensorType.Builder builder, String name, TensorType.Dimension.Type type, int size) {
+        switch (type) {
+            case mapped: builder.mapped(name); break;
+            case indexedUnbound: builder.indexed(name); break;
+            case indexedBound: builder.indexed(name, size); break;
+            default: throw new IllegalArgumentException("Dimension type " + type + " not supported");
+        }
+        return builder.build();
+    }
+
 }