Always serialize dynamic tensors in verbose form

11 files changed, 93 insertions, 7 deletions

diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg
index 0c556aad868..2e0a207d249 100644
--- a/config-model/src/test/derived/tensor/attributes.cfg
+++ b/config-model/src/test/derived/tensor/attributes.cfg
@@ -82,3 +82,24 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype "tensor<float>(x[10])"
 attribute[].imported false
+attribute[].name "f6"
+attribute[].datatype FLOAT
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg
index af1748e484e..72fae572b76 100644
--- a/config-model/src/test/derived/tensor/documenttypes.cfg
+++ b/config-model/src/test/derived/tensor/documenttypes.cfg
@@ -40,6 +40,10 @@ documenttype[].datatype[].sstruct.field[].name "f5"
 documenttype[].datatype[].sstruct.field[].id 329055840
 documenttype[].datatype[].sstruct.field[].datatype 21
 documenttype[].datatype[].sstruct.field[].detailedtype "tensor<float>(x[10])"
+documenttype[].datatype[].sstruct.field[].name "f6"
+documenttype[].datatype[].sstruct.field[].id 596352344
+documenttype[].datatype[].sstruct.field[].datatype 1
+documenttype[].datatype[].sstruct.field[].detailedtype ""
 documenttype[].datatype[].id -1903234535
 documenttype[].datatype[].type STRUCT
 documenttype[].datatype[].array.element.id 0
@@ -60,3 +64,4 @@ documenttype[].fieldsets{[document]}.fields[] "f2"
 documenttype[].fieldsets{[document]}.fields[] "f3"
 documenttype[].fieldsets{[document]}.fields[] "f4"
 documenttype[].fieldsets{[document]}.fields[] "f5"
+documenttype[].fieldsets{[document]}.fields[] "f6"
diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg
index 1ce9227d323..d28746bd520 100644
--- a/config-model/src/test/derived/tensor/rank-profiles.cfg
+++ b/config-model/src/test/derived/tensor/rank-profiles.cfg
@@ -80,3 +80,16 @@ rankprofile[].fef.property[].name "vespa.type.attribute.f4"
 rankprofile[].fef.property[].value "tensor(x[10],y[20])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f5"
 rankprofile[].fef.property[].value "tensor<float>(x[10])"
+rankprofile[].name "profile5"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "rankingExpression(firstphase)"
+rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(tensor<float>(d0[1],x[2]):{{d0:0,x:0}:attribute(f6),{d0:0,x:1}:reduce(attribute(f5), sum)}, sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f2"
+rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
+rankprofile[].fef.property[].name "vespa.type.attribute.f3"
+rankprofile[].fef.property[].value "tensor(x{})"
+rankprofile[].fef.property[].name "vespa.type.attribute.f4"
+rankprofile[].fef.property[].value "tensor(x[10],y[20])"
+rankprofile[].fef.property[].name "vespa.type.attribute.f5"
+rankprofile[].fef.property[].value "tensor<float>(x[10])"
diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg
index 903b6033297..fb32eacbb4c 100644
--- a/config-model/src/test/derived/tensor/summary.cfg
+++ b/config-model/src/test/derived/tensor/summary.cfg
@@ -15,7 +15,7 @@ classes[].fields[].name "summaryfeatures"
 classes[].fields[].type "featuredata"
 classes[].fields[].name "documentid"
 classes[].fields[].type "longstring"
-classes[].id 193983608
+classes[].id 1476352352
 classes[].name "attributeprefetch"
 classes[].fields[].name "f2"
 classes[].fields[].type "tensor"
@@ -25,6 +25,8 @@ classes[].fields[].name "f4"
 classes[].fields[].type "tensor"
 classes[].fields[].name "f5"
 classes[].fields[].type "tensor"
+classes[].fields[].name "f6"
+classes[].fields[].type "float"
 classes[].fields[].name "rankfeatures"
 classes[].fields[].type "featuredata"
 classes[].fields[].name "summaryfeatures"
diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd
index b31352a2105..fa139d1e769 100644
--- a/config-model/src/test/derived/tensor/tensor.sd
+++ b/config-model/src/test/derived/tensor/tensor.sd
@@ -17,6 +17,9 @@ search tensor {
     field f5 type tensor<float>(x[10]) {
       indexing: attribute | summary
     }
+    field f6 type float {
+      indexing: attribute
+    }
   }
   
   rank-profile profile1 {
@@ -55,4 +58,12 @@ search tensor {
   
   }
 
+  rank-profile profile5 {
+
+    first-phase {
+      expression: sum(tensor<float>(d0[1],x[2]):[[attribute(f6), sum(attribute(f5))]])
+    }
+
+  }
+
 }
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
index a41f24b3b8a..efebdb310f7 100755
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java
@@ -156,9 +156,9 @@ public class RankingExpressionTestCase {
                             "xw_plus_b(matmul(constant(tensor0), attribute(tensor1), x), attribute(tensor1), query(tensor2), y)");
         assertSerialization("tensor(x{}):{{x:a}:1 + 2 + 3,{x:b}:if (1 > 2, 3, 4),{x:c}:reduce(tensor0 * tensor1, sum)}",
                             "tensor(x{}):{ {x:a}:1+2+3, {x:b}:if(1>2,3,4), {x:c}:sum(tensor0*tensor1) }");
-        assertSerialization("tensor(x[3]):[1.0,2.0,3]",
+        assertSerialization("tensor(x[3]):{{x:0}:1.0,{x:1}:2.0,{x:2}:3}",
                             "tensor(x[3]):[1.0, 2.0, 3]");
-        assertSerialization("tensor(x[3]):[1.0,reduce(tensor0 * tensor1, sum),3]",
+        assertSerialization("tensor(x[3]):{{x:0}:1.0,{x:1}:reduce(tensor0 * tensor1, sum),{x:2}:3}",
                             "tensor(x[3]):[1.0, sum(tensor0*tensor1), 3]");
     }
 
diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json
index 6a93a17a8c1..8b87c5dc79f 100644
--- a/vespajlib/abi-spec.json
+++ b/vespajlib/abi-spec.json
@@ -707,6 +707,7 @@
       "final"
     ],
     "methods": [
+      "public static com.yahoo.tensor.DimensionSizes of(com.yahoo.tensor.TensorType)",
       "public long size(int)",
       "public int dimensions()",
       "public long totalSize()",
@@ -820,7 +821,9 @@
       "abstract"
     ],
     "methods": [
+      "public static com.yahoo.tensor.IndexedTensor$Indexes of(com.yahoo.tensor.TensorType)",
       "public static com.yahoo.tensor.IndexedTensor$Indexes of(com.yahoo.tensor.DimensionSizes)",
+      "public com.yahoo.tensor.TensorAddress toAddress()",
       "public long[] indexesCopy()",
       "public long[] indexesForReading()",
       "public java.util.List toList()",
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java
index c0d817459d0..d81c02fb75f 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/DimensionSizes.java
@@ -18,6 +18,21 @@ public final class DimensionSizes {
     }
 
     /**
+     * Create sizes from a type containing bound indexed dimensions only.
+     *
+     * @throws IllegalStateException if the type contains dimensions which are not bound and indexed
+     */
+    public static DimensionSizes of(TensorType type) {
+        Builder b = new Builder(type.rank());
+        for (int i = 0; i < type.rank(); i++) {
+            if ( type.dimensions().get(i).type() != TensorType.Dimension.Type.indexedBound)
+                throw new IllegalArgumentException(type + " contains dimensions without a size");
+            b.set(i, type.dimensions().get(i).size().get());
+        }
+        return b.build();
+    }
+
+    /**
      * Returns the length of this in the nth dimension
      *
      * @throws IllegalArgumentException if the index is larger than the number of dimensions in this tensor minus one
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
index 15476567fb2..176ddfefc13 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
@@ -758,6 +758,15 @@ public abstract class IndexedTensor implements Tensor {
 
         protected final long[] indexes;
 
+        /**
+         * Create indexes from a type containing bound indexed dimensions only.
+         *
+         * @throws IllegalStateException if the type contains dimensions which are not bound and indexed
+         */
+        public static Indexes of(TensorType type) {
+            return of(DimensionSizes.of(type));
+        }
+
         public static Indexes of(DimensionSizes sizes) {
             return of(sizes, sizes);
         }
@@ -824,7 +833,7 @@ public abstract class IndexedTensor implements Tensor {
         }
 
         /** Returns the address of the current position of these indexes */
-        private TensorAddress toAddress() {
+        public TensorAddress toAddress() {
             return TensorAddress.of(indexes);
         }
 
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java
index 9ce2496c65b..1154f962a6f 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java
@@ -129,14 +129,16 @@ public abstract class DynamicTensor extends PrimitiveTensorFunction {
                 return "{" + cells.get(0) + "}";
             }
 
-            StringBuilder b = new StringBuilder("[");
+            IndexedTensor.Indexes indexes = IndexedTensor.Indexes.of(type());
+            StringBuilder b = new StringBuilder("{");
             for (var cell : cells) {
-                b.append(cell);
+                indexes.next();
+                b.append(indexes.toAddress().toString(type())).append(":").append(cell);
                 b.append(",");
             }
             if (b.length() > 1)
                 b.setLength(b.length() - 1);
-            b.append("]");
+            b.append("}");
 
             return b.toString();
         }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/DynamicTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/DynamicTensorTestCase.java
index 82652fb0e5d..474e6200676 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/functions/DynamicTensorTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/DynamicTensorTestCase.java
@@ -24,12 +24,14 @@ public class DynamicTensorTestCase {
         DynamicTensor t1 = DynamicTensor.from(dense,
                                               List.of(new Constant(1), new Constant(2), new Constant(3)));
         assertEquals(Tensor.from(dense, "[1, 2, 3]"), t1.evaluate());
+        assertEquals("tensor(x[3]):{{x:0}:1.0,{x:1}:2.0,{x:2}:3.0}", t1.toString());
 
         TensorType sparse = TensorType.fromSpec("tensor(x{})");
         DynamicTensor t2 = DynamicTensor.from(sparse,
                                               Collections.singletonMap(new TensorAddress.Builder(sparse).add("x", "a").build(),
                                                                        new Constant(5)));
         assertEquals(Tensor.from(sparse, "{{x:a}:5}"), t2.evaluate());
+        assertEquals("tensor(x{}):{{x:a}:5.0}", t2.toString());
     }
 
     private static class Constant implements Function<EvaluationContext<?>, Double> {
@@ -41,6 +43,9 @@ public class DynamicTensorTestCase {
         @Override
         public Double apply(EvaluationContext<?> evaluationContext) { return value; }
 
+        @Override
+        public String toString() { return String.valueOf(value); }
+
     }
 
 }