add "unpack_bits_from_int8" function

6 files changed, 108 insertions, 0 deletions

diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg
index 752e236bb19..d050d553416 100644
--- a/config-model/src/test/derived/tensor/attributes.cfg
+++ b/config-model/src/test/derived/tensor/attributes.cfg
@@ -143,3 +143,32 @@ attribute[].index.hnsw.enabled false
 attribute[].index.hnsw.maxlinkspernode 16
 attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "f7"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor<int8>(p{},x[5])"
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.multithreadedindexing true
diff --git a/config-model/src/test/derived/tensor/documentmanager.cfg b/config-model/src/test/derived/tensor/documentmanager.cfg
index bae2db34040..5676f60ef46 100644
--- a/config-model/src/test/derived/tensor/documentmanager.cfg
+++ b/config-model/src/test/derived/tensor/documentmanager.cfg
@@ -49,6 +49,7 @@ doctype[].fieldsets{[document]}.fields[] "f3"
 doctype[].fieldsets{[document]}.fields[] "f4"
 doctype[].fieldsets{[document]}.fields[] "f5"
 doctype[].fieldsets{[document]}.fields[] "f6"
+doctype[].fieldsets{[document]}.fields[] "f7"
 doctype[].tensortype[].idx 10017
 doctype[].tensortype[].detailedtype "tensor(x[3])"
 doctype[].tensortype[].idx 10018
@@ -59,6 +60,8 @@ doctype[].tensortype[].idx 10020
 doctype[].tensortype[].detailedtype "tensor(x[10],y[10])"
 doctype[].tensortype[].idx 10021
 doctype[].tensortype[].detailedtype "tensor<float>(x[10])"
+doctype[].tensortype[].idx 10022
+doctype[].tensortype[].detailedtype "tensor<int8>(p{},x[5])"
 doctype[].structtype[].idx 10016
 doctype[].structtype[].name "tensor.header"
 doctype[].structtype[].field[].name "f1"
@@ -79,3 +82,6 @@ doctype[].structtype[].field[].type 10021
 doctype[].structtype[].field[].name "f6"
 doctype[].structtype[].field[].internalid 596352344
 doctype[].structtype[].field[].type 10005
+doctype[].structtype[].field[].name "f7"
+doctype[].structtype[].field[].internalid 981728328
+doctype[].structtype[].field[].type 10022
diff --git a/config-model/src/test/derived/tensor/documenttypes.cfg b/config-model/src/test/derived/tensor/documenttypes.cfg
index d10ecd37c8f..d069b3764ba 100644
--- a/config-model/src/test/derived/tensor/documenttypes.cfg
+++ b/config-model/src/test/derived/tensor/documenttypes.cfg
@@ -54,6 +54,7 @@ doctype[].fieldsets{[document]}.fields[] "f3"
 doctype[].fieldsets{[document]}.fields[] "f4"
 doctype[].fieldsets{[document]}.fields[] "f5"
 doctype[].fieldsets{[document]}.fields[] "f6"
+doctype[].fieldsets{[document]}.fields[] "f7"
 doctype[].tensortype[].idx 10017
 doctype[].tensortype[].detailedtype "tensor(x[3])"
 doctype[].tensortype[].idx 10018
@@ -64,6 +65,8 @@ doctype[].tensortype[].idx 10020
 doctype[].tensortype[].detailedtype "tensor(x[10],y[10])"
 doctype[].tensortype[].idx 10021
 doctype[].tensortype[].detailedtype "tensor<float>(x[10])"
+doctype[].tensortype[].idx 10022
+doctype[].tensortype[].detailedtype "tensor<int8>(p{},x[5])"
 doctype[].structtype[].idx 10016
 doctype[].structtype[].name "tensor.header"
 doctype[].structtype[].field[].name "f1"
@@ -84,4 +87,7 @@ doctype[].structtype[].field[].type 10021
 doctype[].structtype[].field[].name "f6"
 doctype[].structtype[].field[].internalid 596352344
 doctype[].structtype[].field[].type 10005
+doctype[].structtype[].field[].name "f7"
+doctype[].structtype[].field[].internalid 981728328
+doctype[].structtype[].field[].type 10022
 doctype[].structtype[].internalid 2125927172
diff --git a/config-model/src/test/derived/tensor/index-info.cfg b/config-model/src/test/derived/tensor/index-info.cfg
index 4d8ce8d1372..c9ce2433e17 100644
--- a/config-model/src/test/derived/tensor/index-info.cfg
+++ b/config-model/src/test/derived/tensor/index-info.cfg
@@ -35,3 +35,9 @@ indexinfo[].command[].indexname "f6"
 indexinfo[].command[].command "numerical"
 indexinfo[].command[].indexname "f6"
 indexinfo[].command[].command "type float"
+indexinfo[].command[].indexname "f7"
+indexinfo[].command[].command "attribute"
+indexinfo[].command[].indexname "f7"
+indexinfo[].command[].command "type tensor<int8>(p{},x[5])"
+indexinfo[].command[].indexname "f7"
+indexinfo[].command[].command "word"
diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg
index 1ec3d67cb47..cd8375cb68d 100644
--- a/config-model/src/test/derived/tensor/rank-profiles.cfg
+++ b/config-model/src/test/derived/tensor/rank-profiles.cfg
@@ -1,4 +1,6 @@
 rankprofile[].name "default"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -16,6 +18,8 @@ rankprofile[].fef.property[].name "vespa.hitcollector.arraysize"
 rankprofile[].fef.property[].value "0"
 rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures"
 rankprofile[].fef.property[].value "true"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -29,6 +33,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(map(attribute(f4), f(x)(x * x)) + reduce(tensor(x[2],y[3])(random), count) * rename(attribute(f4), (x, y), (y, x)), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -42,6 +48,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -59,6 +67,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(rankingExpression(joinedtensors), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -72,6 +82,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(attribute(f5), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -85,6 +97,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(tensor<float>(d0[1],x[2]):{{d0:0,x:0}:(attribute(f6)),{d0:0,x:1}:(reduce(attribute(f5), sum))}, sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -102,6 +116,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(tensor<float>(d0[1],x[2]):{{d0:0,x:0}:(attribute(f6)),{d0:0,x:1}:(reduce(rankingExpression(joinedtensors), sum))}, sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -123,6 +139,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(rankingExpression(reshaped) * rankingExpression(literal), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -138,6 +156,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(tensor(d0[1])((attribute(f3){x:(rankingExpression(functionNotLabel))})), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -153,6 +173,8 @@ rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "rankingExpression(firstphase)"
 rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
 rankprofile[].fef.property[].value "reduce(tensor(shadow[1])((attribute(f3){x:(shadow + rankingExpression(shadow))})), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f2"
 rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f3"
@@ -161,3 +183,26 @@ rankprofile[].fef.property[].name "vespa.type.attribute.f4"
 rankprofile[].fef.property[].value "tensor(x[10],y[10])"
 rankprofile[].fef.property[].name "vespa.type.attribute.f5"
 rankprofile[].fef.property[].value "tensor<float>(x[10])"
+rankprofile[].name "with-unpack"
+rankprofile[].fef.property[].name "rankingExpression(myunpack).rankingScript"
+rankprofile[].fef.property[].value "map_subspaces(attribute(f7), f(denseSubspaceInput)(tensor<float>(x[40])(bit(denseSubspaceInput{x:(x/8)}, 7-(x % 8)))))"
+rankprofile[].fef.property[].name "rankingExpression(myunpack).type"
+rankprofile[].fef.property[].value "tensor<float>(p{},x[40])"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "rankingExpression(firstphase)"
+rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(query(para) * rankingExpression(myunpack) * query(qvec), sum)"
+rankprofile[].fef.property[].name "vespa.type.attribute.f7"
+rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])"
+rankprofile[].fef.property[].name "vespa.type.attribute.f2"
+rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])"
+rankprofile[].fef.property[].name "vespa.type.attribute.f3"
+rankprofile[].fef.property[].value "tensor(x{})"
+rankprofile[].fef.property[].name "vespa.type.attribute.f4"
+rankprofile[].fef.property[].value "tensor(x[10],y[10])"
+rankprofile[].fef.property[].name "vespa.type.attribute.f5"
+rankprofile[].fef.property[].value "tensor<float>(x[10])"
+rankprofile[].fef.property[].name "vespa.type.query.para"
+rankprofile[].fef.property[].value "tensor<float>(p{})"
+rankprofile[].fef.property[].name "vespa.type.query.qvec"
+rankprofile[].fef.property[].value "tensor<float>(x[40])"
diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd
index f2fc57a2018..81230e5c54c 100644
--- a/config-model/src/test/derived/tensor/tensor.sd
+++ b/config-model/src/test/derived/tensor/tensor.sd
@@ -20,6 +20,9 @@ schema tensor {
     field f6 type float {
       indexing: attribute
     }
+    field f7 type tensor<int8>(p{},x[5]) {
+      indexing: attribute
+    }
   }
   
   rank-profile profile1 {
@@ -119,4 +122,17 @@ schema tensor {
 
   }
 
+  rank-profile with-unpack {
+    inputs {
+      query(para) tensor<float>(p{})
+      query(qvec) tensor<float>(x[40])
+    }
+    function myunpack() {
+      expression: unpack_bits_from_int8(attribute(f7))
+    }
+    first-phase {
+      expression: sum(query(para)*myunpack*query(qvec))
+    }
+  }
+
 }