Merge pull request #22248 from vespa-engine/arnej/add-fast-rank

add fast-rank for tensor attributes

13 files changed, 276 insertions, 2 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 7acf5557236..30cb0d9c07d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -216,7 +216,8 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
         }
         aaB.enablebitvectors(attribute.isEnabledBitVectors());
         aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector());
-        if (attribute.isFastSearch()) {
+        if (attribute.isFastSearch() || attribute.isFastRank()) {
+            // TODO make a separate fastrank flag in config instead of overloading fastsearch
             aaB.fastsearch(true);
         }
         if (attribute.isFastAccess()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index b466a78c69b..2c4a2c06d02 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -25,6 +25,7 @@ import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.tensor.TensorType;
 
 import java.io.Serializable;
+import java.util.function.Supplier;
 import java.util.LinkedHashSet;
 import java.util.Objects;
 import java.util.Optional;
@@ -52,6 +53,7 @@ public final class Attribute implements Cloneable, Serializable {
     private boolean enableBitVectors = false;
     private boolean enableOnlyBitVector = false;
 
+    private boolean fastRank = false;
     private boolean fastSearch = false;
     private boolean fastAccess = false;
     private boolean huge = false;
@@ -195,6 +197,7 @@ public final class Attribute implements Cloneable, Serializable {
     public boolean isEnabledBitVectors()    { return enableBitVectors; }
     public boolean isEnabledOnlyBitVector() { return enableOnlyBitVector; }
     public boolean isFastSearch()           { return fastSearch; }
+    public boolean isFastRank()            {  return fastRank; }
     public boolean isFastAccess()           { return fastAccess; }
     public boolean isHuge()                 { return huge; }
     public boolean isPaged()                { return paged; }
@@ -228,9 +231,20 @@ public final class Attribute implements Cloneable, Serializable {
     public void setPrefetch(Boolean prefetch)                    { this.prefetch = prefetch; }
     public void setEnableBitVectors(boolean enableBitVectors)    { this.enableBitVectors = enableBitVectors; }
     public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; }
+    public void setFastRank(boolean value) {
+        Supplier<IllegalArgumentException> badGen = () -> new IllegalArgumentException("fast-rank is only valid for tensor attributes, invalid for: "+this);
+        var tt = tensorType.orElseThrow(badGen);
+        for (var dim : tt.dimensions()) {
+            if (dim.isMapped()) {
+                this.fastRank = value;
+                return;
+            }
+        }
+        throw badGen.get();
+    }
     public void setFastSearch(boolean fastSearch)                { this.fastSearch = fastSearch; }
     public void setHuge(boolean huge)                            { this.huge = huge; }
-    public void setPaged(boolean paged)                  { this.paged = paged; }
+    public void setPaged(boolean paged)                          { this.paged = paged; }
     public void setFastAccess(boolean fastAccess)                { this.fastAccess = fastAccess; }
     public void setPosition(boolean position)                    { this.isPosition = position; }
     public void setMutable(boolean mutable)                      { this.mutable = mutable; }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
index 92e099bc1fe..0e79356abd2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java
@@ -79,6 +79,9 @@ public class ConvertParsedFields {
         attribute.setHuge(parsed.getHuge());
         attribute.setPaged(parsed.getPaged());
         attribute.setFastSearch(parsed.getFastSearch());
+        if (parsed.getFastRank()) {
+            attribute.setFastRank(parsed.getFastRank());
+        }
         attribute.setFastAccess(parsed.getFastAccess());
         attribute.setMutable(parsed.getMutable());
         attribute.setEnableBitVectors(parsed.getEnableBitVectors());
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java
index b5dabcdb608..b48bad89114 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java
@@ -17,6 +17,7 @@ class ParsedAttribute extends ParsedBlock {
     private boolean enableBitVectors = false;
     private boolean enableOnlyBitVector = false;
     private boolean enableFastAccess = false;
+    private boolean enableFastRank = false;
     private boolean enableFastSearch = false;
     private boolean enableHuge = false;
     private boolean enableMutable = false;
@@ -35,6 +36,7 @@ class ParsedAttribute extends ParsedBlock {
     boolean getEnableBitVectors() { return this.enableBitVectors; }
     boolean getEnableOnlyBitVector() { return this.enableOnlyBitVector; }
     boolean getFastAccess() { return this.enableFastAccess; }
+    boolean getFastRank() { return this.enableFastRank; }
     boolean getFastSearch() { return this.enableFastSearch; }
     boolean getHuge() { return this.enableHuge; }
     boolean getMutable() { return this.enableMutable; }
@@ -59,6 +61,7 @@ class ParsedAttribute extends ParsedBlock {
     void setEnableBitVectors(boolean value) { this.enableBitVectors = value; }
     void setEnableOnlyBitVector(boolean value) { this.enableOnlyBitVector = value; }
     void setFastAccess(boolean value) { this.enableFastAccess = true; }
+    void setFastRank(boolean value) { this.enableFastRank = true; }
     void setFastSearch(boolean value) { this.enableFastSearch = true; }
     void setHuge(boolean value) { this.enableHuge = true; }
     void setMutable(boolean value) { this.enableMutable = true; }
diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj
index 4767c286f25..91dd5867307 100644
--- a/config-model/src/main/javacc/IntermediateParser.jj
+++ b/config-model/src/main/javacc/IntermediateParser.jj
@@ -263,6 +263,7 @@ TOKEN :
 | < FASTACCESS: "fast-access" >
 | < MUTABLE: "mutable" >
 | < PAGED: "paged" >
+| < FASTRANK: "fast-rank" >
 | < FASTSEARCH: "fast-search" >
 | < HUGE: "huge" >
 | < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])+ ")" >
@@ -1112,6 +1113,7 @@ void attributeSetting(ParsedAttribute attribute) :
 {
     (
         <HUGE>                 { attribute.setHuge(true); }
+      | <FASTRANK>             { attribute.setFastRank(true); }
       | <FASTSEARCH>           { attribute.setFastSearch(true); }
       | <FASTACCESS>           { attribute.setFastAccess(true); }
       | <MUTABLE>              { attribute.setMutable(true); }
@@ -2531,6 +2533,7 @@ String identifier() : { }
       | <EXACTTERMINATOR>
       | <FALSE>
       | <FASTACCESS>
+      | <FASTRANK>
       | <FASTSEARCH>
       | <FIELD>
       | <FIELDS>
diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg
index b29f0494fec..cbc3280f213 100644
--- a/config-model/src/test/derived/attributes/attributes.cfg
+++ b/config-model/src/test/derived/attributes/attributes.cfg
@@ -510,6 +510,38 @@ attribute[].index.hnsw.maxlinkspernode 16
 attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].index.hnsw.distancemetric EUCLIDEAN
 attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "a13"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor(x{})"
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
 attribute[].name "a7_arr"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
diff --git a/config-model/src/test/derived/attributes/attributes.sd b/config-model/src/test/derived/attributes/attributes.sd
index 3dfce32e22d..f38087fbc6f 100644
--- a/config-model/src/test/derived/attributes/attributes.sd
+++ b/config-model/src/test/derived/attributes/attributes.sd
@@ -102,6 +102,11 @@ search attributes {
       rank: filter
     }
 
+    field a13 type tensor(x{}) {
+      indexing: attribute
+      attribute: fast-rank
+    }
+
   }
 
   field a7_arr type array<string> {
diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg
index 536394ab11d..42fda653618 100644
--- a/config-model/src/test/derived/attributes/ilscripts.cfg
+++ b/config-model/src/test/derived/attributes/ilscripts.cfg
@@ -20,6 +20,7 @@ ilscript[].docfield[] "a9"
 ilscript[].docfield[] "a10"
 ilscript[].docfield[] "a11"
 ilscript[].docfield[] "a12"
+ilscript[].docfield[] "a13"
 ilscript[].content[] "clear_state | guard { input a7 | split \";\" | attribute a7_arr; }"
 ilscript[].content[] "clear_state | guard { input a8 | split \";\" | attribute a8_arr; }"
 ilscript[].content[] "clear_state | guard { input a1 | attribute a1 | summary a1; }"
@@ -41,3 +42,4 @@ ilscript[].content[] "clear_state | guard { input a9 | attribute a9; }"
 ilscript[].content[] "clear_state | guard { input a10 | attribute a10; }"
 ilscript[].content[] "clear_state | guard { input a11 | attribute a11; }"
 ilscript[].content[] "clear_state | guard { input a12 | attribute a12; }"
+ilscript[].content[] "clear_state | guard { input a13 | attribute a13; }"
diff --git a/config-model/src/test/derived/attributes/index-info.cfg b/config-model/src/test/derived/attributes/index-info.cfg
index aa400c7de0a..9aeb79a7e2d 100644
--- a/config-model/src/test/derived/attributes/index-info.cfg
+++ b/config-model/src/test/derived/attributes/index-info.cfg
@@ -169,6 +169,14 @@ indexinfo[].command[].indexname "a12"
 indexinfo[].command[].command "numerical"
 indexinfo[].command[].indexname "a12"
 indexinfo[].command[].command "type int"
+indexinfo[].command[].indexname "a13"
+indexinfo[].command[].command "index"
+indexinfo[].command[].indexname "a13"
+indexinfo[].command[].command "attribute"
+indexinfo[].command[].indexname "a13"
+indexinfo[].command[].command "type tensor(x{})"
+indexinfo[].command[].indexname "a13"
+indexinfo[].command[].command "word"
 indexinfo[].command[].indexname "a7_arr"
 indexinfo[].command[].command "index"
 indexinfo[].command[].indexname "a7_arr"
diff --git a/config-model/src/test/derived/attributes/summarymap.cfg b/config-model/src/test/derived/attributes/summarymap.cfg
index a96f1e3858d..78bf22b7109 100644
--- a/config-model/src/test/derived/attributes/summarymap.cfg
+++ b/config-model/src/test/derived/attributes/summarymap.cfg
@@ -50,3 +50,6 @@ override[].arguments "a11"
 override[].field "a12"
 override[].command "attribute"
 override[].arguments "a12"
+override[].field "a13"
+override[].command "attribute"
+override[].arguments "a13"
diff --git a/config-model/src/test/derived/function_arguments/attributes.cfg b/config-model/src/test/derived/function_arguments/attributes.cfg
new file mode 100644
index 00000000000..8320063ab66
--- /dev/null
+++ b/config-model/src/test/derived/function_arguments/attributes.cfg
@@ -0,0 +1,96 @@
+attribute[].name "f1"
+attribute[].datatype FLOAT
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "t1"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor<float>(x{})"
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "t2"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor<float>(x{})"
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
diff --git a/config-model/src/test/derived/function_arguments_with_expressions/attributes.cfg b/config-model/src/test/derived/function_arguments_with_expressions/attributes.cfg
new file mode 100644
index 00000000000..8bcad42c882
--- /dev/null
+++ b/config-model/src/test/derived/function_arguments_with_expressions/attributes.cfg
@@ -0,0 +1,96 @@
+attribute[].name "i1"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "t1"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor<float>(x{})"
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "t2"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match UNCASED
+attribute[].match UNCASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor<float>(x{})"
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.multithreadedindexing true
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
index 9e53bd57d77..b9e6bfe7dc9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
@@ -128,6 +128,14 @@ public class TensorFieldTestCase {
                 "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-search }", "t1").isFastSearch());
     }
 
+    @Test
+    public void tensors_with_at_least_one_mapped_dimension_can_be_fast_rank() throws ParseException {
+        assertTrue(getAttributeFromSd(
+                "field t1 type tensor(x{}) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank());
+        assertTrue(getAttributeFromSd(
+                "field t1 type tensor(x{},y{},z[4]) { indexing: attribute \n attribute: fast-rank }", "t1").isFastRank());
+    }
+
     private static String getSd(String field) {
         return joinLines("search test {",
                 "  document test {",