Merge pull request #31191 from vespa-engine/toregge/dont-implicitly-set-word-match-for-tensor-attribute-fields

Don't implicitly set word match for tensor attribute fields.

6 files changed, 58 insertions, 20 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java
index 767593b82d0..769f0c9de92 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java
@@ -2,6 +2,7 @@
 package com.yahoo.schema.processing;
 
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.schema.RankProfileRegistry;
 import com.yahoo.document.DataType;
 import com.yahoo.schema.Schema;
@@ -45,6 +46,8 @@ public class AttributesImplicitWord extends Processor {
     private boolean fieldImplicitlyWordMatch(ImmutableSDField field) {
         // numeric types should not trigger exact-match query parsing
         if (field.getDataType().getPrimitiveType() instanceof NumericDataType) return false;
+        // Tensor type should not trigger exact-match query parsing
+        if (field.getDataType() instanceof TensorDataType) return false;
 
         return (! field.hasIndex()
                 && !field.getAttributes().isEmpty()
diff --git a/config-model/src/test/derived/attributes/index-info.cfg b/config-model/src/test/derived/attributes/index-info.cfg
index 1d4e8f485b3..245cff48d15 100644
--- a/config-model/src/test/derived/attributes/index-info.cfg
+++ b/config-model/src/test/derived/attributes/index-info.cfg
@@ -175,8 +175,6 @@ indexinfo[].command[].indexname "a13"
 indexinfo[].command[].command "attribute"
 indexinfo[].command[].indexname "a13"
 indexinfo[].command[].command "type tensor(x{})"
-indexinfo[].command[].indexname "a13"
-indexinfo[].command[].command "word"
 indexinfo[].command[].indexname "a7_arr"
 indexinfo[].command[].command "lowercase"
 indexinfo[].command[].indexname "a7_arr"
diff --git a/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg
index ab9a96f819b..ec06d01f05a 100644
--- a/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg
+++ b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg
@@ -3,25 +3,25 @@ searchall 1
 fieldspec[].name "vec_a"
 fieldspec[].searchmethod NEAREST_NEIGHBOR
 fieldspec[].arg1 "EUCLIDEAN"
-fieldspec[].normalize LOWERCASE
+fieldspec[].normalize LOWERCASE_AND_FOLD
 fieldspec[].maxlength 1048576
 fieldspec[].fieldtype ATTRIBUTE
 fieldspec[].name "vec_b"
 fieldspec[].searchmethod NEAREST_NEIGHBOR
 fieldspec[].arg1 "ANGULAR"
-fieldspec[].normalize LOWERCASE
+fieldspec[].normalize LOWERCASE_AND_FOLD
 fieldspec[].maxlength 1048576
 fieldspec[].fieldtype ATTRIBUTE
 fieldspec[].name "vec_c"
 fieldspec[].searchmethod NEAREST_NEIGHBOR
 fieldspec[].arg1 "INNERPRODUCT"
-fieldspec[].normalize LOWERCASE
+fieldspec[].normalize LOWERCASE_AND_FOLD
 fieldspec[].maxlength 1048576
 fieldspec[].fieldtype ATTRIBUTE
 fieldspec[].name "vec_d"
 fieldspec[].searchmethod NONE
 fieldspec[].arg1 ""
-fieldspec[].normalize LOWERCASE
+fieldspec[].normalize LOWERCASE_AND_FOLD
 fieldspec[].maxlength 1048576
 fieldspec[].fieldtype ATTRIBUTE
 documenttype[].name "test"
diff --git a/config-model/src/test/derived/tensor/index-info.cfg b/config-model/src/test/derived/tensor/index-info.cfg
index c9ce2433e17..2402f074837 100644
--- a/config-model/src/test/derived/tensor/index-info.cfg
+++ b/config-model/src/test/derived/tensor/index-info.cfg
@@ -9,26 +9,18 @@ indexinfo[].command[].indexname "f2"
 indexinfo[].command[].command "attribute"
 indexinfo[].command[].indexname "f2"
 indexinfo[].command[].command "type tensor<float>(x[2],y[1])"
-indexinfo[].command[].indexname "f2"
-indexinfo[].command[].command "word"
 indexinfo[].command[].indexname "f3"
 indexinfo[].command[].command "attribute"
 indexinfo[].command[].indexname "f3"
 indexinfo[].command[].command "type tensor(x{})"
-indexinfo[].command[].indexname "f3"
-indexinfo[].command[].command "word"
 indexinfo[].command[].indexname "f4"
 indexinfo[].command[].command "attribute"
 indexinfo[].command[].indexname "f4"
 indexinfo[].command[].command "type tensor(x[10],y[10])"
-indexinfo[].command[].indexname "f4"
-indexinfo[].command[].command "word"
 indexinfo[].command[].indexname "f5"
 indexinfo[].command[].command "attribute"
 indexinfo[].command[].indexname "f5"
 indexinfo[].command[].command "type tensor<float>(x[10])"
-indexinfo[].command[].indexname "f5"
-indexinfo[].command[].command "word"
 indexinfo[].command[].indexname "f6"
 indexinfo[].command[].command "attribute"
 indexinfo[].command[].indexname "f6"
@@ -39,5 +31,3 @@ indexinfo[].command[].indexname "f7"
 indexinfo[].command[].command "attribute"
 indexinfo[].command[].indexname "f7"
 indexinfo[].command[].command "type tensor<int8>(p{},x[5])"
-indexinfo[].command[].indexname "f7"
-indexinfo[].command[].command "word"
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
index 8778f0c26c0..0677cabafb0 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
@@ -29,11 +29,18 @@ public abstract class ContentClusterFixture {
         nextCluster = createCluster(nextSd);
     }
 
+    protected ContentClusterFixture(ContentCluster currentCluster, ContentCluster nextCluster) {
+        this.currentCluster = currentCluster;
+        this.nextCluster = nextCluster;
+    }
+
     public ContentClusterFixture(String entireSd) throws Exception {
-        currentCluster = new ContentClusterBuilder().build(
-            ContentClusterUtils.createMockRoot(List.of(entireSd)));
-        nextCluster = new ContentClusterBuilder().build(
-            ContentClusterUtils.createMockRoot(List.of(entireSd)));
+        currentCluster = createClusterFromEntireSd(entireSd);
+        nextCluster = createClusterFromEntireSd(entireSd);
+    }
+
+    protected static ContentCluster createClusterFromEntireSd(String sdContent) throws Exception {
+        return new ContentClusterBuilder().build(ContentClusterUtils.createMockRoot(List.of(sdContent)));
     }
 
     private static ContentCluster createCluster(String sdContent) throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
index cd54a20523f..247f01068fa 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java
@@ -39,6 +39,21 @@ public class IndexingScriptChangeValidatorTest {
         }
     }
 
+    private static class ComplexFixture extends ContentClusterFixture {
+        IndexingScriptChangeValidator validator;
+        public ComplexFixture(String currentSd, String nextSd) throws Exception {
+            super(createClusterFromEntireSd(currentSd), createClusterFromEntireSd(nextSd));
+            validator = new IndexingScriptChangeValidator(ClusterSpec.Id.from("test"),
+                    currentDb().getDerivedConfiguration().getSchema(),
+                    nextDb().getDerivedConfiguration().getSchema());
+        }
+
+        @Override
+        public List<VespaConfigChangeAction> validate() {
+            return validator.validate();
+        }
+    }
+
     private static class ScriptFixture {
 
         private final ScriptExpression currentScript;
@@ -56,6 +71,9 @@ public class IndexingScriptChangeValidatorTest {
 
     private static final String FIELD = "field f1 type string";
     private static final String FIELD_F2 = "field f2 type string";
+    private static final String TENSOR_FIELD_F1 = "field f1 type tensor(x[2])";
+    private static final String TENSOR_FIELD_F2 = "field f2 type tensor(x[2])";
+    private static final String TENSOR_FIELD_F3 = "field f3 type tensor(x[2])";
 
     private static VespaConfigChangeAction expectedReindexingAction(String changedMsg, String fromScript, String toScript) {
         return expectedReindexingAction("f1", changedMsg, fromScript, toScript);
@@ -115,6 +133,28 @@ public class IndexingScriptChangeValidatorTest {
     }
 
     @Test
+    void requireThatAddingIndexAspectForExtraTensorFieldWithChangedInputRequireReindexing() throws Exception {
+        new ComplexFixture(joinLines("schema test {",
+                "  document test {",
+                "    " + TENSOR_FIELD_F1 + " { }",
+                "    " + TENSOR_FIELD_F2 + " { }",
+                "  }",
+                "  " + TENSOR_FIELD_F3 + " { indexing: input f1 | attribute }",
+                "}"),
+                joinLines("schema test {",
+                        "  document test {",
+                        "    " + TENSOR_FIELD_F1 + " { }",
+                        "    " + TENSOR_FIELD_F2 + " { }",
+                        "  }",
+                        "  " + TENSOR_FIELD_F3 + " { indexing: input f2 | index | attribute }",
+                        "}")).
+                assertValidation(List.of(expectedReindexingAction("f3", "add index aspect",
+                                "{ input f1 | attribute f3; }",
+                                "{ input f2 | index f3 | attribute f3; }")));
+    }
+
+
+    @Test
     void requireThatSettingDynamicSummaryIsOk() throws Exception {
         new Fixture(FIELD + " { indexing: summary }",
                 FIELD + " { indexing: summary \n summary: dynamic }").