Validate that complex fields do not have struct fields with 'indexing: index'.

This is not supported for indexed search clusters, and should be replaced with 'indexing: attribute'.

7 files changed, 123 insertions, 4 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java
index 335942de99d..221280dc1e4 100644
--- a/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java
+++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java
@@ -83,6 +83,11 @@ public class ImmutableImportedSDField implements ImmutableSDField {
     }
 
     @Override
+    public boolean wasConfiguredToDoIndexing() {
+        return importedField.targetField().wasConfiguredToDoIndexing();
+    }
+
+    @Override
     public DataType getDataType() {
         return importedField.targetField().getDataType();
     }
diff --git a/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java
index 44e442811ba..37cbb3a8171 100644
--- a/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java
+++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java
@@ -45,6 +45,14 @@ public interface ImmutableSDField {
      */
     boolean wasConfiguredToDoAttributing();
 
+    /**
+     * Whether this field at some time was configured to do indexing.
+     *
+     * This function can typically return a different value than doesIndexing(),
+     * which uses the final state of the underlying indexing script instead.
+     */
+    boolean wasConfiguredToDoIndexing();
+
     DataType getDataType();
 
     Index getIndex(String name);
diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java
index 943d6c6fc14..2d79d89a2a0 100644
--- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java
@@ -117,6 +117,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField {
     private boolean isExtraField = false;
 
     private boolean wasConfiguredToDoAttributing = false;
+    private boolean wasConfiguredToDoIndexing = false;
 
     /**
      * Creates a new field. This method is only used to create reserved fields.
@@ -381,6 +382,11 @@ public class SDField extends Field implements TypedKey, ImmutableSDField {
         return wasConfiguredToDoAttributing;
     }
 
+    @Override
+    public boolean wasConfiguredToDoIndexing() {
+        return wasConfiguredToDoIndexing;
+    }
+
     /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */
     public void parseIndexingScript(String script) {
         parseIndexingScript(script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap());
@@ -408,6 +414,9 @@ public class SDField extends Field implements TypedKey, ImmutableSDField {
         if (!wasConfiguredToDoAttributing()) {
             wasConfiguredToDoAttributing = doesAttributing();
         }
+        if (!wasConfiguredToDoIndexing()) {
+            wasConfiguredToDoIndexing = doesIndexing();
+        }
         if (!usesStructOrMap()) {
             new ExpressionVisitor() {
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
index 01bf846d4cb..8515c34a377 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldAttributesValidator.java
@@ -22,7 +22,7 @@ import java.util.stream.Collectors;
  *
  * @author geirst
  */
-public class ComplexAttributeFieldsValidator extends Validator {
+public class ComplexFieldsWithStructFieldAttributesValidator extends Validator {
 
     @Override
     public void validate(VespaModel model, DeployState deployState) {
@@ -39,7 +39,7 @@ public class ComplexAttributeFieldsValidator extends Validator {
     private static void validateComplexFields(String clusterName, Schema schema) {
         String unsupportedFields = schema.allFields()
                                          .filter(field -> isUnsupportedComplexField(field))
-                                         .map(ComplexAttributeFieldsValidator::toString)
+                                         .map(ComplexFieldsWithStructFieldAttributesValidator::toString)
                                          .collect(Collectors.joining(", "));
 
         if (!unsupportedFields.isEmpty()) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
new file mode 100644
index 00000000000..bbbb0c08bf4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexFieldsWithStructFieldIndexesValidator.java
@@ -0,0 +1,72 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.document.ImmutableSDField;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Validates that complex fields (of type struct or map) do not have any struct fields with 'indexing: index'.
+ * This is not supported and will confuse the user if not validated.
+ *
+ * Only applies for indexed search clusters.
+ *
+ * @author geirst
+ */
+public class ComplexFieldsWithStructFieldIndexesValidator extends Validator {
+
+    @Override
+    public void validate(VespaModel model, DeployState deployState) {
+        for (var cluster : model.getSearchClusters()) {
+            if (cluster.isStreaming()) {
+                continue;
+            }
+            for (var spec : cluster.schemas().values()) {
+                validateComplexFields(cluster.getClusterName(), spec.fullSchema());
+            }
+        }
+    }
+
+    private static void validateComplexFields(String clusterName, Schema schema) {
+        String unsupportedFields = schema.allFields()
+                .filter(field -> hasStructFieldsWithIndex(field))
+                .map(ComplexFieldsWithStructFieldIndexesValidator::toString)
+                .collect(Collectors.joining(", "));
+
+        if (!unsupportedFields.isEmpty()) {
+            throw new IllegalArgumentException(
+                    String.format("For cluster '%s', schema '%s': The following complex fields have struct fields with 'indexing: index' which is not supported: %s. " +
+                                  "Change to 'indexing: attribute' instead",
+                                  clusterName, schema.getName(), unsupportedFields));
+        }
+    }
+
+    private static boolean hasStructFieldsWithIndex(ImmutableSDField field) {
+        return (field.usesStructOrMap() && hasStructFieldsWithIndex(field.getStructFields()));
+    }
+
+    private static String toString(ImmutableSDField field) {
+        return field.getName() + " (" + String.join(", ", getStructFieldsWithIndex(field.getStructFields())) + ")";
+    }
+
+    private static boolean hasStructFieldsWithIndex(Collection<? extends ImmutableSDField> structFields) {
+        return !getStructFieldsWithIndex(structFields).isEmpty();
+    }
+
+    private static List<String> getStructFieldsWithIndex(Collection<? extends ImmutableSDField> structFields) {
+        var result = new ArrayList<String>();
+        for (var structField : structFields) {
+            if (structField.wasConfiguredToDoIndexing()) {
+                result.add(structField.getName());
+            }
+            result.addAll(getStructFieldsWithIndex(structField.getStructFields()));
+        }
+        return result;
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index ce61d3edc3b..dab1eeccc96 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -71,7 +71,8 @@ public class Validation {
         new SchemasDirValidator().validate(model, deployState);
         new BundleValidator().validate(model, deployState);
         new SearchDataTypeValidator().validate(model, deployState);
-        new ComplexAttributeFieldsValidator().validate(model, deployState);
+        new ComplexFieldsWithStructFieldAttributesValidator().validate(model, deployState);
+        new ComplexFieldsWithStructFieldIndexesValidator().validate(model, deployState);
         new StreamingValidator().validate(model, deployState);
         new RankSetupValidator(validationParameters.ignoreValidationErrors()).validate(model, deployState);
         new NoPrefixForIndexes().validate(model, deployState);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java
index 85050aa0cf9..11e4ae855a5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidatorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java
@@ -22,7 +22,7 @@ import static com.yahoo.config.model.test.TestUtil.joinLines;
 /**
  * @author geirst
  */
-public class ComplexAttributeFieldsValidatorTestCase {
+public class ComplexFieldsValidatorTestCase {
 
     @SuppressWarnings("deprecation")
     @Rule
@@ -78,6 +78,30 @@ public class ComplexAttributeFieldsValidatorTestCase {
     }
 
     @Test
+    public void throws_when_complex_fields_have_struct_fields_with_index() throws IOException, SAXException {
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage("For cluster 'mycluster', schema 'test': " +
+                "The following complex fields have struct fields with 'indexing: index' which is not supported: " +
+                "topics (topics.id, topics.label). Change to 'indexing: attribute' instead");
+        createModelAndValidate(joinLines(
+                "schema test {",
+                "document test {",
+                "struct topic {",
+                "  field id type string {}",
+                "  field label type string {}",
+                "  field desc type string {}",
+                "}",
+                "field topics type array<topic> {",
+                "  indexing: summary",
+                "  struct-field id { indexing: index }",
+                "  struct-field label { indexing: index | attribute }",
+                "  struct-field desc { indexing: attribute }",
+                "}",
+                "}",
+                "}"));
+    }
+
+    @Test
     public void validation_passes_when_only_supported_struct_field_attributes_are_used() throws IOException, SAXException {
         createModelAndValidate(joinLines("search test {",
                 "  document test {",