Merge pull request #7120 from vespa-engine/balder/passthrough-for-updates-to-non-document-fields

When you want to let them through, you can not only give them partial…

17 files changed, 101 insertions, 85 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
index 6c7396d607b..dc6c17e425e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
@@ -41,17 +41,12 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
         derive(search);
     }
 
-    private boolean hasFullIndexingDocprocRights(ImmutableSDField field) {
-        Attribute self = field.getAttributes().get(field.getName());
-        return (!field.isExtraField() || ((self != null) && self.isMutable()));
-    }
-
     @Override
     protected void derive(ImmutableSDField field, Search search) {
         if (field.isImportedField()) {
             return;
         }
-        if (hasFullIndexingDocprocRights(field)) {
+        if (field.hasFullIndexingDocprocRights()) {
             docFields.add(field.getName());
         }
         if (field.usesStructOrMap() &&
@@ -93,7 +88,7 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
             if (modifiesSelf(exp)) {
                 later.add(exp);
             } else {
-              ilscriptBuilder.content(exp.toString());
+                ilscriptBuilder.content(exp.toString());
             }
             fieldFetcher.visit(exp);
             touchedFields.addAll(fieldFetcher.touchedFields());
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
index 125e6b976b5..ff11c2fdf22 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java
@@ -169,4 +169,9 @@ public class ImmutableImportedSDField implements ImmutableSDField {
         return new UnsupportedOperationException("'" + aspect + "' is not meaningful or relevant for an imported field.");
     }
 
+    @Override
+    public boolean hasFullIndexingDocprocRights() {
+        return importedField.targetField().hasFullIndexingDocprocRights();
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java
index 70553d4b57c..6fe8a4da92b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java
@@ -81,4 +81,5 @@ public interface ImmutableSDField {
      */
     Field asField();
 
+    boolean hasFullIndexingDocprocRights();
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index 0590027a402..e310b6b6858 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -799,4 +799,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
         return aliasToName;
     }
 
+    @Override
+    public boolean hasFullIndexingDocprocRights() {
+        Attribute self = getAttributes().get(getName());
+        return (!isExtraField() || ((self != null) && self.isMutable()));
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
index aeab2bb6638..b0129c0a836 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java
@@ -97,13 +97,11 @@ public class IndexingInputs extends Processor {
         @Override
         protected void doVisit(Expression exp) {
             if ( ! (exp instanceof InputExpression)) return;
-
-            SDDocumentType docType = search.getDocument();
             String inputField = ((InputExpression)exp).getFieldName();
-            if (docType.getField(inputField) != null) return;
+            if (search.getField(inputField).hasFullIndexingDocprocRights()) return;
 
             fail(search, field, "Indexing script refers to field '" + inputField + "' which does not exist " +
-                                "in document type '" + docType.getName() + "'.");
+                                "in document type '" + search.getDocument().getName() + "', and is not a mutable attribute.");
         }
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 8c8c32389e2..3b2e29c4cb3 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -33,6 +33,7 @@ public class Processing {
                 UriHack::new,
                 LiteralBoost::new,
                 TagType::new,
+                ValidateFieldTypesDocumentsOnly::new,
                 IndexingInputs::new,
                 OptimizeIlscript::new,
                 ValidateFieldWithIndexSettingsCreatesIndex::new,
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
index 21b7f1d2675..1cf3d61d8f1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java
@@ -29,16 +29,24 @@ public class ValidateFieldTypes extends Processor {
 
     @Override
     public void process(boolean validate, boolean documentsOnly) {
-        if ( ! validate) return;
+        if (!validate) return;
 
         String searchName = search.getName();
         Map<String, DataType> seenFields = new HashMap<>();
+        verifySearchAndDocFields(searchName, seenFields);
+        verifySummaryFields(searchName, seenFields);
+    }
+
+    final protected void verifySearchAndDocFields(String searchName, Map<String, DataType> seenFields) {
         search.allFields().forEach(field -> {
             checkFieldType(searchName, "index field", field.getName(), field.getDataType(), seenFields);
             for (Map.Entry<String, Attribute> entry : field.getAttributes().entrySet()) {
                 checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), seenFields);
             }
         });
+
+    }
+    final protected void verifySummaryFields(String searchName, Map<String, DataType> seenFields) {
         for (DocumentSummary summary : search.getSummaries().values()) {
             for (SummaryField field : summary.getSummaryFields()) {
                 checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java
new file mode 100644
index 00000000000..ff3e638fff1
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java
@@ -0,0 +1,25 @@
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.document.DataType;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.Search;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ValidateFieldTypesDocumentsOnly extends ValidateFieldTypes {
+    public ValidateFieldTypesDocumentsOnly(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(search, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process(boolean validate, boolean documentsOnly) {
+        if ( ! validate) return;
+
+        String searchName = search.getName();
+        Map<String, DataType> seenFields = new HashMap<>();
+        verifySearchAndDocFields(searchName, seenFields);
+    }
+}
diff --git a/config-model/src/test/examples/duplicatenamesindoc.sd b/config-model/src/test/examples/duplicatenamesindoc.sd
deleted file mode 100644
index 3c55efe953d..00000000000
--- a/config-model/src/test/examples/duplicatenamesindoc.sd
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-search duplicatenamesindoc {
-document {
-  field foo type int {
-    indexing: attribute
-  }
-  
-  field fOo type string {
-    indexing: index
-  }
-}
-}
diff --git a/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd b/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd
deleted file mode 100644
index 502a4bb7aff..00000000000
--- a/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-search duplicatenamesinsearch {
-document {
-  field grpphotoids64 type string { }
-}
-
-field grpphotoids64 type array<long> {
-    indexing: input grpphotoids64 | split " " | for_each {
-      base64decode } | attribute
-    }
-
-}
diff --git a/config-model/src/test/examples/incorrectsummarytypes.sd b/config-model/src/test/examples/incorrectsummarytypes.sd
deleted file mode 100644
index 9304ad5ca20..00000000000
--- a/config-model/src/test/examples/incorrectsummarytypes.sd
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-search incorrectsummarytypes {
-
-  document incorrectsummarytypes {
-
-    field somestring type string {
-      indexing: summary
-    }
-
-  }
-
-  document-summary incorrect {
-
-    summary somestring type int {
-    }
-
-  }
-
-}
diff --git a/config-model/src/test/examples/name-check.sd b/config-model/src/test/examples/name-check.sd
deleted file mode 100644
index bd7c0713e3a..00000000000
--- a/config-model/src/test/examples/name-check.sd
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-# An entry-level configuration.
-# You can get a reasonable configuration by only configuring
-# a document
-search simple {
-
-  document name-check {
-
-    field title type string {
-      indexing: summary | index
-    }
-
-    # reserved name, should trigger error
-    field sddocname type string {
-      indexing: index
-    }
-
-  }
-
-}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
index cadb2cda324..71db60065aa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
@@ -165,7 +165,7 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase {
                     "    }\n" +
                     "  }\n" +
                     "  field m type long {\n" +
-                    "    indexing: 0 | to_long | attribute\n" +
+                    "    indexing: attribute\n" +
                     "    attribute: mutable\n" +
                     "  }\n" +
                     "  field f type long {\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
index f11799495a2..627394bb6ea 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
@@ -17,7 +17,18 @@ public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase {
     @Test
     public void testImportingIncorrect() throws IOException, ParseException {
         try {
-            SearchBuilder.buildFromFile("src/test/examples/incorrectsummarytypes.sd");
+            SearchBuilder.createFromString(
+                    "search incorrectsummarytypes {\n" +
+                    "  document incorrectsummarytypes {\n" +
+                    "    field somestring type string {\n" +
+                    "      indexing: summary\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "  document-summary incorrect {\n" +
+                    "    summary somestring type int {\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "}\n");
             fail("processing should have failed");
         } catch (RuntimeException e) {
             assertEquals("'summary somestring type string' in 'destinations(default )' is inconsistent with 'summary somestring type int' in 'destinations(incorrect )': All declarations of the same summary field must have the same type", e.getMessage());
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
index c19244a4bf2..b539c65150d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
@@ -22,7 +22,18 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
     @Test
     public void testNameField() throws IOException, ParseException {
         try {
-            SearchBuilder.buildFromFile("src/test/examples/name-check.sd");
+            SearchBuilder.createFromString(
+                    "search simple {\n" +
+                    "  document name-check {\n" +
+                    "    field title type string {\n" +
+                    "      indexing: summary | index\n" +
+                    "    }\n" +
+                    "    # reserved name, should trigger error\n" +
+                    "    field sddocname type string {\n" +
+                    "      indexing: index\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "}");
             fail("Should throw exception.");
         } catch (Exception expected) {
             // Success
@@ -32,7 +43,16 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
     @Test
     public void testDuplicateNamesInSearchDifferentType() {
         try {
-            SearchBuilder.buildFromFile("src/test/examples/duplicatenamesinsearchdifferenttype.sd");
+            SearchBuilder.createFromString(
+                    "search duplicatenamesinsearch {\n" +
+                    "  document {\n" +
+                    "    field grpphotoids64 type string { }\n" +
+                    "  }\n" +
+                    "  field grpphotoids64 type array<long> {\n" +
+                    "    indexing: input grpphotoids64 | split \" \" | for_each {\n" +
+                    "      base64decode } | attribute\n" +
+                    "  }\n" +
+                    "}");
             fail("Should throw exception.");
         } catch (Exception e) {
             assertEquals("For search 'duplicatenamesinsearch', field 'grpphotoids64': Incompatible types. Expected Array<long> for index field 'grpphotoids64', got string.", e.getMessage());
@@ -42,7 +62,17 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
     @Test
     public void testDuplicateNamesInDoc() {
         try {
-            SearchBuilder.buildFromFile("src/test/examples/duplicatenamesindoc.sd");
+            SearchBuilder.createFromString(
+                    "search duplicatenamesindoc {\n" +
+                    "  document {\n" +
+                    "    field foo type int {\n" +
+                    "      indexing: attribute\n" +
+                    "    }\n" +
+                    "    field fOo type string {\n" +
+                    "      indexing: index\n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "}");
             fail("Should throw exception.");
         } catch (Exception e) {
             assertTrue(e.getMessage().matches(".*Duplicate.*"));
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java
index 8fb40a73f0a..582f5b08517 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java
@@ -18,27 +18,27 @@ public class IndexingInputsTestCase {
         assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd",
                          "For search 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " +
                          "to field 'bar' which does not exist in document type " +
-                         "'indexing_extra_field_input_extra_field'.");
+                         "'indexing_extra_field_input_extra_field', and is not a mutable attribute.");
     }
 
     @Test
     public void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException {
         assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd",
                          "For search 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " +
-                         "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit'.");
+                         "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit', and is not a mutable attribute.");
     }
 
     @Test
     public void requireThatExtraFieldInputNullThrows() throws IOException, ParseException {
         assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd",
                          "For search 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " +
-                         "'foo' which does not exist in document type 'indexing_extra_field_input_null'.");
+                         "'foo' which does not exist in document type 'indexing_extra_field_input_null', and is not a mutable attribute.");
     }
 
     @Test
     public void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException {
         assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd",
                          "For search 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " +
-                         "'foo' which does not exist in document type 'indexing_extra_field_input_self'.");
+                         "'foo' which does not exist in document type 'indexing_extra_field_input_self', and is not a mutable attribute.");
     }
 }
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
index 10e3ce561d7..a2321e912e1 100644
--- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
+++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
@@ -9,7 +9,6 @@ import com.yahoo.document.datatypes.Array;
 import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.datatypes.MapFieldValue;
 import com.yahoo.document.datatypes.StringFieldValue;
-import com.yahoo.document.datatypes.Struct;
 import com.yahoo.document.datatypes.StructuredFieldValue;
 import com.yahoo.document.datatypes.WeightedSet;
 import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;