Allow mutable fields to be modifiable in indexing docproc and to have an initial value.

2 files changed, 49 insertions, 25 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
index 3f1d6c55536..6c7396d607b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java
@@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.derived;
 import com.yahoo.document.DataType;
 import com.yahoo.document.PositionDataType;
 import com.yahoo.searchdefinition.Search;
+import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.ImmutableSDField;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder;
@@ -22,7 +23,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
@@ -34,19 +34,24 @@ import java.util.Set;
  */
 public final class IndexingScript extends Derived implements IlscriptsConfig.Producer {
 
-    private final List<String> docFields = new LinkedList<>();
-    private final List<Expression> expressions = new LinkedList<>();
+    private final List<String> docFields = new ArrayList<>();
+    private final List<Expression> expressions = new ArrayList<>();
 
     public IndexingScript(Search search) {
         derive(search);
     }
 
+    private boolean hasFullIndexingDocprocRights(ImmutableSDField field) {
+        Attribute self = field.getAttributes().get(field.getName());
+        return (!field.isExtraField() || ((self != null) && self.isMutable()));
+    }
+
     @Override
     protected void derive(ImmutableSDField field, Search search) {
         if (field.isImportedField()) {
             return;
         }
-        if (!field.isExtraField()) {
+        if (hasFullIndexingDocprocRights(field)) {
             docFields.add(field.getName());
         }
         if (field.usesStructOrMap() &&
@@ -75,9 +80,7 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
     public void getConfig(IlscriptsConfig.Builder configBuilder) {
         IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder();
         ilscriptBuilder.doctype(getName());
-        for (String fieldName : docFields) {
-            ilscriptBuilder.docfield(fieldName);
-        }
+        ilscriptBuilder.docfield(docFields);
         addContentInOrder(ilscriptBuilder);
         configBuilder.ilscript(ilscriptBuilder);
     }
@@ -102,9 +105,9 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro
     }
 
     private void generateSyntheticStatementsForUntouchedFields(Builder ilscriptBuilder, Set<String> touchedFields) {
-        Set<String> fieldsWithSyntheticStatements = new HashSet<String>(docFields);
+        Set<String> fieldsWithSyntheticStatements = new HashSet<>(docFields);
         fieldsWithSyntheticStatements.removeAll(touchedFields);
-        List<String> orderedFields = new ArrayList<String>(fieldsWithSyntheticStatements);
+        List<String> orderedFields = new ArrayList<>(fieldsWithSyntheticStatements);
         Collections.sort(orderedFields);
         for (String fieldName : orderedFields) {
             StatementExpression copyField = new StatementExpression(new InputExpression(fieldName),
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
index d7f26cd9851..cadb2cda324 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
@@ -3,11 +3,13 @@ package com.yahoo.searchdefinition;
 
 import com.yahoo.document.StructDataType;
 import com.yahoo.searchdefinition.derived.AttributeFields;
+import com.yahoo.searchdefinition.derived.IndexingScript;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -154,24 +156,28 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase {
         assertTrue(attr.isMutable());
     }
 
+    private Search getSearchWithMutables() throws ParseException {
+        return getSearch(
+                "search test {\n" +
+                    "  document test { \n" +
+                    "    field a type int { \n" +
+                    "      indexing: attribute \n" +
+                    "    }\n" +
+                    "  }\n" +
+                    "  field m type long {\n" +
+                    "    indexing: 0 | to_long | attribute\n" +
+                    "    attribute: mutable\n" +
+                    "  }\n" +
+                    "  field f type long {\n" +
+                    "    indexing: 0 | to_long | attribute\n" +
+                    "  }\n" +
+                    "}\n");
+    }
+
     @Test
-    public void requireThatMutableConfigIsProperlyPropagated() throws ParseException{
+    public void requireThatMutableConfigIsProperlyPropagated() throws ParseException {
 
-        AttributeFields attributes = new AttributeFields(getSearch(
-                "search test {\n" +
-                "  document test { \n" +
-                "    field a type int { \n" +
-                "      indexing: attribute \n" +
-                "    }\n" +
-                "  }\n" +
-                "  field m type long {\n" +
-                "    indexing: 0 | to_long | attribute\n" +
-                "    attribute: mutable\n" +
-                "  }\n" +
-                "  field f type long {\n" +
-                "    indexing: 0 | to_long | attribute\n" +
-                "  }\n" +
-                "}\n"));
+        AttributeFields attributes = new AttributeFields(getSearchWithMutables());
         AttributesConfig.Builder builder = new AttributesConfig.Builder();
         attributes.getConfig(builder);
         AttributesConfig cfg = new AttributesConfig(builder);
@@ -187,6 +193,21 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase {
     }
 
     @Test
+    public void requireThatMutableIsAllowedThroughIndexing() throws ParseException {
+        IndexingScript script = new IndexingScript(getSearchWithMutables());
+        IlscriptsConfig.Builder builder = new IlscriptsConfig.Builder();
+        script.getConfig(builder);
+        IlscriptsConfig cfg = new IlscriptsConfig(builder);
+        assertEquals(1, cfg.ilscript().size());
+        IlscriptsConfig.Ilscript ils = cfg.ilscript(0);
+        assertEquals("test", ils.doctype());
+        assertEquals(2, ils.docfield().size());
+        assertEquals("a", ils.docfield(0));
+        assertEquals("m", ils.docfield(1));
+
+    }
+
+    @Test
     public void attribute_convert_to_array_copies_internal_state() {
         StructDataType refType = new StructDataType("my_struct");
         Attribute single = new Attribute("foo", Attribute.Type.STRING, Attribute.CollectionType.SINGLE,