Pass field path updates verbatim through indexing scripts

Avoids having to do a (potentially lossy) round-trip of
update -> document -> update again during processing.

4 files changed, 145 insertions, 16 deletions

diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java
index 5b1a4412b41..ed88667a14a 100644
--- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java
+++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java
@@ -1,11 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.docprocs.indexing;
 
+import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.DataType;
 import com.yahoo.document.Document;
 import com.yahoo.document.DocumentType;
 import com.yahoo.document.DocumentUpdate;
 import com.yahoo.document.Field;
+import com.yahoo.document.MapDataType;
 import com.yahoo.document.StructDataType;
 import com.yahoo.document.annotation.SpanTree;
 import com.yahoo.document.annotation.SpanTrees;
@@ -16,6 +18,7 @@ import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.Struct;
 import com.yahoo.document.datatypes.WeightedSet;
 import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
 import com.yahoo.document.update.FieldUpdate;
 import com.yahoo.document.update.MapValueUpdate;
 import com.yahoo.document.update.ValueUpdate;
@@ -30,6 +33,7 @@ import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -76,7 +80,7 @@ public class DocumentScriptTestCase {
         out = (StringFieldValue)processFieldUpdate(in).getValue();
         assertSpanTrees(out, "mySpanTree");
 
-        out = (StringFieldValue)processPathUpdate(in).getValue();
+        out = (StringFieldValue)processPathUpdate(in);
         assertSpanTrees(out, "mySpanTree");
     }
 
@@ -93,7 +97,7 @@ public class DocumentScriptTestCase {
         assertEquals(1, out.size());
         assertSpanTrees(out.get(0), "mySpanTree");
 
-        out = (Array<StringFieldValue>)processPathUpdate(in).getValue();
+        out = (Array<StringFieldValue>)processPathUpdate(in);
         assertEquals(1, out.size());
         assertSpanTrees(out.get(0), "mySpanTree");
     }
@@ -111,7 +115,7 @@ public class DocumentScriptTestCase {
         assertEquals(1, out.size());
         assertSpanTrees(out.keySet().iterator().next(), "mySpanTree");
 
-        out = (WeightedSet<StringFieldValue>)processPathUpdate(in).getValue();
+        out = (WeightedSet<StringFieldValue>)processPathUpdate(in);
         assertEquals(1, out.size());
         assertSpanTrees(out.keySet().iterator().next(), "mySpanTree");
     }
@@ -134,7 +138,7 @@ public class DocumentScriptTestCase {
         assertSpanTrees(out.keySet().iterator().next(), "myKeySpanTree");
         assertSpanTrees(out.values().iterator().next(), "myValueSpanTree");
 
-        out = (MapFieldValue<StringFieldValue, StringFieldValue>)processPathUpdate(in).getValue();
+        out = (MapFieldValue<StringFieldValue, StringFieldValue>)processPathUpdate(in);
         assertEquals(1, out.size());
         assertSpanTrees(out.keySet().iterator().next(), "myKeySpanTree");
         assertSpanTrees(out.values().iterator().next(), "myValueSpanTree");
@@ -157,6 +161,62 @@ public class DocumentScriptTestCase {
         assertSpanTrees(str, "mySpanTree");
     }
 
+    private class FieldPathFixture {
+        final DocumentType type;
+        final StructDataType structType;
+        final DataType structMap;
+        final DataType structArray;
+
+        FieldPathFixture() {
+            type = newDocumentType();
+            structType = new StructDataType("mystruct");
+            structType.addField(new Field("title", DataType.STRING));
+            structType.addField(new Field("rating", DataType.INT));
+            structArray = new ArrayDataType(structType);
+            type.addField(new Field("structarray", structArray));
+            structMap = new MapDataType(DataType.STRING, structType);
+            type.addField(new Field("structmap", structMap));
+        }
+
+        FieldPathUpdate executeWithUpdate(String fieldName, FieldPathUpdate updateIn) {
+            DocumentUpdate update = new DocumentUpdate(type, "doc:scheme:");
+            update.addFieldPathUpdate(updateIn);
+            update = newScript(type, fieldName).execute(ADAPTER_FACTORY, update);
+            assertEquals(1, update.getFieldPathUpdates().size());
+            return update.getFieldPathUpdates().get(0);
+        }
+    }
+
+    @Test
+    public void array_field_path_updates_survive_indexing_scripts() {
+        FieldPathFixture f = new FieldPathFixture();
+
+        Struct newElemValue = new Struct(f.structType);
+        newElemValue.setFieldValue("title", "iron moose 2, the moosening");
+
+        FieldPathUpdate updated = f.executeWithUpdate("structarray", new AssignFieldPathUpdate(f.type, "structarray[10]", newElemValue));
+
+        assertTrue(updated instanceof AssignFieldPathUpdate);
+        AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)updated;
+        assertEquals("structarray[10]", assignUpdate.getOriginalFieldPath());
+        assertEquals(newElemValue, assignUpdate.getFieldValue());
+    }
+
+    @Test
+    public void map_field_path_updates_survive_indexing_scripts() {
+        FieldPathFixture f = new FieldPathFixture();
+
+        Struct newElemValue = new Struct(f.structType);
+        newElemValue.setFieldValue("title", "iron moose 3, moose in new york");
+
+        FieldPathUpdate updated = f.executeWithUpdate("structmap", new AssignFieldPathUpdate(f.type, "structmap{foo}", newElemValue));
+
+        assertTrue(updated instanceof AssignFieldPathUpdate);
+        AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)updated;
+        assertEquals("structmap{foo}", assignUpdate.getOriginalFieldPath());
+        assertEquals(newElemValue, assignUpdate.getFieldValue());
+    }
+
     private static FieldValue processDocument(FieldValue fieldValue) {
         DocumentType docType = new DocumentType("myDocumentType");
         docType.addField("myField", fieldValue.getDataType());
@@ -175,20 +235,24 @@ public class DocumentScriptTestCase {
         return update.getFieldUpdate("myField").getValueUpdate(0);
     }
 
-    private static ValueUpdate<?> processPathUpdate(FieldValue fieldValue) {
+    private static FieldValue processPathUpdate(FieldValue fieldValue) {
         DocumentType docType = new DocumentType("myDocumentType");
         docType.addField("myField", fieldValue.getDataType());
         DocumentUpdate update = new DocumentUpdate(docType, "doc:scheme:");
         update.addFieldPathUpdate(new AssignFieldPathUpdate(docType, "myField", fieldValue));
         update = newScript(docType).execute(ADAPTER_FACTORY, update);
-        return update.getFieldUpdate("myField").getValueUpdate(0);
+        return ((AssignFieldPathUpdate)update.getFieldPathUpdates().get(0)).getFieldValue();
+    }
+
+    private static DocumentScript newScript(DocumentType docType, String fieldName) {
+        return new DocumentScript(docType.getName(), Collections.singletonList(fieldName),
+                new StatementExpression(new InputExpression(fieldName),
+                        new IndexExpression(fieldName)));
     }
 
     private static DocumentScript newScript(DocumentType docType) {
         String fieldName = docType.getFields().iterator().next().getName();
-        return new DocumentScript(docType.getName(), Arrays.asList(fieldName),
-                                  new StatementExpression(new InputExpression(fieldName),
-                                                          new IndexExpression(fieldName)));
+        return newScript(docType, fieldName);
     }
 
     private static StringFieldValue newString(String... spanTrees) {
@@ -210,6 +274,7 @@ public class DocumentScriptTestCase {
         DocumentType type = new DocumentType("documentType");
         type.addField("documentField", DataType.STRING);
         type.addField("extraField", DataType.STRING);
+
         return type;
     }
 
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java
index e37a32f28f4..01326bcea62 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/Array.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java
@@ -290,7 +290,8 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i
         if (pos < fieldPath.size()) {
             switch (fieldPath.get(pos).getType()) {
                 case ARRAY_INDEX:
-                    return iterateSubset(fieldPath.get(pos).getLookupIndex(), fieldPath.get(pos).getLookupIndex(), fieldPath, null, pos + 1, handler);
+                    final int elemIndex = fieldPath.get(pos).getLookupIndex();
+                    return iterateSubset(elemIndex, elemIndex, fieldPath, null, pos + 1, handler);
                 case VARIABLE: {
                     FieldPathIteratorHandler.IndexValue val = handler.getVariables().get(fieldPath.get(pos).getVariableName());
                     if (val != null) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/IdentityFieldPathUpdateAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/IdentityFieldPathUpdateAdapter.java
new file mode 100644
index 00000000000..42c9bd8c10c
--- /dev/null
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/IdentityFieldPathUpdateAdapter.java
@@ -0,0 +1,68 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.Document;
+import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.FieldPath;
+import com.yahoo.document.datatypes.FieldValue;
+import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
+import com.yahoo.vespa.indexinglanguage.expressions.Expression;
+import com.yahoo.vespa.indexinglanguage.expressions.FieldValueAdapter;
+
+/**
+ * No-op update adapter which simply passes through the input update unchanged.
+ * I.e. getOutput() will return a DocumentUpdate containing only the FieldPathUpdate
+ * the IdentityFieldPathUpdateAdapter was created with. All other applicable calls are
+ * forwarded to the provided DocumentAdapter instance.
+ *
+ * This removes the need for a potentially lossy round-trip of update -&gt; synthetic document -&gt; update.
+ */
+public class IdentityFieldPathUpdateAdapter implements UpdateAdapter {
+
+    private final FieldPathUpdate update;
+    private final DocumentAdapter fwdAdapter;
+
+    public IdentityFieldPathUpdateAdapter(FieldPathUpdate update, DocumentAdapter fwdAdapter) {
+        this.update = update;
+        this.fwdAdapter = fwdAdapter;
+    }
+
+    @Override
+    public DocumentUpdate getOutput() {
+        Document doc = fwdAdapter.getFullOutput();
+        DocumentUpdate upd = new DocumentUpdate(doc.getDataType(), doc.getId());
+        upd.addFieldPathUpdate(update);
+        return upd;
+    }
+
+    @Override
+    public Expression getExpression(Expression expression) {
+        return expression;
+    }
+
+    @Override
+    public FieldValue getInputValue(String fieldName) {
+        return fwdAdapter.getInputValue(fieldName);
+    }
+
+    @Override
+    public FieldValue getInputValue(FieldPath fieldPath) {
+        return fwdAdapter.getInputValue(fieldPath);
+    }
+
+    @Override
+    public FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) {
+        return fwdAdapter.setOutputValue(exp, fieldName, fieldValue);
+    }
+
+    @Override
+    public DataType getInputType(Expression exp, String fieldName) {
+        return fwdAdapter.getInputType(exp, fieldName);
+    }
+
+    @Override
+    public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        fwdAdapter.tryOutputType(exp, fieldName, valueType);
+    }
+}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
index 2ad09dfbdc4..038e91b7481 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java
@@ -48,12 +48,7 @@ public class SimpleAdapterFactory implements AdapterFactory {
         DocumentId docId = upd.getId();
         Document complete = new Document(docType, upd.getId());
         for (FieldPathUpdate fieldUpd : upd) {
-            if (FieldPathUpdateHelper.isComplete(fieldUpd)) {
-                FieldPathUpdateHelper.applyUpdate(fieldUpd, complete);
-            } else {
-                Document partial = FieldPathUpdateHelper.newPartialDocument(docId, fieldUpd);
-                ret.add(new FieldPathUpdateAdapter(newDocumentAdapter(partial, true), fieldUpd));
-            }
+            ret.add(new IdentityFieldPathUpdateAdapter(fieldUpd, newDocumentAdapter(complete, true)));
         }
         for (FieldUpdate fieldUpd : upd.getFieldUpdates()) {
             Field field = fieldUpd.getField();