Support choice expressions

4 files changed, 50 insertions, 82 deletions

diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
index b50c0f652b5..8968c87694f 100644
--- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
+++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java
@@ -84,28 +84,29 @@ public class DocumentScript {
     private void removeAnyLinguisticsSpanTree(FieldValue value) {
         if (value instanceof StringFieldValue) {
             ((StringFieldValue)value).removeSpanTree(SpanTrees.LINGUISTICS);
-        } else if (value instanceof Array) {
-            Array<?> arr = (Array)value;
-            for (Object obj : arr.getValues()) {
-                removeAnyLinguisticsSpanTree((FieldValue)obj);
+        } else if (value instanceof Array<?> arr) {
+            for (FieldValue fieldValue : arr.getValues()) {
+                removeAnyLinguisticsSpanTree(fieldValue);
             }
-        } else if (value instanceof WeightedSet) {
-            WeightedSet<?> wset = (WeightedSet)value;
-            for (Object obj : wset.keySet()) {
-                removeAnyLinguisticsSpanTree((FieldValue)obj);
+        } else if (value instanceof WeightedSet<?> wset) {
+            for (FieldValue fieldValue : wset.keySet()) {
+                removeAnyLinguisticsSpanTree(fieldValue);
             }
-        } else if (value instanceof MapFieldValue) {
-            MapFieldValue<?,?> map = (MapFieldValue)value;
-            for (Map.Entry<?,?> entry : map.entrySet()) {
+        } else if (value instanceof MapFieldValue<?, ?> map) {
+            for (Map.Entry<?, ?> entry : map.entrySet()) {
                 removeAnyLinguisticsSpanTree((FieldValue)entry.getKey());
                 removeAnyLinguisticsSpanTree((FieldValue)entry.getValue());
             }
-        } else if (value instanceof StructuredFieldValue) {
-            StructuredFieldValue struct = (StructuredFieldValue)value;
+        } else if (value instanceof StructuredFieldValue struct) {
             for (Iterator<Map.Entry<Field, FieldValue>> it = struct.iterator(); it.hasNext();) {
                 removeAnyLinguisticsSpanTree(it.next().getValue());
             }
         }
     }
 
+    @Override
+    public String toString() {
+        return "indexing script for '" + documentType + "' given inputs " + inputFields + ": " + expression;
+    }
+
 }
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
index 00ee3a90420..489e9031b19 100644
--- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
+++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
@@ -82,8 +82,7 @@ public class ScriptManager {
             for (String content : ilscript.content()) {
                 StatementExpression statement = parse(ilscript.doctype(), parserContext, content);
                 expressions.add(statement);
-                InputExpression.InputFieldNameExtractor inputFieldNameExtractor = new InputExpression.InputFieldNameExtractor();
-                statement.select(inputFieldNameExtractor, inputFieldNameExtractor);
+                List<String> inputFieldNames = InputExpression.InputFieldNameExtractor.runOn(statement);
                 OutputExpression.OutputFieldNameExtractor outputFieldNameExtractor = new OutputExpression.OutputFieldNameExtractor();
                 statement.select(outputFieldNameExtractor, outputFieldNameExtractor);
                 statement.select(fieldPathOptimizer, fieldPathOptimizer);
@@ -91,8 +90,8 @@ public class ScriptManager {
                     String outputFieldName = outputFieldNameExtractor.getOutputFieldNames().get(0);
                     statement.setStatementOutput(documentType, documentType.getField(outputFieldName));
                 }
-                if (inputFieldNameExtractor.getInputFieldNames().size() == 1) {
-                    String fieldName = inputFieldNameExtractor.getInputFieldNames().get(0);
+                if (inputFieldNames.size() == 1) {
+                    String fieldName = inputFieldNames.get(0);
                     ScriptExpression script;
                     if (fieldScripts.containsKey(fieldName)) {
                         DocumentScript prev = fieldScripts.get(fieldName);
@@ -102,11 +101,11 @@ public class ScriptManager {
                     } else {
                         script = new ScriptExpression(statement);
                     }
-                    DocumentScript documentScript = new DocumentScript(ilscript.doctype(), inputFieldNameExtractor.getInputFieldNames(), script);
+                    DocumentScript documentScript = new DocumentScript(ilscript.doctype(), inputFieldNames, script);
                     fieldScripts.put(fieldName, documentScript);
                 } else {
-                    log.log(Level.FINE, "Non single(" + inputFieldNameExtractor.getInputFieldNames().size() +"" +
-                                        ") inputs = " + inputFieldNameExtractor.getInputFieldNames() + ". Script = " + statement);
+                    log.log(Level.FINE, "Non single(" + inputFieldNames.size() +"" +
+                                        ") inputs = " + inputFieldNames + ". Script = " + statement);
                 }
             }
 
diff --git a/docprocs/src/test/cfg/ilscripts.cfg b/docprocs/src/test/cfg/ilscripts.cfg
index f54aebb1685..cab3ee0aa0a 100644
--- a/docprocs/src/test/cfg/ilscripts.cfg
+++ b/docprocs/src/test/cfg/ilscripts.cfg
@@ -11,5 +11,5 @@ ilscript[0].content[1] "input title | attribute artist"
 ilscript[0].content[2] "input isbn | passthrough isbn"
 ilscript[0].content[3] "input isbn | attribute song"
 ilscript[0].content[4] "input artist . " ".  input title | index combined"
-ilscript[0].content[5] "input artist . " ".  input title | index combinedWithFallback"
+ilscript[0].content[5] "(input artist || "") . " ".  (input title || "") | index combinedWithFallback"
 
diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java
index f4049777d77..2cec7dd4371 100644
--- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java
+++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java
@@ -24,11 +24,13 @@ import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 /**
  * @author Simon Thoresen Hult
+ * @author bratseth
  */
 public class IndexingProcessorTestCase {
 
@@ -57,51 +59,7 @@ public class IndexingProcessorTestCase {
     }
 
     @Test
-    public void requireThatIndexerProcessesUpdates() {
-        DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music");
-        DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::");
-        input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("isbn"), new StringFieldValue("isbnmarker")));
-        input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("artist"), new StringFieldValue("69")));
-        DocumentOperation output = process(input);
-
-        assertTrue(output instanceof DocumentUpdate);
-        DocumentUpdate docUpdate = (DocumentUpdate) output;
-
-        assertEquals(3, docUpdate.fieldUpdates().size());
-        {
-            FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("song");
-            assertEquals("song", fieldUpdate.getField().getName());
-            assertEquals(1, fieldUpdate.getValueUpdates().size());
-            ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0);
-            assertTrue(valueUpdate instanceof AssignValueUpdate);
-            assertEquals(new StringFieldValue("isbnmarker"), valueUpdate.getValue());
-            fieldUpdate = docUpdate.getFieldUpdate("title");
-            assertEquals("title", fieldUpdate.getField().getName());
-            assertEquals(1, fieldUpdate.getValueUpdates().size());
-            valueUpdate = fieldUpdate.getValueUpdate(0);
-            assertTrue(valueUpdate instanceof AssignValueUpdate);
-            assertEquals(new StringFieldValue("69"), valueUpdate.getValue());
-        }
-
-        {
-            FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("title");
-            ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0);
-            assertEquals("title", fieldUpdate.getField().getName());
-            assertTrue(valueUpdate instanceof AssignValueUpdate);
-            assertEquals(new StringFieldValue("69"), valueUpdate.getValue());
-        }
-        {
-            FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("isbn");
-            ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0);
-            assertEquals("isbn", fieldUpdate.getField().getName());
-            assertTrue(valueUpdate instanceof AssignValueUpdate);
-            assertEquals(new StringFieldValue("isbnmarker"), valueUpdate.getValue());
-        }
-
-    }
-
-    @Test
-    public void testFieldDependingOnTwoInputs() {
+    public void testFieldUpdates() {
         // 'artist' is assigned to 'title' and vice versa
         // 'combined' gets the value of both
         // 'combinedWithFallback' falls back to an empty string if an input is missing
@@ -124,33 +82,42 @@ public class IndexingProcessorTestCase {
             DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music");
             DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::");
             input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("artist"), new StringFieldValue("artist1")));
-            // no title
 
             DocumentUpdate output = (DocumentUpdate)process(input);
-            assertEquals(1, output.fieldUpdates().size());
+            assertEquals(2, output.fieldUpdates().size());
             assertAssignment("title", "artist1", output);
+            assertAssignment("combinedWithFallback", "artist1 ", output);
         }
 
         {   // Just title is set
             DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music");
             DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::");
             input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("title"), new StringFieldValue("title1")));
-            // no title
 
             DocumentUpdate output = (DocumentUpdate)process(input);
-            assertEquals(1, output.fieldUpdates().size());
+            assertEquals(2, output.fieldUpdates().size());
             assertAssignment("artist", "title1", output);
+            assertAssignment("combinedWithFallback", " title1", output);
         }
-    }
 
-    @Test
-    public void requireThatEmptyDocumentUpdateOutputDoesNotThrow() {
-        DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music");
-        DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::");
-        Processing proc = new Processing();
-        proc.getDocumentOperations().add(input);
-        indexer.process(proc);
-        assertEquals(0, proc.getDocumentOperations().size());
+        {   // Neither title nor artist is set: Should not update embeddings even though it has fallbacks for all
+            DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music");
+            DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::");
+            input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("isbn"), new StringFieldValue("isbn1")));
+
+            DocumentUpdate output = (DocumentUpdate)process(input);
+            assertEquals(2, output.fieldUpdates().size());
+            assertAssignment("isbn", "isbn1", output);
+            assertAssignment("song", "isbn1", output);
+        }
+
+        {   // None is set: Should not update anything
+            DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music");
+            DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::");
+
+            DocumentUpdate output = (DocumentUpdate)process(input);
+            assertNull(output);
+        }
     }
 
     @Test
@@ -175,9 +142,10 @@ public class IndexingProcessorTestCase {
         proc.getDocumentOperations().add(input);
         indexer.process(proc);
 
-        List<DocumentOperation> lst = proc.getDocumentOperations();
-        assertEquals(1, lst.size());
-        return lst.get(0);
+        List<DocumentOperation> operations = proc.getDocumentOperations();
+        if (operations.isEmpty()) return null;
+        assertEquals(1, operations.size());
+        return operations.get(0);
     }
 
     @SuppressWarnings("deprecation")