diff options
author | Jon Bratseth <bratseth@gmail.com> | 2023-01-20 13:54:28 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2023-01-20 13:54:28 +0100 |
commit | 97707f8b011edf555776b6a6bf36d35458f583ae (patch) | |
tree | aec9c68dd32e39241c34fdc1bfa270cdf19d4aa2 /docprocs | |
parent | 1e29c4da6f91d13a6889d1b71058a09fcbb4eb43 (diff) |
Support choice expressions
Diffstat (limited to 'docprocs')
4 files changed, 50 insertions, 82 deletions
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java index b50c0f652b5..8968c87694f 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java @@ -84,28 +84,29 @@ public class DocumentScript { private void removeAnyLinguisticsSpanTree(FieldValue value) { if (value instanceof StringFieldValue) { ((StringFieldValue)value).removeSpanTree(SpanTrees.LINGUISTICS); - } else if (value instanceof Array) { - Array<?> arr = (Array)value; - for (Object obj : arr.getValues()) { - removeAnyLinguisticsSpanTree((FieldValue)obj); + } else if (value instanceof Array<?> arr) { + for (FieldValue fieldValue : arr.getValues()) { + removeAnyLinguisticsSpanTree(fieldValue); } - } else if (value instanceof WeightedSet) { - WeightedSet<?> wset = (WeightedSet)value; - for (Object obj : wset.keySet()) { - removeAnyLinguisticsSpanTree((FieldValue)obj); + } else if (value instanceof WeightedSet<?> wset) { + for (FieldValue fieldValue : wset.keySet()) { + removeAnyLinguisticsSpanTree(fieldValue); } - } else if (value instanceof MapFieldValue) { - MapFieldValue<?,?> map = (MapFieldValue)value; - for (Map.Entry<?,?> entry : map.entrySet()) { + } else if (value instanceof MapFieldValue<?, ?> map) { + for (Map.Entry<?, ?> entry : map.entrySet()) { removeAnyLinguisticsSpanTree((FieldValue)entry.getKey()); removeAnyLinguisticsSpanTree((FieldValue)entry.getValue()); } - } else if (value instanceof StructuredFieldValue) { - StructuredFieldValue struct = (StructuredFieldValue)value; + } else if (value instanceof StructuredFieldValue struct) { for (Iterator<Map.Entry<Field, FieldValue>> it = struct.iterator(); it.hasNext();) { removeAnyLinguisticsSpanTree(it.next().getValue()); } } } + @Override + public String toString() { + return "indexing script for '" + documentType + "' given inputs " + inputFields + ": " + expression; + } + } diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 00ee3a90420..489e9031b19 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -82,8 +82,7 @@ public class ScriptManager { for (String content : ilscript.content()) { StatementExpression statement = parse(ilscript.doctype(), parserContext, content); expressions.add(statement); - InputExpression.InputFieldNameExtractor inputFieldNameExtractor = new InputExpression.InputFieldNameExtractor(); - statement.select(inputFieldNameExtractor, inputFieldNameExtractor); + List<String> inputFieldNames = InputExpression.InputFieldNameExtractor.runOn(statement); OutputExpression.OutputFieldNameExtractor outputFieldNameExtractor = new OutputExpression.OutputFieldNameExtractor(); statement.select(outputFieldNameExtractor, outputFieldNameExtractor); statement.select(fieldPathOptimizer, fieldPathOptimizer); @@ -91,8 +90,8 @@ public class ScriptManager { String outputFieldName = outputFieldNameExtractor.getOutputFieldNames().get(0); statement.setStatementOutput(documentType, documentType.getField(outputFieldName)); } - if (inputFieldNameExtractor.getInputFieldNames().size() == 1) { - String fieldName = inputFieldNameExtractor.getInputFieldNames().get(0); + if (inputFieldNames.size() == 1) { + String fieldName = inputFieldNames.get(0); ScriptExpression script; if (fieldScripts.containsKey(fieldName)) { DocumentScript prev = fieldScripts.get(fieldName); @@ -102,11 +101,11 @@ public class ScriptManager { } else { script = new ScriptExpression(statement); } - DocumentScript documentScript = new DocumentScript(ilscript.doctype(), inputFieldNameExtractor.getInputFieldNames(), script); + DocumentScript documentScript = new DocumentScript(ilscript.doctype(), inputFieldNames, script); fieldScripts.put(fieldName, documentScript); } else { - log.log(Level.FINE, "Non single(" + inputFieldNameExtractor.getInputFieldNames().size() +"" + - ") inputs = " + inputFieldNameExtractor.getInputFieldNames() + ". Script = " + statement); + log.log(Level.FINE, "Non single(" + inputFieldNames.size() +"" + + ") inputs = " + inputFieldNames + ". Script = " + statement); } } diff --git a/docprocs/src/test/cfg/ilscripts.cfg b/docprocs/src/test/cfg/ilscripts.cfg index f54aebb1685..cab3ee0aa0a 100644 --- a/docprocs/src/test/cfg/ilscripts.cfg +++ b/docprocs/src/test/cfg/ilscripts.cfg @@ -11,5 +11,5 @@ ilscript[0].content[1] "input title | attribute artist" ilscript[0].content[2] "input isbn | passthrough isbn" ilscript[0].content[3] "input isbn | attribute song" ilscript[0].content[4] "input artist . " ". input title | index combined" -ilscript[0].content[5] "input artist . " ". input title | index combinedWithFallback" +ilscript[0].content[5] "(input artist || "") . " ". (input title || "") | index combinedWithFallback" diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java index f4049777d77..2cec7dd4371 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java @@ -24,11 +24,13 @@ import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** * @author Simon Thoresen Hult + * @author bratseth */ public class IndexingProcessorTestCase { @@ -57,51 +59,7 @@ public class IndexingProcessorTestCase { } @Test - public void requireThatIndexerProcessesUpdates() { - DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music"); - DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::"); - input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("isbn"), new StringFieldValue("isbnmarker"))); - input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("artist"), new StringFieldValue("69"))); - DocumentOperation output = process(input); - - assertTrue(output instanceof DocumentUpdate); - DocumentUpdate docUpdate = (DocumentUpdate) output; - - assertEquals(3, docUpdate.fieldUpdates().size()); - { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("song"); - assertEquals("song", fieldUpdate.getField().getName()); - assertEquals(1, fieldUpdate.getValueUpdates().size()); - ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); - assertTrue(valueUpdate instanceof AssignValueUpdate); - assertEquals(new StringFieldValue("isbnmarker"), valueUpdate.getValue()); - fieldUpdate = docUpdate.getFieldUpdate("title"); - assertEquals("title", fieldUpdate.getField().getName()); - assertEquals(1, fieldUpdate.getValueUpdates().size()); - valueUpdate = fieldUpdate.getValueUpdate(0); - assertTrue(valueUpdate instanceof AssignValueUpdate); - assertEquals(new StringFieldValue("69"), valueUpdate.getValue()); - } - - { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("title"); - ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); - assertEquals("title", fieldUpdate.getField().getName()); - assertTrue(valueUpdate instanceof AssignValueUpdate); - assertEquals(new StringFieldValue("69"), valueUpdate.getValue()); - } - { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("isbn"); - ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); - assertEquals("isbn", fieldUpdate.getField().getName()); - assertTrue(valueUpdate instanceof AssignValueUpdate); - assertEquals(new StringFieldValue("isbnmarker"), valueUpdate.getValue()); - } - - } - - @Test - public void testFieldDependingOnTwoInputs() { + public void testFieldUpdates() { // 'artist' is assigned to 'title' and vice versa // 'combined' gets the value of both // 'combinedWithFallback' falls back to an empty string if an input is missing @@ -124,33 +82,42 @@ public class IndexingProcessorTestCase { DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music"); DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::"); input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("artist"), new StringFieldValue("artist1"))); - // no title DocumentUpdate output = (DocumentUpdate)process(input); - assertEquals(1, output.fieldUpdates().size()); + assertEquals(2, output.fieldUpdates().size()); assertAssignment("title", "artist1", output); + assertAssignment("combinedWithFallback", "artist1 ", output); } { // Just title is set DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music"); DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::"); input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("title"), new StringFieldValue("title1"))); - // no title DocumentUpdate output = (DocumentUpdate)process(input); - assertEquals(1, output.fieldUpdates().size()); + assertEquals(2, output.fieldUpdates().size()); assertAssignment("artist", "title1", output); + assertAssignment("combinedWithFallback", " title1", output); } - } - @Test - public void requireThatEmptyDocumentUpdateOutputDoesNotThrow() { - DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music"); - DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::"); - Processing proc = new Processing(); - proc.getDocumentOperations().add(input); - indexer.process(proc); - assertEquals(0, proc.getDocumentOperations().size()); + { // Neither title nor artist is set: Should not update embeddings even though it has fallbacks for all + DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music"); + DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::"); + input.addFieldUpdate(FieldUpdate.createAssign(inputType.getField("isbn"), new StringFieldValue("isbn1"))); + + DocumentUpdate output = (DocumentUpdate)process(input); + assertEquals(2, output.fieldUpdates().size()); + assertAssignment("isbn", "isbn1", output); + assertAssignment("song", "isbn1", output); + } + + { // None is set: Should not update anything + DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music"); + DocumentUpdate input = new DocumentUpdate(inputType, "id:ns:music::"); + + DocumentUpdate output = (DocumentUpdate)process(input); + assertNull(output); + } } @Test @@ -175,9 +142,10 @@ public class IndexingProcessorTestCase { proc.getDocumentOperations().add(input); indexer.process(proc); - List<DocumentOperation> lst = proc.getDocumentOperations(); - assertEquals(1, lst.size()); - return lst.get(0); + List<DocumentOperation> operations = proc.getDocumentOperations(); + if (operations.isEmpty()) return null; + assertEquals(1, operations.size()); + return operations.get(0); } @SuppressWarnings("deprecation") |