From 9fdcf8f92eaf3b47053fa2c131832dea1c792d0c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 30 Sep 2021 13:46:50 +0200 Subject: Pass destination This allows embedders to switch on it to enable bucket testing and similar. --- .../expressions/EmbedExpression.java | 12 +++++++++--- .../indexinglanguage/expressions/Expression.java | 5 ++++- .../indexinglanguage/expressions/ExpressionList.java | 6 ++++-- .../expressions/ForEachExpression.java | 5 +++++ .../expressions/GuardExpression.java | 5 +++-- .../expressions/IfThenExpression.java | 10 ++++++++++ .../expressions/ParenthesisExpression.java | 6 ++++++ .../expressions/SelectInputExpression.java | 8 ++++++++ .../expressions/SwitchExpression.java | 9 +++++++++ .../yahoo/vespa/indexinglanguage/ScriptTestCase.java | 20 ++++++++++++++------ 10 files changed, 72 insertions(+), 14 deletions(-) (limited to 'indexinglanguage/src') diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index aa579ed729e..043a30ce66d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; @@ -18,6 +20,9 @@ public class EmbedExpression extends Expression { private final Embedder embedder; + /** The destination the embedding will be written to on the form [schema name].[field name] */ + private String destination; + /** The target type we are embedding into. */ private TensorType targetType; @@ -27,14 +32,15 @@ public class EmbedExpression extends Expression { } @Override - public void setStatementOutputType(DataType type) { - targetType = ((TensorDataType)type).getTensorType(); + public void setStatementOutput(DocumentType documentType, Field field) { + targetType = ((TensorDataType)field.getDataType()).getTensorType(); + destination = documentType.getName() + "." + field.getName(); } @Override protected void doExecute(ExecutionContext context) { StringFieldValue input = (StringFieldValue) context.getValue(); - Tensor tensor = embedder.embed(input.getString(), context.getLanguage(), targetType); + Tensor tensor = embedder.embed(input.getString(), context.getLanguage(), destination, targetType); context.setValue(new TensorFieldValue(tensor)); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index 20a0c9804a9..3533f159f8c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.Document; +import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; @@ -31,7 +33,8 @@ public abstract class Expression extends Selectable { this.inputType = inputType; } - public void setStatementOutputType(DataType type) {} + /** Sets the document type and field the statement this expression is part of will write to */ + public void setStatementOutput(DocumentType documentType, Field field) {} public final FieldValue execute(FieldValue val) { return execute(new ExecutionContext().setValue(val)); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index 0f7c2a411de..c143d921e0e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -25,9 +27,9 @@ public abstract class ExpressionList extends CompositeExpr } @Override - public void setStatementOutputType(DataType type) { + public void setStatementOutput(DocumentType documentType, Field field) { for (Expression expression : expressions) - expression.setStatementOutputType(type); + expression.setStatementOutput(documentType, field); } public int size() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index 84dbe07d418..d8a9dea83ae 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -26,6 +26,11 @@ public final class ForEachExpression extends CompositeExpression { return exp; } + @Override + public void setStatementOutput(DocumentType documentType, Field field) { + exp.setStatementOutput(documentType, field); + } + @Override protected void doExecute(final ExecutionContext context) { FieldValue input = context.getValue(); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index 78c261caccb..ad97db1c366 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.UpdateAdapter; import com.yahoo.vespa.objects.ObjectOperation; @@ -27,8 +28,8 @@ public final class GuardExpression extends CompositeExpression { } @Override - public void setStatementOutputType(DataType type) { - exp.setStatementOutputType(type); + public void setStatementOutput(DocumentType documentType, Field field) { + exp.setStatementOutput(documentType, field); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index a6dfa4dd13e..d59c6ce9da1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.NumericFieldValue; import com.yahoo.vespa.objects.ObjectOperation; @@ -53,6 +55,14 @@ public final class IfThenExpression extends CompositeExpression { this.ifFalse = ifFalse; } + @Override + public void setStatementOutput(DocumentType documentType, Field field) { + lhs.setStatementOutput(documentType, field); + rhs.setStatementOutput(documentType, field); + ifTrue.setStatementOutput(documentType, field); + ifFalse.setStatementOutput(documentType, field); + } + public Expression getLeftHandSide() { return lhs; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java index 8f7e6bc7040..000bde7fcb7 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -22,6 +23,11 @@ public class ParenthesisExpression extends CompositeExpression { return innerExp; } + @Override + public void setStatementOutput(DocumentType documentType, Field field) { + innerExp.setStatementOutput(documentType, field); + } + @Override protected void doExecute(ExecutionContext context) { innerExp.execute(context); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index c711b92c7c1..47e44f2c07c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.collections.Pair; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -28,6 +30,12 @@ public final class SelectInputExpression extends CompositeExpression { this.cases = cases; } + @Override + public void setStatementOutput(DocumentType documentType, Field field) { + for (var casePair : cases) + casePair.getSecond().setStatementOutput(documentType, field); + } + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index 9a8e8719461..49b9218bdb1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.text.StringUtilities; @@ -44,6 +46,13 @@ public final class SwitchExpression extends CompositeExpression { return defaultExp; } + @Override + public void setStatementOutput(DocumentType documentType, Field field) { + defaultExp.setStatementOutput(documentType, field); + for (var expression : cases.values()) + expression.setStatementOutput(documentType, field); + } + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index 188426b1a06..e0c0a9faba8 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -104,12 +104,14 @@ public class ScriptTestCase { TensorType tensorType = TensorType.fromSpec("tensor(d[4])"); var expression = Expression.fromString("input myText | embed | attribute 'myTensor'", new SimpleLinguistics(), - new MockEncoder()); + new MockEmbedder("myDocument.myTensor")); SimpleTestAdapter adapter = new SimpleTestAdapter(); adapter.createField(new Field("myText", DataType.STRING)); - adapter.createField(new Field("myTensor", new TensorDataType(tensorType))); + var tensorField = new Field("myTensor", new TensorDataType(tensorType)); + adapter.createField(tensorField); adapter.setValue("myText", new StringFieldValue("input text")); + expression.setStatementOutput(new DocumentType("myDocument"), tensorField); // Necessary to resolve output type VerificationContext verificationContext = new VerificationContext(adapter); @@ -119,21 +121,27 @@ public class ScriptTestCase { context.setValue(new StringFieldValue("input text")); expression.execute(context); assertNotNull(context); - //assertTrue(context.getOutputType() instanceof TensorDataType); assertTrue(adapter.values.containsKey("myTensor")); assertEquals(Tensor.from(tensorType, "[7,3,0,0]"), ((TensorFieldValue)adapter.values.get("myTensor")).getTensor().get()); } - private static class MockEncoder implements Embedder { + private static class MockEmbedder implements Embedder { + + private final String expectedDestination; + + public MockEmbedder(String expectedDestination) { + this.expectedDestination = expectedDestination; + } @Override - public List embed(String text, Language language) { + public List embed(String text, Language language, String destination) { return null; } @Override - public Tensor embed(String text, Language language, TensorType tensorType) { + public Tensor embed(String text, Language language, String destination, TensorType tensorType) { + assertEquals(expectedDestination, destination); return Tensor.from(tensorType, "[7,3,0,0]"); } -- cgit v1.2.3