diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-09-30 13:46:50 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-09-30 13:46:50 +0200 |
commit | 9fdcf8f92eaf3b47053fa2c131832dea1c792d0c (patch) | |
tree | c4d8f2a7c8297fce1b4b6f07a32ab0daeac35aaa | |
parent | 1bc2cca4b527bb9a5a8c67744b0796c9fafbe024 (diff) |
Pass destination
This allows embedders to switch on it to enable bucket testing
and similar.
22 files changed, 128 insertions, 51 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b7ce40f19a2..e4c1a561dee 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -6259,7 +6259,8 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Embedder, java.util.Map)", + "public void <init>(java.lang.String, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Embedder, java.util.Map)", + "public java.lang.String destination()", "public static com.yahoo.search.query.profile.types.ConversionContext empty()" ], "fields": [] diff --git a/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java index d6bfd6dcd82..e8a2875b9b9 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java @@ -20,7 +20,6 @@ import com.yahoo.search.searchchain.Execution; * semantics are not necessary for the application. * * @see FederationSearcher - * @since 5.0.13 * @author Steinar Knutsen */ @After("*") diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 53be827073c..bcf4f568998 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -121,7 +121,8 @@ public class QueryProfileProperties extends Properties { if (fieldDescription != null) { if (i == name.size() - 1) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, new ConversionContext(profile.getRegistry(), + value = fieldDescription.getType().convertFrom(value, new ConversionContext(localName, + profile.getRegistry(), embedder, context)); if (value == null) diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java index e5b9eb1c1cd..8dfb67a9d5f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java @@ -12,29 +12,35 @@ import java.util.Map; */ public class ConversionContext { + private final String destination; private final CompiledQueryProfileRegistry registry; private final Embedder embedder; private final Language language; - public ConversionContext(CompiledQueryProfileRegistry registry, Embedder embedder, Map<String, String> context) { + public ConversionContext(String destination, CompiledQueryProfileRegistry registry, Embedder embedder, + Map<String, String> context) { + this.destination = destination; this.registry = registry; this.embedder = embedder; this.language = context.containsKey("language") ? Language.fromLanguageTag(context.get("language")) : Language.UNKNOWN; } + /** Returns the local name of the field which will receive the converted value (or null when this is empty) */ + public String destination() { return destination; } + /** Returns the profile registry, or null if none */ - CompiledQueryProfileRegistry getRegistry() {return registry;} + CompiledQueryProfileRegistry registry() {return registry;} - /** Returns the configured encoder, never null */ - Embedder getEncoder() { return embedder; } + /** Returns the configured embedder, never null */ + Embedder embedder() { return embedder; } /** Returns the language, which is never null but may be UNKNOWN */ - Language getLanguage() { return language; } + Language language() { return language; } /** Returns an empty context */ public static ConversionContext empty() { - return new ConversionContext(null, Embedder.throwsOnUse, Map.of()); + return new ConversionContext(null, null, Embedder.throwsOnUse, Map.of()); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java index ff12224823f..2b649d81ec8 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java @@ -60,7 +60,7 @@ public class QueryProfileFieldType extends FieldType { String profileId = object.toString(); if (profileId.startsWith("ref:")) profileId = profileId.substring("ref:".length()); - CompiledQueryProfile profile = context.getRegistry().getComponent(profileId); + CompiledQueryProfile profile = context.registry().getComponent(profileId); if (profile == null) return null; if (type != null && ! type.equals(profile.getType())) return null; return profile; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index cd21f0b3a61..05befb24da0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; -import com.yahoo.language.Language; -import com.yahoo.language.process.Embedder; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -44,21 +42,17 @@ public class TensorFieldType extends FieldType { @Override public Object convertFrom(Object o, ConversionContext context) { - return convertFrom(o, context.getEncoder(), context.getLanguage()); - } - - private Object convertFrom(Object o, Embedder embedder, Language language) { if (o instanceof Tensor) return o; - if (o instanceof String && ((String)o).startsWith("embed(")) return encode((String)o, embedder, language); + if (o instanceof String && ((String)o).startsWith("embed(")) return encode((String)o, context); if (o instanceof String) return Tensor.from(type, (String)o); return null; } - private Tensor encode(String s, Embedder embedder, Language language) { + private Tensor encode(String s, ConversionContext context) { if ( ! s.endsWith(")")) throw new IllegalArgumentException("Expected any string enclosed in embed(), but the argument does not end by ')'"); String text = s.substring("embed(".length(), s.length() - 1); - return embedder.embed(text, language, type); + return context.embedder().embed(text, context.language(), context.destination(), type); } public static TensorFieldType fromTypeString(String s) { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 3a426656185..1fa91c73fb0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -380,7 +380,7 @@ public class QueryProperties extends Properties { if (type == null) return value; // no type info -> keep as string FieldDescription field = type.getField(key); if (field == null) return value; // ditto - return field.getType().convertFrom(value, new ConversionContext(profileRegistry, embedder, context)); + return field.getType().convertFrom(value, new ConversionContext(key, profileRegistry, embedder, context)); } private void throwIllegalParameter(String key,String namespace) { diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index e22263070e0..e63c7711ff2 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -447,24 +447,31 @@ public class QueryProfileTypeTestCase { CompiledQueryProfileRegistry cRegistry = registry.compile(); String textToEmbed = "text to embed into a tensor"; + String destinationFeature = "query(myTensor4)"; Tensor expectedTensor = Tensor.from("tensor<float>(x[5]):[3,7,4,0,0]]"); - Query query1 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") + + Query query1 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features." + destinationFeature) + "=" + urlEncode("embed(" + textToEmbed + ")"), com.yahoo.jdisc.http.HttpRequest.Method.GET)) .setQueryProfile(cRegistry.getComponent("test")) - .setEmbedder(new MockEmbedder(textToEmbed, Language.UNKNOWN, expectedTensor)) + .setEmbedder(new MockEmbedder(textToEmbed, + Language.UNKNOWN, + destinationFeature, + expectedTensor)) .build(); assertEquals(0, query1.errors().size()); assertEquals(expectedTensor, query1.properties().get("ranking.features.query(myTensor4)")); assertEquals(expectedTensor, query1.getRanking().getFeatures().getTensor("query(myTensor4)").get()); // Explicit language - Query query2 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") + + Query query2 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features." + destinationFeature) + "=" + urlEncode("embed(" + textToEmbed + ")") + "&language=en", com.yahoo.jdisc.http.HttpRequest.Method.GET)) .setQueryProfile(cRegistry.getComponent("test")) - .setEmbedder(new MockEmbedder(textToEmbed, Language.ENGLISH, expectedTensor)) + .setEmbedder(new MockEmbedder(textToEmbed, + Language.ENGLISH, + destinationFeature, + expectedTensor)) .build(); assertEquals(0, query2.errors().size()); assertEquals(expectedTensor, query2.properties().get("ranking.features.query(myTensor4)")); @@ -726,26 +733,30 @@ public class QueryProfileTypeTestCase { private final String expectedText; private final Language expectedLanguage; + private final String expectedDestination; private final Tensor tensorToReturn; public MockEmbedder(String expectedText, Language expectedLanguage, + String expectedDestination, Tensor tensorToReturn) { this.expectedText = expectedText; this.expectedLanguage = expectedLanguage; + this.expectedDestination = expectedDestination; this.tensorToReturn = tensorToReturn; } @Override - public List<Integer> embed(String text, Language language) { + public List<Integer> embed(String text, Language language, String destination) { fail("Unexpected call"); return null; } @Override - public Tensor embed(String text, Language language, TensorType tensorType) { + public Tensor embed(String text, Language language, String destination, TensorType tensorType) { assertEquals(expectedText, text); assertEquals(expectedLanguage, language); + assertEquals(expectedDestination, destination); assertEquals(tensorToReturn.type(), tensorType); return tensorToReturn; } diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 7e1d5b5b6ce..2202d8b99f5 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -82,7 +82,8 @@ public class ScriptManager { parserContext.getAnnotatorConfig().setMaxTokenLength(config.fieldmatchmaxlength()); for (IlscriptsConfig.Ilscript ilscript : config.ilscript()) { - InputExpression.FieldPathOptimizer fieldPathOptimizer = new InputExpression.FieldPathOptimizer(docTypeMgr.getDocumentType(ilscript.doctype())); + DocumentType documentType = docTypeMgr.getDocumentType(ilscript.doctype()); + InputExpression.FieldPathOptimizer fieldPathOptimizer = new InputExpression.FieldPathOptimizer(documentType); List<StatementExpression> expressions = new ArrayList<>(ilscript.content().size()); Map<String, DocumentScript> fieldScripts = new HashMap<>(ilscript.content().size()); for (String content : ilscript.content()) { @@ -95,7 +96,7 @@ public class ScriptManager { statement.select(fieldPathOptimizer, fieldPathOptimizer); if ( ! outputFieldNameExtractor.getOutputFieldNames().isEmpty()) { String outputFieldName = outputFieldNameExtractor.getOutputFieldNames().get(0); - statement.setStatementOutputType(docTypeMgr.getDocumentType(ilscript.doctype()).getField(outputFieldName).getDataType()); + statement.setStatementOutput(documentType, documentType.getField(outputFieldName)); } if (inputFieldNameExtractor.getInputFieldNames().size() == 1) { String fieldName = inputFieldNameExtractor.getInputFieldNames().get(0); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index aa579ed729e..043a30ce66d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; @@ -18,6 +20,9 @@ public class EmbedExpression extends Expression { private final Embedder embedder; + /** The destination the embedding will be written to on the form [schema name].[field name] */ + private String destination; + /** The target type we are embedding into. */ private TensorType targetType; @@ -27,14 +32,15 @@ public class EmbedExpression extends Expression { } @Override - public void setStatementOutputType(DataType type) { - targetType = ((TensorDataType)type).getTensorType(); + public void setStatementOutput(DocumentType documentType, Field field) { + targetType = ((TensorDataType)field.getDataType()).getTensorType(); + destination = documentType.getName() + "." + field.getName(); } @Override protected void doExecute(ExecutionContext context) { StringFieldValue input = (StringFieldValue) context.getValue(); - Tensor tensor = embedder.embed(input.getString(), context.getLanguage(), targetType); + Tensor tensor = embedder.embed(input.getString(), context.getLanguage(), destination, targetType); context.setValue(new TensorFieldValue(tensor)); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index 20a0c9804a9..3533f159f8c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.Document; +import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; @@ -31,7 +33,8 @@ public abstract class Expression extends Selectable { this.inputType = inputType; } - public void setStatementOutputType(DataType type) {} + /** Sets the document type and field the statement this expression is part of will write to */ + public void setStatementOutput(DocumentType documentType, Field field) {} public final FieldValue execute(FieldValue val) { return execute(new ExecutionContext().setValue(val)); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index 0f7c2a411de..c143d921e0e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -25,9 +27,9 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr } @Override - public void setStatementOutputType(DataType type) { + public void setStatementOutput(DocumentType documentType, Field field) { for (Expression expression : expressions) - expression.setStatementOutputType(type); + expression.setStatementOutput(documentType, field); } public int size() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index 84dbe07d418..d8a9dea83ae 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -27,6 +27,11 @@ public final class ForEachExpression extends CompositeExpression { } @Override + public void setStatementOutput(DocumentType documentType, Field field) { + exp.setStatementOutput(documentType, field); + } + + @Override protected void doExecute(final ExecutionContext context) { FieldValue input = context.getValue(); if (input instanceof Array || input instanceof WeightedSet) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index 78c261caccb..ad97db1c366 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.UpdateAdapter; import com.yahoo.vespa.objects.ObjectOperation; @@ -27,8 +28,8 @@ public final class GuardExpression extends CompositeExpression { } @Override - public void setStatementOutputType(DataType type) { - exp.setStatementOutputType(type); + public void setStatementOutput(DocumentType documentType, Field field) { + exp.setStatementOutput(documentType, field); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index a6dfa4dd13e..d59c6ce9da1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.NumericFieldValue; import com.yahoo.vespa.objects.ObjectOperation; @@ -53,6 +55,14 @@ public final class IfThenExpression extends CompositeExpression { this.ifFalse = ifFalse; } + @Override + public void setStatementOutput(DocumentType documentType, Field field) { + lhs.setStatementOutput(documentType, field); + rhs.setStatementOutput(documentType, field); + ifTrue.setStatementOutput(documentType, field); + ifFalse.setStatementOutput(documentType, field); + } + public Expression getLeftHandSide() { return lhs; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java index 8f7e6bc7040..000bde7fcb7 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -23,6 +24,11 @@ public class ParenthesisExpression extends CompositeExpression { } @Override + public void setStatementOutput(DocumentType documentType, Field field) { + innerExp.setStatementOutput(documentType, field); + } + + @Override protected void doExecute(ExecutionContext context) { innerExp.execute(context); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index c711b92c7c1..47e44f2c07c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.collections.Pair; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -29,6 +31,12 @@ public final class SelectInputExpression extends CompositeExpression { } @Override + public void setStatementOutput(DocumentType documentType, Field field) { + for (var casePair : cases) + casePair.getSecond().setStatementOutput(documentType, field); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); for (Pair<String, Expression> entry : cases) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index 9a8e8719461..49b9218bdb1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.text.StringUtilities; @@ -45,6 +47,13 @@ public final class SwitchExpression extends CompositeExpression { } @Override + public void setStatementOutput(DocumentType documentType, Field field) { + defaultExp.setStatementOutput(documentType, field); + for (var expression : cases.values()) + expression.setStatementOutput(documentType, field); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); Expression exp = null; diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index 188426b1a06..e0c0a9faba8 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -104,12 +104,14 @@ public class ScriptTestCase { TensorType tensorType = TensorType.fromSpec("tensor(d[4])"); var expression = Expression.fromString("input myText | embed | attribute 'myTensor'", new SimpleLinguistics(), - new MockEncoder()); + new MockEmbedder("myDocument.myTensor")); SimpleTestAdapter adapter = new SimpleTestAdapter(); adapter.createField(new Field("myText", DataType.STRING)); - adapter.createField(new Field("myTensor", new TensorDataType(tensorType))); + var tensorField = new Field("myTensor", new TensorDataType(tensorType)); + adapter.createField(tensorField); adapter.setValue("myText", new StringFieldValue("input text")); + expression.setStatementOutput(new DocumentType("myDocument"), tensorField); // Necessary to resolve output type VerificationContext verificationContext = new VerificationContext(adapter); @@ -119,21 +121,27 @@ public class ScriptTestCase { context.setValue(new StringFieldValue("input text")); expression.execute(context); assertNotNull(context); - //assertTrue(context.getOutputType() instanceof TensorDataType); assertTrue(adapter.values.containsKey("myTensor")); assertEquals(Tensor.from(tensorType, "[7,3,0,0]"), ((TensorFieldValue)adapter.values.get("myTensor")).getTensor().get()); } - private static class MockEncoder implements Embedder { + private static class MockEmbedder implements Embedder { + + private final String expectedDestination; + + public MockEmbedder(String expectedDestination) { + this.expectedDestination = expectedDestination; + } @Override - public List<Integer> embed(String text, Language language) { + public List<Integer> embed(String text, Language language, String destination) { return null; } @Override - public Tensor embed(String text, Language language, TensorType tensorType) { + public Tensor embed(String text, Language language, String destination, TensorType tensorType) { + assertEquals(expectedDestination, destination); return Tensor.from(tensorType, "[7,3,0,0]"); } diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index d257c451739..5865c28bbb6 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -338,8 +338,8 @@ ], "methods": [ "public void <init>()", - "public java.util.List embed(java.lang.String, com.yahoo.language.Language)", - "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)" + "public java.util.List embed(java.lang.String, com.yahoo.language.Language, java.lang.String)", + "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, java.lang.String, com.yahoo.tensor.TensorType)" ], "fields": [] }, @@ -352,8 +352,8 @@ "abstract" ], "methods": [ - "public abstract java.util.List embed(java.lang.String, com.yahoo.language.Language)", - "public abstract com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)" + "public abstract java.util.List embed(java.lang.String, com.yahoo.language.Language, java.lang.String)", + "public abstract com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, java.lang.String, com.yahoo.tensor.TensorType)" ], "fields": [ "public static final com.yahoo.language.process.Embedder throwsOnUse" diff --git a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java index 56c401a7c61..1f4473220d7 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java @@ -22,10 +22,13 @@ public interface Embedder { * * @param text the text to embed * @param language the language of the text, or UNKNOWN to use language independent embedding + * @param destination the name of the recipient of this tensor, either a query feature name + * ("query(feature)"), or a schema and field name concatenated by a dot ("schema.field"). + * This is useful for embedder components that alters behavior depending on the receiver. * @return the text embedded as a list of token ids * @throws IllegalArgumentException if the language is not supported by this embedder */ - List<Integer> embed(String text, Language language); + List<Integer> embed(String text, Language language, String destination); /** * Converts text into tokens in a tensor. @@ -33,21 +36,24 @@ public interface Embedder { * * @param text the text to embed * @param language the language of the text, or UNKNOWN to use language independent embedding + * @param destination the name of the recipient of this tensor, either a query feature name + * ("query(feature)"), or a schema and field name concatenated by a dot ("schema.field"). + * This is useful for embedder components that alters behavior depending on the receiver. * @param tensorType the type of the tensor to be returned * @return the tensor embedding of the text, as the spoecified tensor type * @throws IllegalArgumentException if the language or tensor type is not supported by this embedder */ - Tensor embed(String text, Language language, TensorType tensorType); + Tensor embed(String text, Language language, String destination, TensorType tensorType); class FailingEmbedder implements Embedder { @Override - public List<Integer> embed(String text, Language language) { + public List<Integer> embed(String text, Language language, String destination) { throw new IllegalStateException("No embedder has been configured"); } @Override - public Tensor embed(String text, Language language, TensorType tensorType) { + public Tensor embed(String text, Language language, String destination, TensorType tensorType) { throw new IllegalStateException("No embedder has been configured"); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java index 071fc6e9fc2..b3fad5e9853 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java @@ -17,7 +17,7 @@ import com.yahoo.tensor.TensorType; */ public abstract class Value { - private boolean frozen=false; + private boolean frozen = false; /** Returns the type of this value */ public abstract TensorType type(); |