summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2021-09-30 13:46:50 +0200
committerJon Bratseth <bratseth@gmail.com>2021-09-30 13:46:50 +0200
commit9fdcf8f92eaf3b47053fa2c131832dea1c792d0c (patch)
treec4d8f2a7c8297fce1b4b6f07a32ab0daeac35aaa
parent1bc2cca4b527bb9a5a8c67744b0796c9fafbe024 (diff)
Pass destination
This allows embedders to switch on it to enable bucket testing and similar.
-rw-r--r--container-search/abi-spec.json3
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java1
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java3
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java18
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java2
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java12
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java2
-rw-r--r--container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java23
-rw-r--r--docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java5
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java12
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java5
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java5
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java5
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java10
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java6
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java8
-rw-r--r--indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java9
-rw-r--r--indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java20
-rw-r--r--linguistics/abi-spec.json8
-rw-r--r--linguistics/src/main/java/com/yahoo/language/process/Embedder.java14
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java2
22 files changed, 128 insertions, 51 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index b7ce40f19a2..e4c1a561dee 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -6259,7 +6259,8 @@
"public"
],
"methods": [
- "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Embedder, java.util.Map)",
+ "public void <init>(java.lang.String, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Embedder, java.util.Map)",
+ "public java.lang.String destination()",
"public static com.yahoo.search.query.profile.types.ConversionContext empty()"
],
"fields": []
diff --git a/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java
index d6bfd6dcd82..e8a2875b9b9 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/ForwardingSearcher.java
@@ -20,7 +20,6 @@ import com.yahoo.search.searchchain.Execution;
* semantics are not necessary for the application.
*
* @see FederationSearcher
- * @since 5.0.13
* @author Steinar Knutsen
*/
@After("*")
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
index 53be827073c..bcf4f568998 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
@@ -121,7 +121,8 @@ public class QueryProfileProperties extends Properties {
if (fieldDescription != null) {
if (i == name.size() - 1) { // at the end of the path, check the assignment type
- value = fieldDescription.getType().convertFrom(value, new ConversionContext(profile.getRegistry(),
+ value = fieldDescription.getType().convertFrom(value, new ConversionContext(localName,
+ profile.getRegistry(),
embedder,
context));
if (value == null)
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java
index e5b9eb1c1cd..8dfb67a9d5f 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java
@@ -12,29 +12,35 @@ import java.util.Map;
*/
public class ConversionContext {
+ private final String destination;
private final CompiledQueryProfileRegistry registry;
private final Embedder embedder;
private final Language language;
- public ConversionContext(CompiledQueryProfileRegistry registry, Embedder embedder, Map<String, String> context) {
+ public ConversionContext(String destination, CompiledQueryProfileRegistry registry, Embedder embedder,
+ Map<String, String> context) {
+ this.destination = destination;
this.registry = registry;
this.embedder = embedder;
this.language = context.containsKey("language") ? Language.fromLanguageTag(context.get("language"))
: Language.UNKNOWN;
}
+ /** Returns the local name of the field which will receive the converted value (or null when this is empty) */
+ public String destination() { return destination; }
+
/** Returns the profile registry, or null if none */
- CompiledQueryProfileRegistry getRegistry() {return registry;}
+ CompiledQueryProfileRegistry registry() {return registry;}
- /** Returns the configured encoder, never null */
- Embedder getEncoder() { return embedder; }
+ /** Returns the configured embedder, never null */
+ Embedder embedder() { return embedder; }
/** Returns the language, which is never null but may be UNKNOWN */
- Language getLanguage() { return language; }
+ Language language() { return language; }
/** Returns an empty context */
public static ConversionContext empty() {
- return new ConversionContext(null, Embedder.throwsOnUse, Map.of());
+ return new ConversionContext(null, null, Embedder.throwsOnUse, Map.of());
}
}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
index ff12224823f..2b649d81ec8 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
@@ -60,7 +60,7 @@ public class QueryProfileFieldType extends FieldType {
String profileId = object.toString();
if (profileId.startsWith("ref:"))
profileId = profileId.substring("ref:".length());
- CompiledQueryProfile profile = context.getRegistry().getComponent(profileId);
+ CompiledQueryProfile profile = context.registry().getComponent(profileId);
if (profile == null) return null;
if (type != null && ! type.equals(profile.getType())) return null;
return profile;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
index cd21f0b3a61..05befb24da0 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
@@ -1,8 +1,6 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.query.profile.types;
-import com.yahoo.language.Language;
-import com.yahoo.language.process.Embedder;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
@@ -44,21 +42,17 @@ public class TensorFieldType extends FieldType {
@Override
public Object convertFrom(Object o, ConversionContext context) {
- return convertFrom(o, context.getEncoder(), context.getLanguage());
- }
-
- private Object convertFrom(Object o, Embedder embedder, Language language) {
if (o instanceof Tensor) return o;
- if (o instanceof String && ((String)o).startsWith("embed(")) return encode((String)o, embedder, language);
+ if (o instanceof String && ((String)o).startsWith("embed(")) return encode((String)o, context);
if (o instanceof String) return Tensor.from(type, (String)o);
return null;
}
- private Tensor encode(String s, Embedder embedder, Language language) {
+ private Tensor encode(String s, ConversionContext context) {
if ( ! s.endsWith(")"))
throw new IllegalArgumentException("Expected any string enclosed in embed(), but the argument does not end by ')'");
String text = s.substring("embed(".length(), s.length() - 1);
- return embedder.embed(text, language, type);
+ return context.embedder().embed(text, context.language(), context.destination(), type);
}
public static TensorFieldType fromTypeString(String s) {
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index 3a426656185..1fa91c73fb0 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -380,7 +380,7 @@ public class QueryProperties extends Properties {
if (type == null) return value; // no type info -> keep as string
FieldDescription field = type.getField(key);
if (field == null) return value; // ditto
- return field.getType().convertFrom(value, new ConversionContext(profileRegistry, embedder, context));
+ return field.getType().convertFrom(value, new ConversionContext(key, profileRegistry, embedder, context));
}
private void throwIllegalParameter(String key,String namespace) {
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
index e22263070e0..e63c7711ff2 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
@@ -447,24 +447,31 @@ public class QueryProfileTypeTestCase {
CompiledQueryProfileRegistry cRegistry = registry.compile();
String textToEmbed = "text to embed into a tensor";
+ String destinationFeature = "query(myTensor4)";
Tensor expectedTensor = Tensor.from("tensor<float>(x[5]):[3,7,4,0,0]]");
- Query query1 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") +
+ Query query1 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features." + destinationFeature) +
"=" + urlEncode("embed(" + textToEmbed + ")"),
com.yahoo.jdisc.http.HttpRequest.Method.GET))
.setQueryProfile(cRegistry.getComponent("test"))
- .setEmbedder(new MockEmbedder(textToEmbed, Language.UNKNOWN, expectedTensor))
+ .setEmbedder(new MockEmbedder(textToEmbed,
+ Language.UNKNOWN,
+ destinationFeature,
+ expectedTensor))
.build();
assertEquals(0, query1.errors().size());
assertEquals(expectedTensor, query1.properties().get("ranking.features.query(myTensor4)"));
assertEquals(expectedTensor, query1.getRanking().getFeatures().getTensor("query(myTensor4)").get());
// Explicit language
- Query query2 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") +
+ Query query2 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features." + destinationFeature) +
"=" + urlEncode("embed(" + textToEmbed + ")") +
"&language=en",
com.yahoo.jdisc.http.HttpRequest.Method.GET))
.setQueryProfile(cRegistry.getComponent("test"))
- .setEmbedder(new MockEmbedder(textToEmbed, Language.ENGLISH, expectedTensor))
+ .setEmbedder(new MockEmbedder(textToEmbed,
+ Language.ENGLISH,
+ destinationFeature,
+ expectedTensor))
.build();
assertEquals(0, query2.errors().size());
assertEquals(expectedTensor, query2.properties().get("ranking.features.query(myTensor4)"));
@@ -726,26 +733,30 @@ public class QueryProfileTypeTestCase {
private final String expectedText;
private final Language expectedLanguage;
+ private final String expectedDestination;
private final Tensor tensorToReturn;
public MockEmbedder(String expectedText,
Language expectedLanguage,
+ String expectedDestination,
Tensor tensorToReturn) {
this.expectedText = expectedText;
this.expectedLanguage = expectedLanguage;
+ this.expectedDestination = expectedDestination;
this.tensorToReturn = tensorToReturn;
}
@Override
- public List<Integer> embed(String text, Language language) {
+ public List<Integer> embed(String text, Language language, String destination) {
fail("Unexpected call");
return null;
}
@Override
- public Tensor embed(String text, Language language, TensorType tensorType) {
+ public Tensor embed(String text, Language language, String destination, TensorType tensorType) {
assertEquals(expectedText, text);
assertEquals(expectedLanguage, language);
+ assertEquals(expectedDestination, destination);
assertEquals(tensorToReturn.type(), tensorType);
return tensorToReturn;
}
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
index 7e1d5b5b6ce..2202d8b99f5 100644
--- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
+++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java
@@ -82,7 +82,8 @@ public class ScriptManager {
parserContext.getAnnotatorConfig().setMaxTokenLength(config.fieldmatchmaxlength());
for (IlscriptsConfig.Ilscript ilscript : config.ilscript()) {
- InputExpression.FieldPathOptimizer fieldPathOptimizer = new InputExpression.FieldPathOptimizer(docTypeMgr.getDocumentType(ilscript.doctype()));
+ DocumentType documentType = docTypeMgr.getDocumentType(ilscript.doctype());
+ InputExpression.FieldPathOptimizer fieldPathOptimizer = new InputExpression.FieldPathOptimizer(documentType);
List<StatementExpression> expressions = new ArrayList<>(ilscript.content().size());
Map<String, DocumentScript> fieldScripts = new HashMap<>(ilscript.content().size());
for (String content : ilscript.content()) {
@@ -95,7 +96,7 @@ public class ScriptManager {
statement.select(fieldPathOptimizer, fieldPathOptimizer);
if ( ! outputFieldNameExtractor.getOutputFieldNames().isEmpty()) {
String outputFieldName = outputFieldNameExtractor.getOutputFieldNames().get(0);
- statement.setStatementOutputType(docTypeMgr.getDocumentType(ilscript.doctype()).getField(outputFieldName).getDataType());
+ statement.setStatementOutput(documentType, documentType.getField(outputFieldName));
}
if (inputFieldNameExtractor.getInputFieldNames().size() == 1) {
String fieldName = inputFieldNameExtractor.getInputFieldNames().get(0);
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java
index aa579ed729e..043a30ce66d 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java
@@ -2,6 +2,8 @@
package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
import com.yahoo.document.TensorDataType;
import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.document.datatypes.TensorFieldValue;
@@ -18,6 +20,9 @@ public class EmbedExpression extends Expression {
private final Embedder embedder;
+ /** The destination the embedding will be written to on the form [schema name].[field name] */
+ private String destination;
+
/** The target type we are embedding into. */
private TensorType targetType;
@@ -27,14 +32,15 @@ public class EmbedExpression extends Expression {
}
@Override
- public void setStatementOutputType(DataType type) {
- targetType = ((TensorDataType)type).getTensorType();
+ public void setStatementOutput(DocumentType documentType, Field field) {
+ targetType = ((TensorDataType)field.getDataType()).getTensorType();
+ destination = documentType.getName() + "." + field.getName();
}
@Override
protected void doExecute(ExecutionContext context) {
StringFieldValue input = (StringFieldValue) context.getValue();
- Tensor tensor = embedder.embed(input.getString(), context.getLanguage(), targetType);
+ Tensor tensor = embedder.embed(input.getString(), context.getLanguage(), destination, targetType);
context.setValue(new TensorFieldValue(tensor));
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
index 20a0c9804a9..3533f159f8c 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java
@@ -3,7 +3,9 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.Document;
+import com.yahoo.document.DocumentType;
import com.yahoo.document.DocumentUpdate;
+import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.language.Linguistics;
import com.yahoo.language.process.Embedder;
@@ -31,7 +33,8 @@ public abstract class Expression extends Selectable {
this.inputType = inputType;
}
- public void setStatementOutputType(DataType type) {}
+ /** Sets the document type and field the statement this expression is part of will write to */
+ public void setStatementOutput(DocumentType documentType, Field field) {}
public final FieldValue execute(FieldValue val) {
return execute(new ExecutionContext().setValue(val));
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
index 0f7c2a411de..c143d921e0e 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java
@@ -2,6 +2,8 @@
package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -25,9 +27,9 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr
}
@Override
- public void setStatementOutputType(DataType type) {
+ public void setStatementOutput(DocumentType documentType, Field field) {
for (Expression expression : expressions)
- expression.setStatementOutputType(type);
+ expression.setStatementOutput(documentType, field);
}
public int size() {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
index 84dbe07d418..d8a9dea83ae 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java
@@ -27,6 +27,11 @@ public final class ForEachExpression extends CompositeExpression {
}
@Override
+ public void setStatementOutput(DocumentType documentType, Field field) {
+ exp.setStatementOutput(documentType, field);
+ }
+
+ @Override
protected void doExecute(final ExecutionContext context) {
FieldValue input = context.getValue();
if (input instanceof Array || input instanceof WeightedSet) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
index 78c261caccb..ad97db1c366 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
import com.yahoo.vespa.indexinglanguage.ExpressionVisitor;
import com.yahoo.vespa.indexinglanguage.UpdateAdapter;
import com.yahoo.vespa.objects.ObjectOperation;
@@ -27,8 +28,8 @@ public final class GuardExpression extends CompositeExpression {
}
@Override
- public void setStatementOutputType(DataType type) {
- exp.setStatementOutputType(type);
+ public void setStatementOutput(DocumentType documentType, Field field) {
+ exp.setStatementOutput(documentType, field);
}
@Override
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
index a6dfa4dd13e..d59c6ce9da1 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java
@@ -2,6 +2,8 @@
package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.NumericFieldValue;
import com.yahoo.vespa.objects.ObjectOperation;
@@ -53,6 +55,14 @@ public final class IfThenExpression extends CompositeExpression {
this.ifFalse = ifFalse;
}
+ @Override
+ public void setStatementOutput(DocumentType documentType, Field field) {
+ lhs.setStatementOutput(documentType, field);
+ rhs.setStatementOutput(documentType, field);
+ ifTrue.setStatementOutput(documentType, field);
+ ifFalse.setStatementOutput(documentType, field);
+ }
+
public Expression getLeftHandSide() {
return lhs;
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
index 8f7e6bc7040..000bde7fcb7 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -23,6 +24,11 @@ public class ParenthesisExpression extends CompositeExpression {
}
@Override
+ public void setStatementOutput(DocumentType documentType, Field field) {
+ innerExp.setStatementOutput(documentType, field);
+ }
+
+ @Override
protected void doExecute(ExecutionContext context) {
innerExp.execute(context);
}
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
index c711b92c7c1..47e44f2c07c 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.collections.Pair;
import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.vespa.objects.ObjectOperation;
import com.yahoo.vespa.objects.ObjectPredicate;
@@ -29,6 +31,12 @@ public final class SelectInputExpression extends CompositeExpression {
}
@Override
+ public void setStatementOutput(DocumentType documentType, Field field) {
+ for (var casePair : cases)
+ casePair.getSecond().setStatementOutput(documentType, field);
+ }
+
+ @Override
protected void doExecute(ExecutionContext context) {
FieldValue input = context.getValue();
for (Pair<String, Expression> entry : cases) {
diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
index 9a8e8719461..49b9218bdb1 100644
--- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
+++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java
@@ -2,6 +2,8 @@
package com.yahoo.vespa.indexinglanguage.expressions;
import com.yahoo.document.DataType;
+import com.yahoo.document.DocumentType;
+import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.StringFieldValue;
import com.yahoo.text.StringUtilities;
@@ -45,6 +47,13 @@ public final class SwitchExpression extends CompositeExpression {
}
@Override
+ public void setStatementOutput(DocumentType documentType, Field field) {
+ defaultExp.setStatementOutput(documentType, field);
+ for (var expression : cases.values())
+ expression.setStatementOutput(documentType, field);
+ }
+
+ @Override
protected void doExecute(ExecutionContext context) {
FieldValue input = context.getValue();
Expression exp = null;
diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java
index 188426b1a06..e0c0a9faba8 100644
--- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java
+++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java
@@ -104,12 +104,14 @@ public class ScriptTestCase {
TensorType tensorType = TensorType.fromSpec("tensor(d[4])");
var expression = Expression.fromString("input myText | embed | attribute 'myTensor'",
new SimpleLinguistics(),
- new MockEncoder());
+ new MockEmbedder("myDocument.myTensor"));
SimpleTestAdapter adapter = new SimpleTestAdapter();
adapter.createField(new Field("myText", DataType.STRING));
- adapter.createField(new Field("myTensor", new TensorDataType(tensorType)));
+ var tensorField = new Field("myTensor", new TensorDataType(tensorType));
+ adapter.createField(tensorField);
adapter.setValue("myText", new StringFieldValue("input text"));
+ expression.setStatementOutput(new DocumentType("myDocument"), tensorField);
// Necessary to resolve output type
VerificationContext verificationContext = new VerificationContext(adapter);
@@ -119,21 +121,27 @@ public class ScriptTestCase {
context.setValue(new StringFieldValue("input text"));
expression.execute(context);
assertNotNull(context);
- //assertTrue(context.getOutputType() instanceof TensorDataType);
assertTrue(adapter.values.containsKey("myTensor"));
assertEquals(Tensor.from(tensorType, "[7,3,0,0]"),
((TensorFieldValue)adapter.values.get("myTensor")).getTensor().get());
}
- private static class MockEncoder implements Embedder {
+ private static class MockEmbedder implements Embedder {
+
+ private final String expectedDestination;
+
+ public MockEmbedder(String expectedDestination) {
+ this.expectedDestination = expectedDestination;
+ }
@Override
- public List<Integer> embed(String text, Language language) {
+ public List<Integer> embed(String text, Language language, String destination) {
return null;
}
@Override
- public Tensor embed(String text, Language language, TensorType tensorType) {
+ public Tensor embed(String text, Language language, String destination, TensorType tensorType) {
+ assertEquals(expectedDestination, destination);
return Tensor.from(tensorType, "[7,3,0,0]");
}
diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json
index d257c451739..5865c28bbb6 100644
--- a/linguistics/abi-spec.json
+++ b/linguistics/abi-spec.json
@@ -338,8 +338,8 @@
],
"methods": [
"public void <init>()",
- "public java.util.List embed(java.lang.String, com.yahoo.language.Language)",
- "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)"
+ "public java.util.List embed(java.lang.String, com.yahoo.language.Language, java.lang.String)",
+ "public com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, java.lang.String, com.yahoo.tensor.TensorType)"
],
"fields": []
},
@@ -352,8 +352,8 @@
"abstract"
],
"methods": [
- "public abstract java.util.List embed(java.lang.String, com.yahoo.language.Language)",
- "public abstract com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, com.yahoo.tensor.TensorType)"
+ "public abstract java.util.List embed(java.lang.String, com.yahoo.language.Language, java.lang.String)",
+ "public abstract com.yahoo.tensor.Tensor embed(java.lang.String, com.yahoo.language.Language, java.lang.String, com.yahoo.tensor.TensorType)"
],
"fields": [
"public static final com.yahoo.language.process.Embedder throwsOnUse"
diff --git a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java
index 56c401a7c61..1f4473220d7 100644
--- a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java
+++ b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java
@@ -22,10 +22,13 @@ public interface Embedder {
*
* @param text the text to embed
* @param language the language of the text, or UNKNOWN to use language independent embedding
+ * @param destination the name of the recipient of this tensor, either a query feature name
+ * ("query(feature)"), or a schema and field name concatenated by a dot ("schema.field").
+ * This is useful for embedder components that alters behavior depending on the receiver.
* @return the text embedded as a list of token ids
* @throws IllegalArgumentException if the language is not supported by this embedder
*/
- List<Integer> embed(String text, Language language);
+ List<Integer> embed(String text, Language language, String destination);
/**
* Converts text into tokens in a tensor.
@@ -33,21 +36,24 @@ public interface Embedder {
*
* @param text the text to embed
* @param language the language of the text, or UNKNOWN to use language independent embedding
+ * @param destination the name of the recipient of this tensor, either a query feature name
+ * ("query(feature)"), or a schema and field name concatenated by a dot ("schema.field").
+ * This is useful for embedder components that alters behavior depending on the receiver.
* @param tensorType the type of the tensor to be returned
* @return the tensor embedding of the text, as the spoecified tensor type
* @throws IllegalArgumentException if the language or tensor type is not supported by this embedder
*/
- Tensor embed(String text, Language language, TensorType tensorType);
+ Tensor embed(String text, Language language, String destination, TensorType tensorType);
class FailingEmbedder implements Embedder {
@Override
- public List<Integer> embed(String text, Language language) {
+ public List<Integer> embed(String text, Language language, String destination) {
throw new IllegalStateException("No embedder has been configured");
}
@Override
- public Tensor embed(String text, Language language, TensorType tensorType) {
+ public Tensor embed(String text, Language language, String destination, TensorType tensorType) {
throw new IllegalStateException("No embedder has been configured");
}
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
index 071fc6e9fc2..b3fad5e9853 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
@@ -17,7 +17,7 @@ import com.yahoo.tensor.TensorType;
*/
public abstract class Value {
- private boolean frozen=false;
+ private boolean frozen = false;
/** Returns the type of this value */
public abstract TensorType type();