diff options
14 files changed, 336 insertions, 43 deletions
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 087be0f17c5..0c4709e4a2c 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -17,6 +17,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.test.MockMetric; import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Encoder; import com.yahoo.language.simple.SimpleLinguistics; import java.io.File; @@ -140,6 +141,7 @@ public class HandlersConfigurerTestWrapper { protected void configure() { // Needed by e.g. SearchHandler bind(Linguistics.class).to(SimpleLinguistics.class).in(Scopes.SINGLETON); + bind(Encoder.class).to(Encoder.FailingEncoder.class).in(Scopes.SINGLETON); bind(ContainerThreadPool.class).to(SimpleContainerThreadpool.class); bind(Metric.class).to(MockMetric.class); } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b577660c1b9..7016eff3185 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1786,6 +1786,27 @@ ], "fields": [] }, + "com.yahoo.search.Query$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public com.yahoo.search.Query$Builder setRequest(java.lang.String)", + "public com.yahoo.search.Query$Builder setRequest(com.yahoo.container.jdisc.HttpRequest)", + "public com.yahoo.container.jdisc.HttpRequest getRequest()", + "public com.yahoo.search.Query$Builder setRequestMap(java.util.Map)", + "public java.util.Map getRequestMap()", + "public com.yahoo.search.Query$Builder setQueryProfile(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", + "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", + "public com.yahoo.search.Query$Builder setEncoder(com.yahoo.language.process.Encoder)", + "public com.yahoo.language.process.Encoder getEncoder()", + "public com.yahoo.search.Query build()" + ], + "fields": [] + }, "com.yahoo.search.Query$Type": { "superClass": "java.lang.Enum", "interfaces": [], @@ -4237,6 +4258,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.language.process.Encoder, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", @@ -5863,6 +5885,7 @@ ], "methods": [ "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", + "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, com.yahoo.language.process.Encoder)", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", @@ -6229,6 +6252,18 @@ ], "fields": [] }, + "com.yahoo.search.query.profile.types.ConversionContext": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Encoder, java.util.Map)", + "public static com.yahoo.search.query.profile.types.ConversionContext empty()" + ], + "fields": [] + }, "com.yahoo.search.query.profile.types.FieldDescription": { "superClass": "java.lang.Object", "interfaces": [ @@ -6276,7 +6311,7 @@ "public abstract java.lang.String toString()", "public abstract java.lang.String toInstanceDescription()", "public abstract java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", - "public abstract java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public abstract java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public com.yahoo.tensor.TensorType asTensorType()", "public static com.yahoo.search.query.profile.types.FieldType fromString(java.lang.String, com.yahoo.search.query.profile.types.QueryProfileTypeRegistry)", "public static boolean isLegalFieldValue(java.lang.Object)" @@ -6303,7 +6338,7 @@ "public java.lang.String stringValue()", "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", - "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", "public int hashCode()", "public boolean equals(java.lang.Object)" @@ -6323,7 +6358,7 @@ "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", - "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)" + "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)" ], "fields": [] }, @@ -6342,11 +6377,11 @@ "public java.lang.String stringValue()", "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", - "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public com.yahoo.search.query.profile.QueryProfile convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", "public int hashCode()", "public boolean equals(java.lang.Object)", - "public bridge synthetic java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public bridge synthetic java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public bridge synthetic java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)" ], "fields": [] @@ -6419,7 +6454,7 @@ "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", - "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public static com.yahoo.search.query.profile.types.TensorFieldType fromTypeString(java.lang.String)" ], "fields": [] @@ -6496,7 +6531,7 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.Query, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public void <init>(com.yahoo.search.Query, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Encoder)", "public void setParentQuery(com.yahoo.search.Query)", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 8c3a30a5a4d..06b71599103 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -7,6 +7,7 @@ import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; +import com.yahoo.language.process.Encoder; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; @@ -333,20 +334,32 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile) { super(new QueryPropertyAliases(propertyAliases)); this.httpRequest = request; - init(requestMap, queryProfile); + init(requestMap, queryProfile, Encoder.throwsOnUse); } - private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile) { + // TODO: Deprecate most constructors above here + + private Query(Builder builder) { + this(builder.getRequest(), builder.getRequestMap(), builder.getQueryProfile(), builder.getEncoder()); + } + + private Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile, Encoder encoder) { + super(new QueryPropertyAliases(propertyAliases)); + this.httpRequest = request; + init(requestMap, queryProfile, encoder); + } + + private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile, Encoder encoder) { startTime = httpRequest.getJDiscRequest().creationTime(TimeUnit.MILLISECONDS); if (queryProfile != null) { // Move all request parameters to the query profile just to validate that the parameter settings are legal - Properties queryProfileProperties = new QueryProfileProperties(queryProfile); + Properties queryProfileProperties = new QueryProfileProperties(queryProfile, encoder); properties().chain(queryProfileProperties); // TODO: Just checking legality rather than actually setting would be faster setPropertiesFromRequestMap(requestMap, properties(), true); // Adds errors to the query for illegal set attempts // Create the full chain - properties().chain(new QueryProperties(this, queryProfile.getRegistry())). + properties().chain(new QueryProperties(this, queryProfile.getRegistry(), encoder)). chain(new ModelObjectMap()). chain(new RequestContextProperties(requestMap)). chain(queryProfileProperties). @@ -365,7 +378,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } else { // bypass these complications if there is no query profile to get values from and validate against properties(). - chain(new QueryProperties(this, CompiledQueryProfileRegistry.empty)). + chain(new QueryProperties(this, CompiledQueryProfileRegistry.empty, encoder)). chain(new PropertyMap()). chain(new DefaultProperties()); setPropertiesFromRequestMap(requestMap, properties(), false); @@ -1112,4 +1125,59 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { getRanking().prepare(); } + public static class Builder { + + private HttpRequest request = null; + private Map<String, String> requestMap = null; + private CompiledQueryProfile queryProfile = null; + private Encoder encoder = Encoder.throwsOnUse; + + public Builder setRequest(String query) { + request = HttpRequest.createTestRequest(query, com.yahoo.jdisc.http.HttpRequest.Method.GET); + return this; + } + + public Builder setRequest(HttpRequest request) { + this.request = request; + return this; + } + + public HttpRequest getRequest() { + if (request == null) + return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.GET); + return request; + } + + /** Sets the request mao to use explicitly. If not set, the request map will be getRequest().propertyMap() */ + public Builder setRequestMap(Map<String, String> requestMap) { + this.requestMap = requestMap; + return this; + } + + public Map<String, String> getRequestMap() { + if (requestMap == null) + return getRequest().propertyMap(); + return requestMap; + } + + public Builder setQueryProfile(CompiledQueryProfile queryProfile) { + this.queryProfile = queryProfile; + return this; + } + + /** Returns the query profile of this query, or null if none. */ + public CompiledQueryProfile getQueryProfile() { return queryProfile; } + + public Builder setEncoder(Encoder encoder) { + this.encoder = encoder; + return this; + } + + public Encoder getEncoder() { return encoder; } + + /** Creates a new query from this builder. No properties are required to before calling this. */ + public Query build() { return new Query(this); } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 9f67603f62b..d1e57a30206 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -23,6 +23,7 @@ import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Encoder; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; import com.yahoo.prelude.query.parser.ParseException; @@ -105,6 +106,8 @@ public class SearchHandler extends LoggingRequestHandler { private final String selfHostname = HostName.getLocalhost(); + private final Encoder encoder; + private final ExecutionFactory executionFactory; private final AtomicLong numRequestsLeftToTrace; @@ -129,6 +132,22 @@ public class SearchHandler extends LoggingRequestHandler { public SearchHandler(Statistics statistics, Metric metric, ContainerThreadPool threadpool, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig config, + Encoder encoder, + ExecutionFactory executionFactory) { + this(statistics, metric, threadpool.executor(), queryProfileRegistry, encoder, executionFactory, + config.numQueriesToTraceOnDebugAfterConstruction(), + config.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of(config.hostResponseHeaderKey())); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, + ContainerThreadPool threadpool, AccessLog ignored, CompiledQueryProfileRegistry queryProfileRegistry, ContainerHttpConfig config, @@ -136,6 +155,10 @@ public class SearchHandler extends LoggingRequestHandler { this(statistics, metric, threadpool.executor(), ignored, queryProfileRegistry, config, executionFactory); } + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 public SearchHandler(Statistics statistics, Metric metric, Executor executor, @@ -147,6 +170,7 @@ public class SearchHandler extends LoggingRequestHandler { metric, executor, queryProfileRegistry, + Encoder.throwsOnUse, executionFactory, containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? @@ -168,12 +192,17 @@ public class SearchHandler extends LoggingRequestHandler { metric, executor, QueryProfileConfigurer.createFromConfig(queryProfileConfig).compile(), + Encoder.throwsOnUse, executionFactory, containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); } + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 public SearchHandler(Statistics statistics, Metric metric, Executor executor, @@ -181,19 +210,22 @@ public class SearchHandler extends LoggingRequestHandler { CompiledQueryProfileRegistry queryProfileRegistry, ExecutionFactory executionFactory, Optional<String> hostResponseHeaderKey) { - this(statistics, metric, executor, queryProfileRegistry, executionFactory, 0, hostResponseHeaderKey); + this(statistics, metric, executor, queryProfileRegistry, Encoder.throwsOnUse, + executionFactory, 0, hostResponseHeaderKey); } private SearchHandler(Statistics statistics, Metric metric, Executor executor, CompiledQueryProfileRegistry queryProfileRegistry, + Encoder encoder, ExecutionFactory executionFactory, long numQueriesToTraceOnDebugAfterStartup, Optional<String> hostResponseHeaderKey) { super(executor, metric, true); log.log(Level.FINE, () -> "SearchHandler.init " + System.identityHashCode(this)); this.queryProfileRegistry = queryProfileRegistry; + this.encoder = encoder; this.executionFactory = executionFactory; this.maxThreads = examineExecutor(executor); @@ -297,7 +329,11 @@ public class SearchHandler extends LoggingRequestHandler { String queryProfileName = requestMap.getOrDefault("queryProfile", null); CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName); - Query query = new Query(request, requestMap, queryProfile); + Query query = new Query.Builder().setRequest(request) + .setRequestMap(requestMap) + .setQueryProfile(queryProfile) + .setEncoder(encoder) + .build(); boolean benchmarking = VespaHeaders.benchmarkOutput(request); boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarking, request.getJDiscRequest().headers()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 34fe376150d..e555000272d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -2,14 +2,15 @@ package com.yahoo.search.query.profile; import com.yahoo.collections.Pair; +import com.yahoo.language.process.Encoder; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.processing.request.properties.PropertyMap; import com.yahoo.protect.Validator; -import com.yahoo.search.Query; import com.yahoo.search.query.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.DimensionalValue; +import com.yahoo.search.query.profile.types.ConversionContext; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -29,6 +30,7 @@ import java.util.Map; public class QueryProfileProperties extends Properties { private final CompiledQueryProfile profile; + private final Encoder encoder; // Note: The priority order is: values has precedence over references @@ -42,10 +44,15 @@ public class QueryProfileProperties extends Properties { */ private List<Pair<CompoundName, CompiledQueryProfile>> references = null; - /** Creates an instance from a profile, throws an exception if the given profile is null */ public QueryProfileProperties(CompiledQueryProfile profile) { + this(profile, Encoder.throwsOnUse); + } + + /** Creates an instance from a profile, throws an exception if the given profile is null */ + public QueryProfileProperties(CompiledQueryProfile profile, Encoder encoder) { Validator.ensureNotNull("The profile wrapped by this cannot be null", profile); this.profile = profile; + this.encoder = encoder; } /** Returns the query profile backing this, or null if none */ @@ -114,7 +121,9 @@ public class QueryProfileProperties extends Properties { if (fieldDescription != null) { if (i == name.size() - 1) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); + value = fieldDescription.getType().convertFrom(value, new ConversionContext(profile.getRegistry(), + encoder, + context)); if (value == null) throw new IllegalInputException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java new file mode 100644 index 00000000000..4aa95741b06 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.language.Language; +import com.yahoo.language.process.Encoder; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; + +import java.util.Map; + +/** + * @author bratseth + */ +public class ConversionContext { + + private final CompiledQueryProfileRegistry registry; + private final Encoder encoder; + private final Language language; + + public ConversionContext(CompiledQueryProfileRegistry registry, Encoder encoder, Map<String, String> context) { + this.registry = registry; + this.encoder = encoder; + this.language = context.containsKey("language") ? Language.fromLanguageTag(context.get("language")) + : Language.UNKNOWN; + } + + /** Returns the profile registry, or null if none */ + CompiledQueryProfileRegistry getRegistry() {return registry;} + + /** Returns the configured encoder, never null */ + Encoder getEncoder() { return encoder; } + + /** Returns the language, which is never null but may be UNKNOWN */ + Language getLanguage() { return language; } + + /** Returns an empty context */ + public static ConversionContext empty() { + return new ConversionContext(null, Encoder.throwsOnUse, Map.of()); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index daab5f6a378..7f8836ef2c1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -33,7 +33,7 @@ public class FieldDescription implements Comparable<FieldDescription> { } public FieldDescription(String name, String type) { - this(name,FieldType.fromString(type,null)); + this(name,FieldType.fromString(type, null)); } public FieldDescription(String name, FieldType type, boolean mandatory) { @@ -60,7 +60,7 @@ public class FieldDescription implements Comparable<FieldDescription> { * @param overridable whether this can be overridden when first set in a profile. Default: true */ public FieldDescription(String name, String typeString, String aliases, boolean mandatory, boolean overridable) { - this(name,FieldType.fromString(typeString,null),aliases,mandatory,overridable); + this(name,FieldType.fromString(typeString, null), aliases, mandatory, overridable); } public FieldDescription(String name, FieldType type, boolean mandatory, boolean overridable) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java index 3bfd33668e6..511b64c7b6e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; @@ -41,7 +42,7 @@ public abstract class FieldType { public abstract Object convertFrom(Object o, QueryProfileRegistry registry); /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */ - public abstract Object convertFrom(Object o, CompiledQueryProfileRegistry registry); + public abstract Object convertFrom(Object o, ConversionContext context); /** * Returns this type as a tensor type: The true tensor type is this is a tensor field an an empty type - @@ -77,7 +78,7 @@ public abstract class FieldType { if ("query-profile".equals(typeString)) return genericQueryProfileType; if (typeString.startsWith("query-profile:")) - return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()),registry); + return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()), registry); throw new IllegalArgumentException("Unknown type '" + typeString + "'"); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java index 1e904e4f970..b1a9820c6fa 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; @@ -37,7 +38,7 @@ public class PrimitiveFieldType extends FieldType { } @Override - public Object convertFrom(Object object, CompiledQueryProfileRegistry registry) { + public Object convertFrom(Object object, ConversionContext context) { return convertFrom(object, (QueryProfileRegistry)null); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java index 1797a2bd59f..09c1a4d0cc0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.yql.YqlQuery; @@ -32,7 +33,7 @@ public class QueryFieldType extends FieldType { } @Override - public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { + public Object convertFrom(Object o, ConversionContext context) { return convertFrom(o, (QueryProfileRegistry)null); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java index fda2d27e682..6958318bee4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; @@ -57,11 +58,11 @@ public class QueryProfileFieldType extends FieldType { } @Override - public CompiledQueryProfile convertFrom(Object object, CompiledQueryProfileRegistry registry) { + public CompiledQueryProfile convertFrom(Object object, ConversionContext context) { String profileId = object.toString(); if (profileId.startsWith("ref:")) profileId = profileId.substring("ref:".length()); - CompiledQueryProfile profile = registry.getComponent(profileId); + CompiledQueryProfile profile = context.getRegistry().getComponent(profileId); if (profile == null) return null; if (type != null && ! type.equals(profile.getType())) return null; return profile; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index 9699a72cb31..34a9f8d41c3 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.Language; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.tensor.Tensor; @@ -38,14 +40,26 @@ public class TensorFieldType extends FieldType { @Override public Object convertFrom(Object o, QueryProfileRegistry registry) { + return convertFrom(o, ConversionContext.empty()); + } + + @Override + public Object convertFrom(Object o, ConversionContext context) { + return convertFrom(o, context.getEncoder(), context.getLanguage()); + } + + private Object convertFrom(Object o, Encoder encoder, Language language) { if (o instanceof Tensor) return o; + if (o instanceof String && ((String)o).startsWith("encode(")) return encode((String)o, encoder, language); if (o instanceof String) return Tensor.from(type, (String)o); return null; } - @Override - public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { - return convertFrom(o, (QueryProfileRegistry)null); + private Tensor encode(String s, Encoder encoder, Language language) { + if ( ! s.endsWith(")")) + throw new IllegalArgumentException("Expected any string enclosed in encode(), but the argument does not end by ')'"); + String text = s.substring("encode(".length(), s.length() - 1); + return encoder.encode(text, language, type); } public static TensorFieldType fromTypeString(String s) { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 4c65e8003e5..02648f84066 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.properties; +import com.yahoo.language.process.Encoder; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -11,6 +12,7 @@ import com.yahoo.search.query.Properties; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.Select; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.ConversionContext; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.ranking.Diversity; @@ -32,10 +34,12 @@ public class QueryProperties extends Properties { private Query query; private final CompiledQueryProfileRegistry profileRegistry; + private final Encoder encoder; - public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry) { + public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry, Encoder encoder) { this.query = query; this.profileRegistry = profileRegistry; + this.encoder = encoder; } public void setParentQuery(Query query) { @@ -256,9 +260,15 @@ public class QueryProperties extends Properties { else if (key.size() > 2) { String restKey = key.rest().rest().toString(); if (key.get(1).equals(Ranking.FEATURES)) - setRankingFeature(query, restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("features"))); + setRankingFeature(query, restKey, toSpecifiedType(restKey, + value, + profileRegistry.getTypeRegistry().getComponent("features"), + context)); else if (key.get(1).equals(Ranking.PROPERTIES)) - ranking.getProperties().put(restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("properties"))); + ranking.getProperties().put(restKey, toSpecifiedType(restKey, + value, + profileRegistry.getTypeRegistry().getComponent("properties"), + context)); else throwIllegalParameter(key.rest().toString(), Ranking.RANKING); } @@ -294,9 +304,15 @@ public class QueryProperties extends Properties { } } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated - setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features"))); + setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), + value, + profileRegistry.getTypeRegistry().getComponent("features"), + context)); } else if (key.first().equals("rankproperty")) { - query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("properties"))); + query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), + value, + profileRegistry.getTypeRegistry().getComponent("properties"), + context)); } else if (key.size()==1) { if (key.equals(Query.HITS)) query.setHits(asInteger(value,10)); @@ -359,12 +375,12 @@ public class QueryProperties extends Properties { } } - private Object toSpecifiedType(String key, Object value, QueryProfileType type) { + private Object toSpecifiedType(String key, Object value, QueryProfileType type, Map<String,String> context) { if ( ! ( value instanceof String)) return value; // already typed if (type == null) return value; // no type info -> keep as string FieldDescription field = type.getField(key); if (field == null) return value; // ditto - return field.getType().convertFrom(value, profileRegistry); + return field.getType().convertFrom(value, new ConversionContext(profileRegistry, encoder, context)); } private void throwIllegalParameter(String key,String namespace) { diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index 39ba607b741..45f53a1cdb9 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -3,7 +3,10 @@ package com.yahoo.search.query.profile.types.test; import com.yahoo.component.ComponentId; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.language.Language; +import com.yahoo.language.process.Encoder; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.processing.request.CompoundName; @@ -21,6 +24,8 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.List; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; @@ -80,6 +85,7 @@ public class QueryProfileTypeTestCase { type.addField(new FieldDescription("ranking.features.query(myTensor1)", FieldType.fromString("tensor(a{},b{})", registry)), registry); type.addField(new FieldDescription("ranking.features.query(myTensor2)", FieldType.fromString("tensor(x[2],y[2])", registry)), registry); type.addField(new FieldDescription("ranking.features.query(myTensor3)", FieldType.fromString("tensor<float>(x{})",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor4)", FieldType.fromString("tensor<float>(x[5])",registry)), registry); type.addField(new FieldDescription("myQuery", FieldType.fromString("query", registry)), registry); type.addField(new FieldDescription("myQueryProfile", FieldType.fromString("query-profile", registry),"qp"), registry); } @@ -400,15 +406,15 @@ public class QueryProfileTypeTestCase { } @Test - public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException { + public void testTensorRankFeatureInRequest() { QueryProfile profile = new QueryProfile("test"); profile.setType(testtype); registry.register(profile); CompiledQueryProfileRegistry cRegistry = registry.compile(); String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; - Query query = new Query(HttpRequest.createTestRequest("?" + encode("ranking.features.query(myTensor1)") + - "=" + encode(tensorString), + Query query = new Query(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor1)") + + "=" + urlEncode(tensorString), com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); assertEquals(0, query.errors().size()); @@ -418,15 +424,15 @@ public class QueryProfileTypeTestCase { // Expected to work exactly as testTensorRankFeatureInRequest @Test - public void testTensorRankFeatureInRequestWithInheritedQueryProfileType() throws UnsupportedEncodingException { + public void testTensorRankFeatureInRequestWithInheritedQueryProfileType() { QueryProfile profile = new QueryProfile("test"); profile.setType(emptyInheritingTesttype); registry.register(profile); CompiledQueryProfileRegistry cRegistry = registry.compile(); String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; - Query query = new Query(HttpRequest.createTestRequest("?" + encode("ranking.features.query(myTensor1)") + - "=" + encode(tensorString), + Query query = new Query(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor1)") + + "=" + urlEncode(tensorString), com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); assertEquals(0, query.errors().size()); @@ -434,8 +440,41 @@ public class QueryProfileTypeTestCase { assertEquals(Tensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); } - private String encode(String s) throws UnsupportedEncodingException { - return URLEncoder.encode(s, "utf8"); + @Test + public void testUnencodedTensorRankFeatureInRequest() { + QueryProfile profile = new QueryProfile("test"); + profile.setType(testtype); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + String textToEncode = "text to encode as tensor"; + Tensor expectedTensor = Tensor.from("tensor<float>(x[5]):[3,7,4,0,0]]"); + Query query1 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") + + "=" + urlEncode("encode(" + textToEncode + ")"), + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .setQueryProfile(cRegistry.getComponent("test")) + .setEncoder(new MockEncoder(textToEncode, Language.UNKNOWN, expectedTensor)) + .build(); + assertEquals(0, query1.errors().size()); + assertEquals(expectedTensor, query1.properties().get("ranking.features.query(myTensor4)")); + assertEquals(expectedTensor, query1.getRanking().getFeatures().getTensor("query(myTensor4)").get()); + + // Explicit language + Query query2 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") + + "=" + urlEncode("encode(" + textToEncode + ")") + + "&language=en", + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .setQueryProfile(cRegistry.getComponent("test")) + .setEncoder(new MockEncoder(textToEncode, Language.ENGLISH, expectedTensor)) + .build(); + assertEquals(0, query2.errors().size()); + assertEquals(expectedTensor, query2.properties().get("ranking.features.query(myTensor4)")); + assertEquals(expectedTensor, query2.getRanking().getFeatures().getTensor("query(myTensor4)").get()); + + } + + private String urlEncode(String s) { + return URLEncoder.encode(s, StandardCharsets.UTF_8); } @Test @@ -684,4 +723,34 @@ public class QueryProfileTypeTestCase { } } + private static final class MockEncoder implements Encoder { + + private final String expectedText; + private final Language expectedLanguage; + private final Tensor tensorToReturn; + + public MockEncoder(String expectedText, + Language expectedLanguage, + Tensor tensorToReturn) { + this.expectedText = expectedText; + this.expectedLanguage = expectedLanguage; + this.tensorToReturn = tensorToReturn; + } + + @Override + public List<Integer> encode(String text, Language language) { + fail("Unexpected call"); + return null; + } + + @Override + public Tensor encode(String text, Language language, TensorType tensorType) { + assertEquals(expectedText, text); + assertEquals(expectedLanguage, language); + assertEquals(tensorToReturn.type(), tensorType); + return tensorToReturn; + } + + } + } |