Encode tensors passed as encode(text)

9 files changed, 104 insertions, 21 deletions

diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
index 34fe376150d..e555000272d 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
@@ -2,14 +2,15 @@
 package com.yahoo.search.query.profile;
 
 import com.yahoo.collections.Pair;
+import com.yahoo.language.process.Encoder;
 import com.yahoo.processing.IllegalInputException;
 import com.yahoo.processing.request.CompoundName;
 import com.yahoo.processing.request.properties.PropertyMap;
 import com.yahoo.protect.Validator;
-import com.yahoo.search.Query;
 import com.yahoo.search.query.Properties;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
 import com.yahoo.search.query.profile.compiled.DimensionalValue;
+import com.yahoo.search.query.profile.types.ConversionContext;
 import com.yahoo.search.query.profile.types.FieldDescription;
 import com.yahoo.search.query.profile.types.QueryProfileFieldType;
 import com.yahoo.search.query.profile.types.QueryProfileType;
@@ -29,6 +30,7 @@ import java.util.Map;
 public class QueryProfileProperties extends Properties {
 
     private final CompiledQueryProfile profile;
+    private final Encoder encoder;
 
     // Note: The priority order is: values has precedence over references
 
@@ -42,10 +44,15 @@ public class QueryProfileProperties extends Properties {
      */
     private List<Pair<CompoundName, CompiledQueryProfile>> references = null;
 
-    /** Creates an instance from a profile, throws an exception if the given profile is null */
     public QueryProfileProperties(CompiledQueryProfile profile) {
+        this(profile, Encoder.throwsOnUse);
+    }
+
+    /** Creates an instance from a profile, throws an exception if the given profile is null */
+    public QueryProfileProperties(CompiledQueryProfile profile, Encoder encoder) {
         Validator.ensureNotNull("The profile wrapped by this cannot be null", profile);
         this.profile = profile;
+        this.encoder = encoder;
     }
 
     /** Returns the query profile backing this, or null if none */
@@ -114,7 +121,9 @@ public class QueryProfileProperties extends Properties {
 
                     if (fieldDescription != null) {
                         if (i == name.size() - 1) { // at the end of the path, check the assignment type
-                            value = fieldDescription.getType().convertFrom(value, profile.getRegistry());
+                            value = fieldDescription.getType().convertFrom(value, new ConversionContext(profile.getRegistry(),
+                                                                                                        encoder,
+                                                                                                        context));
                             if (value == null)
                                 throw new IllegalInputException("'" + value + "' is not a " +
                                                                 fieldDescription.getType().toInstanceDescription());
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java
new file mode 100644
index 00000000000..4aa95741b06
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java
@@ -0,0 +1,40 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query.profile.types;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.process.Encoder;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+
+import java.util.Map;
+
+/**
+ * @author bratseth
+ */
+public class ConversionContext {
+
+    private final CompiledQueryProfileRegistry registry;
+    private final Encoder encoder;
+    private final Language language;
+
+    public ConversionContext(CompiledQueryProfileRegistry registry, Encoder encoder, Map<String, String> context) {
+        this.registry = registry;
+        this.encoder = encoder;
+        this.language = context.containsKey("language") ? Language.fromLanguageTag(context.get("language"))
+                                                        : Language.UNKNOWN;
+    }
+
+    /** Returns the profile registry, or null if none */
+    CompiledQueryProfileRegistry getRegistry() {return registry;}
+
+    /** Returns the configured encoder, never null */
+    Encoder getEncoder() { return encoder; }
+
+    /** Returns the language, which is never null but may be UNKNOWN */
+    Language getLanguage() { return language; }
+
+    /** Returns an empty context */
+    public static ConversionContext empty() {
+        return new ConversionContext(null, Encoder.throwsOnUse, Map.of());
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
index daab5f6a378..7f8836ef2c1 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
@@ -33,7 +33,7 @@ public class FieldDescription implements Comparable<FieldDescription> {
     }
 
     public FieldDescription(String name, String type) {
-        this(name,FieldType.fromString(type,null));
+        this(name,FieldType.fromString(type, null));
     }
 
     public FieldDescription(String name, FieldType type, boolean mandatory) {
@@ -60,7 +60,7 @@ public class FieldDescription implements Comparable<FieldDescription> {
      * @param overridable whether this can be overridden when first set in a profile. Default: true
      */
     public FieldDescription(String name, String typeString, String aliases, boolean mandatory, boolean overridable) {
-        this(name,FieldType.fromString(typeString,null),aliases,mandatory,overridable);
+        this(name,FieldType.fromString(typeString, null), aliases, mandatory, overridable);
     }
 
     public FieldDescription(String name, FieldType type, boolean mandatory, boolean overridable) {
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java
index 3bfd33668e6..511b64c7b6e 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.query.profile.types;
 
+import com.yahoo.language.process.Encoder;
 import com.yahoo.search.query.profile.QueryProfile;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
@@ -41,7 +42,7 @@ public abstract class FieldType {
     public abstract Object convertFrom(Object o, QueryProfileRegistry registry);
 
     /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */
-    public abstract Object convertFrom(Object o, CompiledQueryProfileRegistry registry);
+    public abstract Object convertFrom(Object o, ConversionContext context);
 
     /**
      * Returns this type as a tensor type: The true tensor type is this is a tensor field an an empty type -
@@ -77,7 +78,7 @@ public abstract class FieldType {
         if ("query-profile".equals(typeString))
             return genericQueryProfileType;
         if (typeString.startsWith("query-profile:"))
-            return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()),registry);
+            return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()), registry);
         throw new IllegalArgumentException("Unknown type '" + typeString + "'");
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java
index 1e904e4f970..b1a9820c6fa 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.query.profile.types;
 
+import com.yahoo.language.process.Encoder;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
 
@@ -37,7 +38,7 @@ public class PrimitiveFieldType extends FieldType {
     }
 
     @Override
-    public Object convertFrom(Object object, CompiledQueryProfileRegistry registry) {
+    public Object convertFrom(Object object, ConversionContext context) {
         return convertFrom(object, (QueryProfileRegistry)null);
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java
index 1797a2bd59f..09c1a4d0cc0 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.query.profile.types;
 
+import com.yahoo.language.process.Encoder;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
 import com.yahoo.search.yql.YqlQuery;
@@ -32,7 +33,7 @@ public class QueryFieldType extends FieldType {
     }
 
     @Override
-    public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) {
+    public Object convertFrom(Object o, ConversionContext context) {
         return convertFrom(o, (QueryProfileRegistry)null);
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
index fda2d27e682..6958318bee4 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.query.profile.types;
 
+import com.yahoo.language.process.Encoder;
 import com.yahoo.search.query.profile.QueryProfile;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
@@ -57,11 +58,11 @@ public class QueryProfileFieldType extends FieldType {
     }
 
     @Override
-    public CompiledQueryProfile convertFrom(Object object, CompiledQueryProfileRegistry registry) {
+    public CompiledQueryProfile convertFrom(Object object, ConversionContext context) {
         String profileId = object.toString();
         if (profileId.startsWith("ref:"))
             profileId = profileId.substring("ref:".length());
-        CompiledQueryProfile profile = registry.getComponent(profileId);
+        CompiledQueryProfile profile = context.getRegistry().getComponent(profileId);
         if (profile == null) return null;
         if (type != null && ! type.equals(profile.getType())) return null;
         return profile;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
index 9699a72cb31..34a9f8d41c3 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.query.profile.types;
 
+import com.yahoo.language.Language;
+import com.yahoo.language.process.Encoder;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
 import com.yahoo.tensor.Tensor;
@@ -38,14 +40,26 @@ public class TensorFieldType extends FieldType {
 
     @Override
     public Object convertFrom(Object o, QueryProfileRegistry registry) {
+        return convertFrom(o, ConversionContext.empty());
+    }
+
+    @Override
+    public Object convertFrom(Object o, ConversionContext context) {
+        return convertFrom(o, context.getEncoder(), context.getLanguage());
+    }
+
+    private Object convertFrom(Object o, Encoder encoder, Language language) {
         if (o instanceof Tensor) return o;
+        if (o instanceof String && ((String)o).startsWith("encode(")) return encode((String)o, encoder, language);
         if (o instanceof String) return Tensor.from(type, (String)o);
         return null;
     }
 
-    @Override
-    public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) {
-        return convertFrom(o, (QueryProfileRegistry)null);
+    private Tensor encode(String s, Encoder encoder, Language language) {
+        if ( ! s.endsWith(")"))
+            throw new IllegalArgumentException("Expected any string enclosed in encode(), but the argument does not end by ')'");
+        String text = s.substring("encode(".length(), s.length() - 1);
+        return encoder.encode(text, language, type);
     }
 
     public static TensorFieldType fromTypeString(String s) {
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index 4c65e8003e5..02648f84066 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.query.properties;
 
+import com.yahoo.language.process.Encoder;
 import com.yahoo.processing.IllegalInputException;
 import com.yahoo.processing.request.CompoundName;
 import com.yahoo.search.Query;
@@ -11,6 +12,7 @@ import com.yahoo.search.query.Properties;
 import com.yahoo.search.query.Ranking;
 import com.yahoo.search.query.Select;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.types.ConversionContext;
 import com.yahoo.search.query.profile.types.FieldDescription;
 import com.yahoo.search.query.profile.types.QueryProfileType;
 import com.yahoo.search.query.ranking.Diversity;
@@ -32,10 +34,12 @@ public class QueryProperties extends Properties {
 
     private Query query;
     private final CompiledQueryProfileRegistry profileRegistry;
+    private final Encoder encoder;
 
-    public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry) {
+    public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry, Encoder encoder) {
         this.query = query;
         this.profileRegistry = profileRegistry;
+        this.encoder = encoder;
     }
 
     public void setParentQuery(Query query) {
@@ -256,9 +260,15 @@ public class QueryProperties extends Properties {
                 else if (key.size() > 2) {
                     String restKey = key.rest().rest().toString();
                     if (key.get(1).equals(Ranking.FEATURES))
-                        setRankingFeature(query, restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("features")));
+                        setRankingFeature(query, restKey, toSpecifiedType(restKey,
+                                                                          value,
+                                                                          profileRegistry.getTypeRegistry().getComponent("features"),
+                                                                          context));
                     else if (key.get(1).equals(Ranking.PROPERTIES))
-                        ranking.getProperties().put(restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("properties")));
+                        ranking.getProperties().put(restKey, toSpecifiedType(restKey,
+                                                                             value,
+                                                                             profileRegistry.getTypeRegistry().getComponent("properties"),
+                                                                             context));
                     else
                         throwIllegalParameter(key.rest().toString(), Ranking.RANKING);
                 }
@@ -294,9 +304,15 @@ public class QueryProperties extends Properties {
                 }
             }
             else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated
-                setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features")));
+                setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(),
+                                                                                value,
+                                                                                profileRegistry.getTypeRegistry().getComponent("features"),
+                                                                                context));
             } else if (key.first().equals("rankproperty")) {
-                query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("properties")));
+                query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(),
+                                                                                              value,
+                                                                                              profileRegistry.getTypeRegistry().getComponent("properties"),
+                                                                                              context));
             } else if (key.size()==1) {
                 if (key.equals(Query.HITS))
                     query.setHits(asInteger(value,10));
@@ -359,12 +375,12 @@ public class QueryProperties extends Properties {
         }
     }
 
-    private Object toSpecifiedType(String key, Object value, QueryProfileType type) {
+    private Object toSpecifiedType(String key, Object value, QueryProfileType type, Map<String,String> context) {
         if ( ! ( value instanceof String)) return value; // already typed
         if (type == null) return value; // no type info -> keep as string
         FieldDescription field = type.getField(key);
         if (field == null) return value; // ditto
-        return field.getType().convertFrom(value, profileRegistry);
+        return field.getType().convertFrom(value, new ConversionContext(profileRegistry, encoder, context));
     }
 
     private void throwIllegalParameter(String key,String namespace) {