Wire in property presentation.format.tensors with alias format.tensors

- Move presentation.format.tensors from JsonRenderer to Presentation
- Add aliases recursively
- Allow compound aliases
- Allow a query profile type to describe its own value

10 files changed, 142 insertions, 81 deletions

diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index c54a217d34f..63b3990c711 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -5237,13 +5237,16 @@
       "public java.lang.String getFormat()",
       "public void setFormat(java.lang.String)",
       "public java.lang.Object clone()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
       "public boolean getTiming()",
       "public void setTiming(boolean)",
       "public java.util.Set getSummaryFields()",
+      "public void setSummaryFields(java.lang.String)",
+      "public boolean getTensorShortForm()",
+      "public void setTensorShortForm(java.lang.String)",
+      "public void setTensorShortForm(boolean)",
       "public void prepare()",
-      "public void setSummaryFields(java.lang.String)"
+      "public boolean equals(java.lang.Object)",
+      "public int hashCode()"
     ],
     "fields": [
       "public static final java.lang.String PRESENTATION",
@@ -5251,6 +5254,7 @@
       "public static final java.lang.String TIMING",
       "public static final java.lang.String SUMMARY",
       "public static final java.lang.String SUMMARY_FIELDS",
+      "public static final java.lang.String TENSORS",
       "public static final java.lang.String FORMAT"
     ]
   },
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 77cd4f3b292..1c38bb8f876 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -235,18 +235,18 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
     private static final Map<String, CompoundName> propertyAliases;
     static {
         Map<String,CompoundName> propertyAliasesBuilder = new HashMap<>();
-        addAliases(Query.getArgumentType(), propertyAliasesBuilder);
-        addAliases(Ranking.getArgumentType(), propertyAliasesBuilder);
-        addAliases(Model.getArgumentType(), propertyAliasesBuilder);
-        addAliases(Presentation.getArgumentType(), propertyAliasesBuilder);
-        addAliases(Select.getArgumentType(), propertyAliasesBuilder);
+        addAliases(Query.getArgumentType(), CompoundName.empty, propertyAliasesBuilder);
         propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder);
     }
-    private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) {
-        CompoundName prefix = getPrefix(arguments);
+    private static void addAliases(QueryProfileType arguments, CompoundName prefix, Map<String, CompoundName> aliases) {
         for (FieldDescription field : arguments.fields().values()) {
             for (String alias : field.getAliases())
                 aliases.put(alias, prefix.append(field.getName()));
+            if (field.getType() instanceof QueryProfileFieldType) {
+                var type = ((QueryProfileFieldType) field.getType()).getQueryProfileType();
+                if (type != null)
+                    addAliases(type, prefix.append(type.getComponentIdAsCompoundName()), aliases);
+            }
         }
     }
 
@@ -1117,7 +1117,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
 
     /**
      * Prepares this for binary serialization.
-     * <p>
+     *
      * This must be invoked after all changes have been made to this query before it is passed
      * on to a receiving backend. Calling it is somewhat expensive, so it should only happen once.
      * If a prepared query is cloned, it stays prepared.
diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java
index db6b07e0b0d..31641b5c2f0 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java
@@ -8,7 +8,9 @@ import com.yahoo.prelude.query.Highlight;
 import com.yahoo.prelude.query.IndexedItem;
 import com.yahoo.search.Query;
 import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileFieldType;
 import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.ranking.MatchPhase;
 import com.yahoo.search.rendering.RendererRegistry;
 
 import java.util.ArrayList;
@@ -30,19 +32,26 @@ public class Presentation implements Cloneable {
     public static final String TIMING = "timing";
     public static final String SUMMARY = "summary";
     public static final String SUMMARY_FIELDS = "summaryFields";
+    public static final String TENSORS = "tensors";
 
     /** The (short) name of the parameter holding the name of the return format to use */
     public static final String FORMAT = "format";
 
     static {
-        argumentType=new QueryProfileType(PRESENTATION);
+        argumentType = new QueryProfileType(PRESENTATION);
         argumentType.setStrict(true);
         argumentType.setBuiltin(true);
         argumentType.addField(new FieldDescription(BOLDING, "boolean", "bolding"));
         argumentType.addField(new FieldDescription(TIMING, "boolean", "timing"));
         argumentType.addField(new FieldDescription(SUMMARY, "string", "summary"));
-        argumentType.addField(new FieldDescription(FORMAT, "string", "format template"));
         argumentType.addField(new FieldDescription(SUMMARY_FIELDS, "string", "summaryFields"));
+        QueryProfileType formatArgumentType = new QueryProfileType(FORMAT);
+        formatArgumentType.setBuiltin(true);
+        formatArgumentType.setStrict(true);
+        formatArgumentType.addField(new FieldDescription("", "string", "format template"));
+        formatArgumentType.addField(new FieldDescription(TENSORS, "string", "format.tensors"));
+        formatArgumentType.freeze();
+        argumentType.addField(new FieldDescription(FORMAT, new QueryProfileFieldType(formatArgumentType), "format"));
         argumentType.freeze();
     }
     public static QueryProfileType getArgumentType() { return argumentType; }
@@ -53,7 +62,7 @@ public class Presentation implements Cloneable {
     /** The terms to highlight in the result (only used by BoldingSearcher, may be removed later). */
     private List<IndexedItem> boldingData = null;
 
-    /** Whether or not to do highlighting */
+    /** Whether to do highlighting */
     private boolean bolding = true;
 
     /** The summary class to be shown */
@@ -65,6 +74,9 @@ public class Presentation implements Cloneable {
     /** Whether optional timing data should be rendered */
     private boolean timing = false;
 
+    /** Whether to renders tensors in short form */
+    private boolean tensorShortForm = false;
+
     /** Set of explicitly requested summary fields, instead of summary classes */
     private Set<String> summaryFields = LazySet.newHashSet();
 
@@ -98,14 +110,10 @@ public class Presentation implements Cloneable {
         this.format = (format != null) ? format : RendererRegistry.defaultRendererId.toSpecification();
     }
 
-    /**
-     * Get the name of the format desired for result rendering.
-     */
+    /** Get the name of the format desired for result rendering. */
     public String getFormat() { return format.getName(); }
 
-    /**
-     * Set the desired format for result rendering. If null, use the default renderer.
-     */
+    /** Set the desired format for result rendering. If null, use the default renderer. */
     public void setFormat(String format) {
         setRenderer(ComponentSpecification.fromString(format));
     }
@@ -132,21 +140,7 @@ public class Presentation implements Cloneable {
         }
     }
 
-    @Override
-    public boolean equals(Object o) {
-        if ( ! (o instanceof Presentation)) return false;
-        Presentation p = (Presentation) o;
-        return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary);
-    }
-
-    @Override
-    public int hashCode() {
-        return QueryHelper.combineHash(bolding, summary);
-    }
-
-    /**
-     * @return whether to add optional timing data to the rendered result
-     */
+    /** Returns whether to add optional timing data to the rendered result. */
     public boolean getTiming() {
         return timing;
     }
@@ -166,20 +160,13 @@ public class Presentation implements Cloneable {
         return summaryFields;
     }
 
-    /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */
-    public void prepare() {
-        if (highlight != null)
-            highlight.prepare();
-    }
-
     /**
      * Parse the given string as a comma delimited set of field names and
      * overwrite the set of summary fields. Whitespace will be trimmed. If you
      * want to add or remove fields programmatically, use
      * {@link #getSummaryFields()} and modify the returned set.
      *
-     * @param asString
-     *            the summary fields requested, e.g. "price,author,title"
+     * @param asString the summary fields requested, e.g. "price,author,title"
      */
     public void setSummaryFields(String asString) {
         summaryFields.clear();
@@ -189,5 +176,53 @@ public class Presentation implements Cloneable {
 
     }
 
+    /**
+     * Returns whether tensors should use short form in JSON and textual representations, see
+     * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#tensor">https://docs.vespa.ai/en/reference/document-json-format.html#tensor</a>
+     * and <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form</a>.
+     * Default is false.
+     */
+    public boolean getTensorShortForm() { return tensorShortForm; }
+
+    /**
+     * Sets whether tensors should use short form in JSON and textual representations from a string.
+     *
+     * @param value a string which must be either 'short' or 'long'
+     * @throws IllegalArgumentException if any other value is passed
+     */
+    public void setTensorShortForm(String value) {
+        tensorShortForm = toTensorShortForm(value);
+    }
+
+    private boolean toTensorShortForm(String value) {
+        switch (value) {
+            case "short": return true;
+            case "long": return false;
+            default: throw new IllegalArgumentException("Value must be 'long' or 'short', not '" + value + "'");
+        }
+    }
+
+    public void setTensorShortForm(boolean tensorShortForm) {
+        this.tensorShortForm = tensorShortForm;
+    }
+
+    /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */
+    public void prepare() {
+        if (highlight != null)
+            highlight.prepare();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if ( ! (o instanceof Presentation)) return false;
+        Presentation p = (Presentation) o;
+        return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary);
+    }
+
+    @Override
+    public int hashCode() {
+        return QueryHelper.combineHash(bolding, summary);
+    }
+
 }
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
index 1d6f584bfaa..4d3f6582706 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
@@ -81,7 +81,7 @@ public class FieldDescription implements Comparable<FieldDescription> {
     /**
      * Creates a field description
      *
-     * @param name the name of the field
+     * @param name the name of the field, emnpty means it describes the value held by  the query profile itself
      * @param type the type of the field represented as a string - see {@link com.yahoo.search.query.profile.types.FieldType}
      * @param aliases a list of aliases, never null. Aliases are not following dotted
      *        (meaning they are global, not that they cannot contain dots) and are case insensitive.
@@ -89,8 +89,6 @@ public class FieldDescription implements Comparable<FieldDescription> {
      * @param overridable whether this can be overridden when first set in a profile. Default: true
      */
     public FieldDescription(CompoundName name, FieldType type, List<String> aliases, boolean mandatory, boolean overridable) {
-        if (name.isEmpty())
-            throw new IllegalArgumentException("Illegal name ''");
         for (String nameComponent : name.asList())
             QueryProfile.validateName(nameComponent);
         this.name = name;
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
index 87f39b88981..1b036b96e01 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
@@ -38,7 +38,6 @@ public class PropertyAliases extends Properties {
      */
     protected CompoundName unalias(CompoundName nameOrAlias) {
         if (aliases.isEmpty()) return nameOrAlias;
-        if (nameOrAlias.size() > 1) return nameOrAlias; // aliases are simple names
         CompoundName properName = aliases.get(nameOrAlias.getLowerCasedName());
         return (properName != null) ? properName : nameOrAlias;
     }
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index 9a09c23b23b..243915662d2 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -122,12 +122,16 @@ public class QueryProperties extends Properties {
                 if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString();
             }
         }
-        else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) {
-            if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding();
-            if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary();
-            if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat();
-            if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming();
-            if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields();
+        else if (key.first().equals(Presentation.PRESENTATION)) {
+            if (key.size() == 2) {
+                if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding();
+                if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary();
+                if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat();
+                if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming();
+                if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields();
+            } else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) {
+                if (key.last().equals(Presentation.TENSORS)) return query.getPresentation().getTensorShortForm();
+            }
         }
         else if (key.first().equals("rankfeature") || key.first().equals("featureoverride")) { // featureoverride is deprecated
             return query.getRanking().getFeatures().getObject(key.rest().toString());
@@ -273,17 +277,27 @@ public class QueryProperties extends Properties {
                         throwIllegalParameter(key.rest().toString(), Ranking.RANKING);
                 }
             }
-            else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) {
-                if (key.last().equals(Presentation.BOLDING))
-                    query.getPresentation().setBolding(asBoolean(value, true));
-                else if (key.last().equals(Presentation.SUMMARY))
-                    query.getPresentation().setSummary(asString(value, ""));
-                else if (key.last().equals(Presentation.FORMAT))
-                    query.getPresentation().setFormat(asString(value,""));
-                else if (key.last().equals(Presentation.TIMING))
-                    query.getPresentation().setTiming(asBoolean(value, true));
-                else if (key.last().equals(Presentation.SUMMARY_FIELDS))
-                    query.getPresentation().setSummaryFields(asString(value,""));
+            else if (key.first().equals(Presentation.PRESENTATION)) {
+                if (key.size() == 2) {
+                    if (key.last().equals(Presentation.BOLDING))
+                        query.getPresentation().setBolding(asBoolean(value, true));
+                    else if (key.last().equals(Presentation.SUMMARY))
+                        query.getPresentation().setSummary(asString(value, ""));
+                    else if (key.last().equals(Presentation.FORMAT))
+                        query.getPresentation().setFormat(asString(value, ""));
+                    else if (key.last().equals(Presentation.TIMING))
+                        query.getPresentation().setTiming(asBoolean(value, true));
+                    else if (key.last().equals(Presentation.SUMMARY_FIELDS))
+                        query.getPresentation().setSummaryFields(asString(value, ""));
+                    else
+                        throwIllegalParameter(key.last(), Presentation.PRESENTATION);
+                }
+                else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) {
+                    if (key.last().equals(Presentation.TENSORS))
+                        query.getPresentation().setTensorShortForm(asString(value, ""));
+                    else
+                        throwIllegalParameter(key.last(), Presentation.FORMAT);
+                }
                 else
                     throwIllegalParameter(key.last(), Presentation.PRESENTATION);
             }
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
index f9a91558254..8e8e687076f 100644
--- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -79,7 +79,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
     private static final CompoundName WRAP_ALL_MAPS = new CompoundName("renderer.json.jsonMaps");
     private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug");
     private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback");
-    private static final CompoundName TENSOR_FORMAT = new CompoundName("format.tensors");
 
     // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString
     private static final String BUCKET_LIMITS = "limits";
@@ -173,7 +172,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
         beginJsonCallback(stream);
         debugRendering = getDebugRendering(getResult().getQuery());
         jsonMaps = getWrapAllMaps(getResult().getQuery());
-        tensorShortFormRendering = getTensorShortFormRendering(getResult().getQuery());
+        tensorShortFormRendering = getResult().getQuery() != null
+                                   && getResult().getQuery().getPresentation().getTensorShortForm();
         setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering);
         renderedChildren = new ArrayDeque<>();
         generator.writeStartObject();
@@ -212,12 +212,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
         return q != null && q.properties().getBoolean(DEBUG_RENDERING_KEY, false);
     }
 
-    private boolean getTensorShortFormRendering(Query q) {
-        if (q == null || q.properties().get(TENSOR_FORMAT) == null)
-            return false;
-        return q.properties().getString(TENSOR_FORMAT).equalsIgnoreCase("short");
-    }
-
     protected void renderTrace(Trace trace) throws IOException {
         if (!trace.traceNode().children().iterator().hasNext()) return;
         if (getResult().getQuery().getTraceLevel() == 0) return;
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java
index 60b6c6dbe84..3ed044601f0 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java
@@ -11,7 +11,7 @@ import static org.junit.Assert.assertTrue;
  */
 public class DumpToolTestCase {
 
-    private String profileDir = "src/test/java/com/yahoo/search/query/profile/config/test/multiprofile";
+    private final String profileDir = "src/test/java/com/yahoo/search/query/profile/config/test/multiprofile";
 
     @Test
     public void testNoParameters() {
@@ -25,12 +25,13 @@ public class DumpToolTestCase {
 
     @Test
     public void testNoDimensionValues() {
-        assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir).startsWith("a=general-a\n"));
+        System.out.println(new DumpTool().resolveAndDump("multiprofile1", profileDir));
+        assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir).contains("a=general-a\n"));
     }
 
     @Test
     public void testAllParametersSet() {
-        assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir, "").startsWith("a=general-a\n"));
+        assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir, "").contains("a=general-a\n"));
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java
index 4b8edfdde73..fa7508ea5ac 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java
@@ -47,7 +47,6 @@ public class NameTestCase {
         assertLegalFieldName("a/b");
         assertLegalFieldName("/a/b");
         assertLegalFieldName("/a/b/");
-        assertIllegalFieldName("");
         assertIllegalFieldName("aBc.dooEee.ce_d.-some-other.moreHere",
                                "Could not set 'aBc.dooEee.ce_d.-some-other.moreHere' to 'anyValue'",
                                "Illegal name '-some-other'");
diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
index f3a71af0b9e..95ef4ec3051 100644
--- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
@@ -54,6 +54,7 @@ import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.tensor.serialization.TypedBinaryFormat;
 import com.yahoo.text.Utf8;
+import com.yahoo.yolean.Exceptions;
 import com.yahoo.yolean.trace.TraceNode;
 import org.junit.Before;
 import org.junit.Test;
@@ -69,6 +70,7 @@ import java.util.concurrent.ExecutionException;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * Functional testing of {@link JsonRenderer}.
@@ -165,11 +167,26 @@ public class JsonRendererTestCase {
         h.setField("tensor_mixed", new TensorFieldValue(Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}")));
         h.setField("summaryfeatures", summaryFeatures);
 
-        Result r = new Result(new Query("/?format.tensors=short"));
-        r.hits().add(h);
-        r.setTotalHitCount(1L);
-        String summary = render(r);
-        assertEqualJson(expected, summary);
+        Result result1 = new Result(new Query("/?presentation.format.tensors=short"));
+        result1.hits().add(h);
+        result1.setTotalHitCount(1L);
+        String summary1 = render(result1);
+        assertEqualJson(expected, summary1);
+
+        Result result2 = new Result(new Query("/?format.tensors=short"));
+        result2.hits().add(h);
+        result2.setTotalHitCount(1L);
+        String summary2 = render(result2);
+        assertEqualJson(expected, summary2);
+
+        try {
+            render(new Result(new Query("/?presentation.format.tensors=unknown")));
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("Could not set 'presentation.format.tensors' to 'unknown': Value must be 'long' or 'short', not 'unknown'",
+                         Exceptions.toMessageString(e));
+        }
     }
 
     @Test