Parse input default values

3 files changed, 148 insertions, 52 deletions

diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj
index 9b3383f9f41..3fe2b6d3cdd 100644
--- a/config-model/src/main/javacc/IntermediateParser.jj
+++ b/config-model/src/main/javacc/IntermediateParser.jj
@@ -1,9 +1,5 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-// Duplicate Schema parser - work in progress
-// @author arnej27959
-// NOTE: When grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
-
 options {
     UNICODE_INPUT = true;
     CACHE_TOKENS  = false;
@@ -44,6 +40,7 @@ import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
 import com.yahoo.searchlib.rankingexpression.evaluation.Value;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.TensorAddress;
 
 import java.util.Optional;
 import java.util.Map;
@@ -55,6 +52,8 @@ import java.util.logging.Level;
 /**
  * The schema parser
  *
+ * NOTE: When this grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf
+ *
  * @author bratseth
  */
 public class IntermediateParser {
@@ -62,9 +61,7 @@ public class IntermediateParser {
     private DeployLogger deployLogger;
     private ModelContext.Properties properties;
 
-    /**
-     * Creates a parser
-     */
+    /** Creates a parser. */
     public IntermediateParser(SimpleCharStream stream,
                               DeployLogger deployLogger,
                               ModelContext.Properties properties)
@@ -341,7 +338,8 @@ TOKEN :
 | < URI: "uri" >
 | < IDENTIFIER:           ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* >
 | < IDENTIFIER_WITH_DASH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* >
-| < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" >
+| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" >
+| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" >
 | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* >
 | < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ >
 | < INTEGER: ("-")? (["0"-"9"])+ >
@@ -353,8 +351,8 @@ TOKEN :
 | < LESSTHAN: "<" >
 | < GREATERTHAN: ">" >
 | < VARIABLE: "$" <IDENTIFIER> >
-| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
-| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<QUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
+| < ONNX_INPUT_SL: "input" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
+| < ONNX_OUTPUT_SL: "output" (" ")* (<IDENTIFIER>|<DOUBLEQUOTEDSTRING>) (" ")* ":" (" ")* (~["\n"])* ("\n")? >
 }
 
 // Declare a special skip token for comments.
@@ -2068,10 +2066,25 @@ void inputs(ParsedRankProfile profile) :
 }
 {
     <INPUTS> <LBRACE> (<NL>)*
-      ( reference = queryFeature() type = inputType(reference) { profile.addInput(reference, type); } (<NL>)*) *
+      ( input(profile) (<NL>)*) *
     <RBRACE>
 }
 
+void input(ParsedRankProfile profile) :
+{
+    Reference reference;
+    TensorType type;
+    Tensor defaultValue = null;
+}
+{
+    reference = queryFeature() type = inputType(reference) ( <COLON> (<NL>)* defaultValue = tensorValue(type) )?
+    {
+        profile.addInput(reference, type);
+        if (defaultValue != null)
+            new TensorValue(defaultValue);
+    }
+}
+
 TensorType inputType(Reference reference) :
 {
     TensorType type;
@@ -2372,7 +2385,6 @@ void rankDegradationBinLow() :
     { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); }
 }
 
-
 /**
  * This rule consumes part of a rank-degradation statement of a rank profile.
  */
@@ -2439,7 +2451,7 @@ void constantTensor(ParsedRankProfile profile, String name) :
 }
 {
     <LBRACE> (<NL>)*
-      (( tensorString = tensorValue() |
+      (( tensorString = tensorValuePrefixedByValue() |
          tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE>
     {
         if (tensorType != null) {
@@ -2455,7 +2467,93 @@ String constantTensorErrorMessage(String rankProfileName, String constantTensorN
     { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; }
 }
 
-String tensorValue() :
+/**
+ * Parses a tensor written in a tensor literal form,
+ * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form
+ */
+Tensor tensorValue(TensorType type) :
+{
+    Tensor.Builder builder = Tensor.Builder.of(type);
+}
+{
+    ( mappedTensorValue(builder) | indexedTensorValue(builder) )
+    { return builder.build(); }
+}
+
+/** A mapped or mixed tensor value. */
+void mappedTensorValue(Tensor.Builder builder) : {}
+{
+    "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}"
+}
+
+void indexedTensorValue(Tensor.Builder builder) : {}
+{
+    "[" ( indexedTensorCell(builder) )* ( <COMMA> (<NL>)* indexedTensorCell(builder) )* "]"
+}
+
+void mappedTensorBlock(Tensor.Builder builder) :
+{
+    TensorAddress address;
+    double value;
+}
+{
+    address = tensorAddress(builder.type()) <COLON> (<NL>)* value = tensorCellValue()
+    { builder.cell(address, value); }
+}
+
+void indexedTensorCell(Tensor.Builder builder) :
+{
+    double value;
+}
+{
+    value = tensorCellValue()
+    { builder.cell(value); }
+}
+
+TensorAddress tensorAddress(TensorType type) :
+{
+    TensorAddress.Builder builder = new TensorAddress.Builder(type);
+    String label;
+}
+{
+    (
+        label = tensorAddressLabel() { builder.add(label); }
+        |
+        ( "{" ( tensorAddressElement(builder) )* ( <COMMA> tensorAddressElement(builder) )* "}" )
+    )
+    { return builder.build(); }
+}
+
+void tensorAddressElement(TensorAddress.Builder builder) :
+{
+    String dimension;
+    String label;
+}
+{
+    dimension = identifier() <COLON> (<NL>)* label = tensorAddressLabel()
+    { builder.add(dimension, label); }
+}
+
+String tensorAddressLabel() :
+{
+    String label;
+}
+{
+    ( label = identifier() | label = quotedString() )
+    { return label; }
+}
+
+double tensorCellValue() :
+{
+    Number value;
+}
+{
+    value = consumeNumber()
+    { return value.doubleValue(); }
+}
+
+/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */
+String tensorValuePrefixedByValue() :
 {
     String tensor;
 }
@@ -2701,39 +2799,28 @@ String string() : { }
  * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can
  * contain anything but a double quote.
  *
- * @return The unquoted token image.
+ * @return the unquoted token image
  */
 String quotedString() : { }
 {
-    <QUOTEDSTRING> { return token.image.substring(1, token.image.length() - 1); }
+    ( <DOUBLEQUOTEDSTRING> | <SINGLEQUOTEDSTRING> )
+    { return token.image.substring(1, token.image.length() - 1); }
 }
 
-/**
- * This rule consumes a boolean value.
- *
- * @return The consumed boolean value.
- */
+/** A boolean value. */
 Boolean bool() : { }
 {
     ( ( <ON>  | <TRUE> )  { return true; } |
       ( <OFF> | <FALSE> ) { return false; } )
 }
 
-/**
- * This rule consumes an integer token and returns its numeric value.
- *
- * @return The consumed integer value.
- */
+/** Consumes an integer token and returns its numeric value. */
 int integer() : { }
 {
     <INTEGER> { return Integer.parseInt(token.image); }
 }
 
-/**
- * This rule consumes a long or integer token and returns its numeric value.
- *
- * @return The consumed long value.
- */
+/** Consumes a long or integer token and returns its numeric value. */
 long consumeLong() : { }
 {
     ( <INTEGER> { return Long.parseLong(token.image); } |
@@ -2741,11 +2828,7 @@ long consumeLong() : { }
     )
 }
 
-/**
- * This rule consumes a floating-point token and returns its numeric value.
- *
- * @return The consumed value.
- */
+/** Consumes a floating-point token and returns its numeric value. */
 double consumeFloat() : { }
 {
     <DOUBLE> { return Double.valueOf(token.image); }
@@ -2759,9 +2842,7 @@ Number consumeNumber() :
     (num = consumeFloat() | num = consumeLong()) { return num; }
 }
 
-/**
- * This rule consumes an opening brace with leading and trailing newline tokens.
- */
+/** Consumes an opening brace with leading and trailing newline tokens. */
 void lbrace() : { }
 {
     (<NL>)* <LBRACE> (<NL>)*
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 66513a8205a..ba304f6f9ca 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -2197,15 +2197,23 @@ void secondPhaseItem(RankProfile profile) :
 }
 
 /** Consumes an inputs block of a rank profile. */
-void inputs(RankProfile profile) :
+void inputs(RankProfile profile) : {}
+{
+    <INPUTS> <LBRACE> (<NL>)*
+      ( input(profile) (<NL>)*) *
+    <RBRACE>
+}
+
+void input(RankProfile profile) :
 {
     Reference reference;
     TensorType type;
 }
 {
-    <INPUTS> <LBRACE> (<NL>)*
-      ( reference = queryFeature() type = inputType(reference) { profile.addInput(reference, type); } (<NL>)*) *
-    <RBRACE>
+    reference = queryFeature() type = inputType(reference)
+    {
+        profile.addInput(reference, type);
+    }
 }
 
 TensorType inputType(Reference reference) :
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
index a34be715e23..fb2e736a79f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java
@@ -21,6 +21,9 @@ public class SchemaInfoTestCase {
                 "      query(bar) tensor(key{},x[1000])" +
                 "      query(myDouble1) double" +
                 "      query(myDouble2) tensor()" +
+                "      query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" +
+                "      query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" +
+ //               "      query(myMixed) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" +
                 "    }" +
                 "  }";
         List<String> schemas = List.of("type1", "type2");
@@ -48,15 +51,14 @@ public class SchemaInfoTestCase {
                 tester.assertRankProfile(schema, 5, "rankfeatures", false, true);
                 var inputs = tester.assertRankProfile(schema, 6, "inputs", false, false);
 
-                assertEquals(4, inputs.input().size());
-                assertEquals("query(foo)", inputs.input(0).name());
-                assertEquals("tensor<float>(x[10])", inputs.input(0).type());
-                assertEquals("query(bar)", inputs.input(1).name());
-                assertEquals("tensor(key{},x[1000])", inputs.input(1).type());
-                assertEquals("query(myDouble1)", inputs.input(2).name());
-                assertEquals("tensor()", inputs.input(2).type());
-                assertEquals("query(myDouble2)", inputs.input(3).name());
-                assertEquals("tensor()", inputs.input(3).type());
+                assertEquals(6, inputs.input().size());
+                assertInput("query(foo)", "tensor<float>(x[10])", inputs.input(0));
+                assertInput("query(bar)", "tensor(key{},x[1000])", inputs.input(1));
+                assertInput("query(myDouble1)", "tensor()", inputs.input(2));
+                assertInput("query(myDouble2)", "tensor()", inputs.input(3));
+                assertInput("query(myMap)", "tensor(key{})", inputs.input(4));
+                assertInput("query(myVector)", "tensor(x[3])", inputs.input(5));
+ //               assertInput("query(myMixed)", "tensor(key{},x[2])", inputs.input(5));
 
                 assertEquals(2, schema.summaryclass().size());
                 assertEquals("default", schema.summaryclass(0).name());
@@ -71,4 +73,9 @@ public class SchemaInfoTestCase {
         }
     }
 
+    private void assertInput(String name, String type, SchemaInfoConfig.Schema.Rankprofile.Input input) {
+        assertEquals(name, input.name());
+        assertEquals(type, input.type());
+    }
+
 }