Merge pull request #539 from yahoo/gv/rank-constant3

Support ranking constants in search definitions.

4 files changed, 190 insertions, 15 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
index f4046cedb5b..2a4e231dee4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java
@@ -12,7 +12,7 @@ public class RankingConstant {
     /** The search definition-unique name of this constant */
     private final String name;
     private TensorType tensorType = null;
-    private String fileName = "";
+    private String fileName = null;
     private String fileRef = "";
 
     public RankingConstant(String name) {
@@ -28,12 +28,19 @@ public class RankingConstant {
     public String getFileReference() { return fileRef; }
     public String getType() { return tensorType.toString(); }
 
+    public void validate() {
+        if (fileName == null || fileName.isEmpty())
+            throw new IllegalArgumentException("Ranking constants must have a file.");
+        if (tensorType == null)
+            throw new IllegalArgumentException("Ranking constant '" + name + "' must have a type.");
+    }
+
     public String toString() {
         StringBuilder b = new StringBuilder();
         b.append("constant '").append(name)
          .append("' from file '").append(fileName)
          .append("' with ref '").append(fileRef)
-         .append("' of type '" + tensorType)
+         .append("' of type '").append(tensorType)
          .append("'");
         return b.toString();
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index e8030fa6593..2ab634801c2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -141,6 +141,7 @@ public class Search implements Serializable {
     }
 
     public void addRankingConstant(RankingConstant rConstant) {
+        rConstant.validate();
         String name = rConstant.getName();
         if (rankingConstants.get(name) != null) {
             throw new IllegalArgumentException("Ranking constant '"+name+"' defined twice");
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index c93703c3340..57ff9854b1b 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -31,6 +31,7 @@ import com.yahoo.compress.CompressionType;
 import com.yahoo.searchdefinition.document.*;
 import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
 import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
+import com.yahoo.searchdefinition.RankingConstant;
 import com.yahoo.searchdefinition.Index;
 import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.DefaultRankProfile;
@@ -209,6 +210,7 @@ TOKEN :
 | < PREFIX: "prefix" >
 | < SUBSTRING: "substring" >
 | < SUFFIX: "suffix" >
+| < CONSTANT: "constant">
 | < RANKPROFILE: "rank-profile" >
 | < RANKDEGRADATIONFREQ: "rank-degradation-frequency" >
 | < RANKDEGRADATION: "rank-degradation" >
@@ -340,6 +342,7 @@ TOKEN :
 | < INTEGER: ("-")? (["0"-"9"])+ >
 | < LONG: ("-")? (["0"-"9"])+"L" >
 | < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ >
+| < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ >
 | < LESSTHAN: "<" >
 | < GREATERTHAN: ">" >
 | < VARIABLE: "$" <IDENTIFIER> >
@@ -411,6 +414,7 @@ Object rootSearchItem(Search search) : { }
       | documentSummary(search)
       | field(null, search)
       | index(search, null)
+      | rankingConstant(search)
       | rankProfile(search)
       | searchStemming(search)
       | useDocument(search)
@@ -1194,15 +1198,8 @@ Object attributeTensorType(AttributeOperation attribute) :
     TensorType tensorType;
 }
 {
-    <TENSOR_TYPE>
+    tensorType = tensorType("For attribute field '" + attribute.getName() + "'")
     {
-        tensorTypeString = token.image;
-        try {
-            tensorType = TensorType.fromSpec(tensorTypeString);
-        } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException("For attribute field '" + attribute.getName() +
-                "': Illegal tensor type spec: " + e.getMessage());
-        }
         attribute.setTensorType(tensorType);
     }
     { return null; }
@@ -1758,6 +1755,58 @@ Object indexBody(IndexOperation index) :
 }
 
 /**
+ * Consumes a constant block of a search element.
+ *
+ * @param search The search object to add content to.
+ */
+void rankingConstant(Search search) :
+{
+    String name;
+    RankingConstant constant;
+}
+{
+    ( <CONSTANT> name = identifier()
+        {
+            constant = new RankingConstant(name);
+        }
+      lbrace() (rankingConstantItem(constant) (<NL>)*)+ <RBRACE> )
+    {
+        search.addRankingConstant(constant);
+    }
+}
+
+/**
+ * This rule consumes a constant block.
+ *
+ * @param constant The constant to modify.
+ * @return Null.
+ */
+Object rankingConstantItem(RankingConstant constant) :
+{
+    String fileName = null;
+    TensorType type = null;
+}
+{
+    ( (<FILE> <COLON> fileName = filePath() { } (<NL>)*) { constant.setFileName(fileName); }
+      | type = tensorTypeWithPrefix(rankingConstantErrorMessage(constant.getName())) (<NL>)* { constant.setType(type); }
+    )
+    {
+        return null;
+    }
+}
+
+String rankingConstantErrorMessage(String name) : {}
+{
+    { return "For ranking constant ' " + name + "'"; }
+}
+
+String filePath() : { }
+{
+    ( <FILE_PATH> | <STRING> | <IDENTIFIER>)
+    { return token.image; }
+}
+
+/**
  * Consumes a rank-profile block of a search element.
  *
  * @param search The search object to add content to.
@@ -2269,7 +2318,7 @@ void constantTensor(RankProfile profile, String name) :
 {
     <LBRACE> (<NL>)*
       (( tensorString = tensorValue() |
-         tensorType = tensorType(profile.getName(), name) ) (<NL>)* )* <RBRACE>
+         tensorType = tensorTypeWithPrefix(constantTensorErrorMessage(profile.getName(), name)) ) (<NL>)* )* <RBRACE>
     {
         if (tensorType != null) {
             profile.addConstantTensor(name, new TensorValue(Tensor.from(tensorString), tensorType));
@@ -2279,6 +2328,11 @@ void constantTensor(RankProfile profile, String name) :
     }
 }
 
+String constantTensorErrorMessage(String rankProfileName, String constantTensorName) : {}
+{
+    { return "For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName + "'"; }
+}
+
 String tensorValue() :
 {
     String tensor;
@@ -2292,19 +2346,25 @@ String tensorValue() :
     }
 }
 
-TensorType tensorType(String rankProfileName, String constantTensorName) :
+TensorType tensorTypeWithPrefix(String errorMessage) :
+{ TensorType type; }
+{
+    <TYPE> <COLON> type= tensorType(errorMessage)
+    { return type; }
+}
+
+TensorType tensorType(String errorMessage) :
 {
     String tensorTypeString;
 }
 {
-    <TYPE> <COLON> ( <TENSOR_TYPE> { tensorTypeString = token.image; } )
+    <TENSOR_TYPE> { tensorTypeString = token.image; }
     {
         TensorType tensorType;
         try {
             tensorType = TensorType.fromSpec(tensorTypeString);
         } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException("For constant tensor '" + constantTensorName + "' in rank profile '" + rankProfileName +
-                "': Illegal tensor type spec: " + e.getMessage());
+            throw new IllegalArgumentException(errorMessage + ": Illegal tensor type spec: " + e.getMessage());
         }
         return tensorType;
     }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java
new file mode 100644
index 00000000000..a208ac2dec0
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java
@@ -0,0 +1,107 @@
+package com.yahoo.searchdefinition;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * @author gjoranv
+ */
+public class RankingConstantTest {
+
+    @Rule
+    public ExpectedException thrown = ExpectedException.none();
+
+    @Test
+    public void tensor_constant_properties_are_set() throws Exception {
+        final String TENSOR_NAME = "my_global_tensor";
+        final String TENSOR_FILE = "path/my-tensor-file.json";
+        final String TENSOR_TYPE = "tensor(x{})";
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  rank-profile my_rank_profile {",
+                "    first-phase {",
+                "      expression: sum(constant(my_global_tensor))",
+                "    }",
+                "  }",
+                "  constant " + TENSOR_NAME + " {",
+                "    file: " + TENSOR_FILE,
+                "    type: " + TENSOR_TYPE,
+                "  }",
+                "}"
+        ));
+        searchBuilder.build();
+        Search search = searchBuilder.getSearch();
+
+        Iterator<RankingConstant> constantIterator = search.getRankingConstants().iterator();
+        RankingConstant constant = constantIterator.next();
+        assertEquals(TENSOR_NAME, constant.getName());
+        assertEquals(TENSOR_FILE, constant.getFileName());
+        assertEquals(TENSOR_TYPE, constant.getType());
+
+        assertFalse(constantIterator.hasNext());
+    }
+
+    @Test
+    public void tensor_constant_must_have_a_type() throws Exception {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage("must have a type");
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  constant foo {",
+                "    file: bar.baz",
+                "  }",
+                "}"
+        ));
+    }
+
+    @Test
+    public void tensor_constant_must_have_a_file() throws Exception {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage("must have a file");
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  constant foo {",
+                "    type: tensor(x[])",
+                "  }",
+                "}"
+        ));
+    }
+
+    @Test
+    public void constant_file_does_not_need_path_or_ending() throws Exception {
+        RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+        SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
+        searchBuilder.importString(joinLines(
+                "search test {",
+                "  document test { }",
+                "  constant foo {",
+                "    type: tensor(x{})",
+                "    file: simplename",
+                "  }",
+                "}"
+        ));
+        searchBuilder.build();
+        Search search = searchBuilder.getSearch();
+        RankingConstant constant = search.getRankingConstants().iterator().next();
+        assertEquals("simplename", constant.getFileName());
+    }
+
+    private static String joinLines(String... lines) {
+        return String.join("\n", lines);
+    }
+}