Unify canonicalization

10 files changed, 111 insertions, 72 deletions

diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
index 8912d6e55b6..dd03cb8b2a7 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
@@ -1,5 +1,9 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchlib.rankingexpression.evaluation;
+/*
+ * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ *
+ *
+ */
+package com.yahoo.searchdefinition;
 
 import java.util.Arrays;
 import java.util.List;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index b185680d41c..135d983c1ca 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -16,7 +16,6 @@ import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
 import com.yahoo.searchlib.rankingexpression.FeatureList;
 import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.searchlib.rankingexpression.evaluation.FeatureNames;
 import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
 import com.yahoo.searchlib.rankingexpression.evaluation.TypeMapContext;
 import com.yahoo.searchlib.rankingexpression.evaluation.Value;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java
new file mode 100644
index 00000000000..40e9db1413f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java
@@ -0,0 +1,32 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.evaluation.TypeContext;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A context which only contains type information.
+ *
+ * @author bratseth
+ */
+public class TypeMapContext implements TypeContext {
+
+    private final Map<String, TensorType> featureTypes = new HashMap<>();
+
+    public void setType(String name, TensorType type) {
+        featureTypes.put(FeatureNames.canonicalize(name), type);
+    }
+
+    @Override
+    public TensorType getType(String name) {
+        return featureTypes.get(FeatureNames.canonicalize(name));
+    }
+
+    /** Returns an unmodifiable map of the bindings in this */
+    public Map<String, TensorType> bindings() { return Collections.unmodifiableMap(featureTypes); }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
index 3ee64094274..c121cdfdd9a 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -6,6 +6,7 @@ import com.yahoo.search.query.profile.types.FieldDescription;
 import com.yahoo.search.query.profile.types.FieldType;
 import com.yahoo.search.query.profile.types.QueryProfileType;
 import com.yahoo.search.query.profile.types.TensorFieldType;
+import com.yahoo.searchdefinition.FeatureNames;
 import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
@@ -14,7 +15,6 @@ import com.yahoo.searchdefinition.document.ImportedField;
 import com.yahoo.searchdefinition.document.ImportedFields;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.processing.Processor;
-import com.yahoo.searchlib.rankingexpression.evaluation.FeatureNames;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
 
 import java.util.Map;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java
new file mode 100644
index 00000000000..1f60ad870ec
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java
@@ -0,0 +1,58 @@
+/*
+ * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ *
+ *
+ */
+package com.yahoo.searchdefinition;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Tests rank feature names.
+ *
+ * @author bratseth
+ */
+public class FeatureNamesTestCase {
+
+    @Test
+    public void testCanonicalization() {
+        assertFalse(FeatureNames.canonicalizeIfValid("foo").isPresent());
+        assertEquals("query(bar)", FeatureNames.canonicalize("query(bar)"));
+        assertEquals("query(bar)", FeatureNames.canonicalize("query('bar')"));
+        assertEquals("constant(bar)", FeatureNames.canonicalize("constant(\"bar\")"));
+        assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query(ba.r)"));
+        assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query('ba.r')"));
+        assertEquals("attribute(\"ba.r\")", FeatureNames.canonicalize("attribute(\"ba.r\")"));
+    }
+
+    @Test
+    public void testArgument() {
+        assertFalse(FeatureNames.argumentOf("foo(bar)").isPresent());
+        assertFalse(FeatureNames.argumentOf("foo(bar.baz)").isPresent());
+        assertEquals("bar", FeatureNames.argumentOf("query(bar)").get());
+        assertEquals("bar.baz", FeatureNames.argumentOf("query(bar.baz)").get());
+        assertEquals("bar", FeatureNames.argumentOf("attribute(bar)").get());
+        assertEquals("bar.baz", FeatureNames.argumentOf("attribute(bar.baz)").get());
+        assertEquals("bar", FeatureNames.argumentOf("constant(bar)").get());
+        assertEquals("bar.baz", FeatureNames.argumentOf("constant(bar.baz)").get());
+    }
+
+    @Test
+    public void testConstantFeature() {
+        assertEquals("constant(\"foo/bar\")", FeatureNames.asConstantFeature("foo/bar"));
+    }
+
+    @Test
+    public void testAttributeFeature() {
+        assertEquals("attribute(foo)", FeatureNames.asAttributeFeature("foo"));
+    }
+
+    @Test
+    public void testQueryFeature() {
+        assertEquals("query(\"foo.bar\")", FeatureNames.asQueryFeature("foo.bar"));
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index 6458f3b9a6d..28e1f310078 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -100,7 +100,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
             String rankProperty = rankPropertyExpression.getFirst();
             if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) {
                 String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ",""));
-                System.out.println("--> " + rankExpression);
+                System.out.println("expression is --> " + rankExpression);
                 return rankExpression.equals(transformedExpression);
             }
         }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java
index 333af529cb9..a81d0c89f8f 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapContext.java
@@ -27,7 +27,7 @@ public class MapContext extends Context {
      * All the Values of the map will be frozen.
      */
     public MapContext(Map<String,Value> bindings) {
-        bindings.forEach((k, v) -> this.bindings.put(FeatureNames.canonicalize(k), v.freeze()));
+        bindings.forEach((k, v) -> this.bindings.put(k, v.freeze()));
     }
 
     /**
@@ -43,7 +43,7 @@ public class MapContext extends Context {
     /** Returns the type of the given value key, or null if it is not bound. */
     @Override
     public TensorType getType(String key) {
-        Value value = bindings.get(FeatureNames.canonicalize(key));
+        Value value = bindings.get(key);
         if (value == null) return null;
         return value.type();
     }
@@ -51,7 +51,7 @@ public class MapContext extends Context {
     /** Returns the value of a key. 0 is returned if the given key is not bound in this. */
     @Override
     public Value get(String key) {
-        return bindings.getOrDefault(FeatureNames.canonicalize(key), DoubleValue.zero);
+        return bindings.getOrDefault(key, DoubleValue.zero);
     }
 
     /**
@@ -59,7 +59,7 @@ public class MapContext extends Context {
      */
     @Override
     public void put(String key,Value value) {
-        bindings.put(FeatureNames.canonicalize(key), value.freeze());
+        bindings.put(key, value.freeze());
     }
 
     /** Returns an immutable view of the bindings of this. */
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java
index 0335ead4420..ff2088263d8 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java
@@ -18,12 +18,12 @@ public class TypeMapContext implements TypeContext {
     private final Map<String, TensorType> featureTypes = new HashMap<>();
 
     public void setType(String name, TensorType type) {
-        featureTypes.put(FeatureNames.canonicalize(name), type);
+        featureTypes.put(name, type);
     }
 
     @Override
     public TensorType getType(String name) {
-        return featureTypes.get(FeatureNames.canonicalize(name));
+        return featureTypes.get(name);
     }
 
     /** Returns an unmodifiable map of the bindings in this */
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNamesTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNamesTestCase.java
deleted file mode 100644
index 55319e4ec93..00000000000
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/FeatureNamesTestCase.java
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchlib.rankingexpression.evaluation;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests rank feature names.
- *
- * @author bratseth
- */
-public class FeatureNamesTestCase {
-
-    @Test
-    public void testCanonicalization() {
-        assertEquals("foo", FeatureNames.canonicalize("foo"));
-        assertEquals("foo.out", FeatureNames.canonicalize("foo.out"));
-        assertEquals("foo(bar)", FeatureNames.canonicalize("foo(bar)"));
-        assertEquals("foo(bar)", FeatureNames.canonicalize("foo('bar')"));
-        assertEquals("foo(bar)", FeatureNames.canonicalize("foo(\"bar\")"));
-        assertEquals("foo(bar).out", FeatureNames.canonicalize("foo(bar).out"));
-        assertEquals("foo(bar).out", FeatureNames.canonicalize("foo('bar').out"));
-        assertEquals("foo(bar).out", FeatureNames.canonicalize("foo(\"bar\").out"));
-        assertEquals("foo(\"ba.r\")", FeatureNames.canonicalize("foo(ba.r)"));
-        assertEquals("foo(\"ba.r\")", FeatureNames.canonicalize("foo('ba.r')"));
-        assertEquals("foo(\"ba.r\")", FeatureNames.canonicalize("foo(\"ba.r\")"));
-        assertEquals("foo(bar1,\"b.ar2\",\"ba/r3\").out",
-                     FeatureNames.canonicalize("foo(bar1,b.ar2,ba/r3).out"));
-        assertEquals("foo(bar1,\"b.ar2\",\"ba/r3\").out",
-                     FeatureNames.canonicalize("foo(bar1,'b.ar2',\"ba/r3\").out"));
-    }
-
-    @Test
-    public void testArgument() {
-        assertEquals("bar", FeatureNames.argumentOf("foo(bar)"));
-        assertEquals("bar.baz", FeatureNames.argumentOf("foo(bar.baz)"));
-    }
-
-    @Test
-    public void testConstantFeature() {
-        assertEquals("constant(\"foo/bar\")", FeatureNames.asConstantFeature("foo/bar"));
-    }
-
-    @Test
-    public void testAttributeFeature() {
-        assertEquals("attribute(foo)", FeatureNames.asAttributeFeature("foo"));
-    }
-
-    @Test
-    public void testQueryFeature() {
-        assertEquals("query(\"foo.bar\")", FeatureNames.asQueryFeature("foo.bar"));
-    }
-
-}
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java
index 5cac2215a00..c882c887c8d 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java
@@ -19,11 +19,11 @@ public class TypeResolutionTestCase {
     @Test
     public void testTypeResolution() {
         TypeMapContext context = new TypeMapContext();
-        context.setType("query('x1')", TensorType.fromSpec("tensor(x[])"));
-        context.setType("query('x2')", TensorType.fromSpec("tensor(x[10])"));
-        context.setType("query('y1')", TensorType.fromSpec("tensor(y[])"));
-        context.setType("query('xy1')", TensorType.fromSpec("tensor(x[10],y[])"));
-        context.setType("query('xy2')", TensorType.fromSpec("tensor(x[],y[10])"));
+        context.setType("query(x1)", TensorType.fromSpec("tensor(x[])"));
+        context.setType("query(x2)", TensorType.fromSpec("tensor(x[10])"));
+        context.setType("query(y1)", TensorType.fromSpec("tensor(y[])"));
+        context.setType("query(xy1)", TensorType.fromSpec("tensor(x[10],y[])"));
+        context.setType("query(xy2)", TensorType.fromSpec("tensor(x[],y[10])"));
 
         assertType("tensor(x[])", "query(x1)", context);
         assertType("tensor(x[])", "if (1>0, query(x1), query(x2))", context);
@@ -46,7 +46,8 @@ public class TypeResolutionTestCase {
             fail("Expected type incompatibility exception");
         }
         catch (IllegalArgumentException expected) {
-            assertEquals("An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[]) while the 'false' type is tensor(y[])",
+            assertEquals("An if expression must produce compatible types in both alternatives, " +
+                         "but the 'true' type is tensor(x[]) while the 'false' type is tensor(y[])",
                          expected.getMessage());
         }
         catch (ParseException e) {