Unify canonicalization

6 files changed, 212 insertions, 3 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
new file mode 100644
index 00000000000..dd03cb8b2a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
@@ -0,0 +1,120 @@
+/*
+ * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ *
+ *
+ */
+package com.yahoo.searchdefinition;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Utility methods for query, document and constant rank feature names
+ *
+ * @author bratseth
+ */
+public class FeatureNames {
+
+    private static final Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*");
+
+    /**
+     * <p>Returns the given query, document or constant feature in canonical form.
+     * A feature name consists of a feature type name (query, attribute or constant),
+     * followed by one argument enclosed in quotes.
+     * The argument may be an identifier or any string single or double quoted.</p>
+     *
+     * <p>Argument string values may not contain comma, single quote nor double quote characters.</p>
+     *
+     * <p><i>The canonical form use no quotes for arguments which are identifiers, and double quotes otherwise.</i></p>
+     *
+     * <p>Note that the above definition is not true for features in general, which accept any ranking expression
+     * as argument.</p>
+     *
+     * @throws IllegalArgumentException if the feature name is not valid
+     */
+    // Note that this implementation is more general than what is described above:
+    // It accepts any number of arguments and an optional output
+    public static String canonicalize(String feature) {
+        return canonicalizeIfValid(feature).orElseThrow(() ->
+            new IllegalArgumentException("A feature name must be on the form query(name), attribute(name) or " +
+                                         "constant(name), but was '" + feature + "'"
+        ));
+    }
+
+    /**
+     * Canonicalizes the given argument as in canonicalize, but returns empty instead of throwing an exception if
+     * the argument is not a valid feature
+     */
+    public static Optional<String> canonicalizeIfValid(String feature) {
+        int startParenthesis = feature.indexOf('(');
+        if (startParenthesis < 0)
+            return Optional.empty();
+        int endParenthesis = feature.lastIndexOf(')');
+        String featureType = feature.substring(0, startParenthesis);
+        if ( ! ( featureType.equals("query") || featureType.equals("attribute") || featureType.equals("constant")))
+            return Optional.empty();
+        if (startParenthesis < 1) return Optional.of(feature); // No arguments
+        if (endParenthesis < startParenthesis)
+            return Optional.empty();
+        String argumentString = feature.substring(startParenthesis + 1, endParenthesis);
+        List<String> canonicalizedArguments =
+                Arrays.stream(argumentString.split(","))
+                        .map(FeatureNames::canonicalizeArgument)
+                        .collect(Collectors.toList());
+        return Optional.of(featureType + "(" +
+                           canonicalizedArguments.stream().collect(Collectors.joining(",")) +
+                           feature.substring(endParenthesis));
+    }
+
+    /** Canomicalizes a single argument */
+    private static String canonicalizeArgument(String argument) {
+        if (argument.startsWith("'")) {
+            if ( ! argument.endsWith("'"))
+                throw new IllegalArgumentException("Feature arguments starting by a single quote " +
+                                                   "must end by a single quote, but was \"" + argument + "\"");
+            argument = argument.substring(1, argument.length() - 1);
+        }
+        if (argument.startsWith("\"")) {
+            if ( ! argument.endsWith("\""))
+                throw new IllegalArgumentException("Feature arguments starting by a double quote " +
+                                                   "must end by a double quote, but was '" + argument + "'");
+            argument = argument.substring(1, argument.length() - 1);
+        }
+        if (identifierRegexp.matcher(argument).matches())
+            return argument;
+        else
+            return "\"" + argument + "\"";
+    }
+
+    public static String asConstantFeature(String constantName) {
+        return canonicalize("constant(\"" + constantName + "\")");
+    }
+
+    public static String asAttributeFeature(String attributeName) {
+        return canonicalize("attribute(\"" + attributeName + "\")");
+    }
+
+    public static String asQueryFeature(String propertyName) {
+        return canonicalize("query(\"" + propertyName + "\")");
+    }
+
+    /**
+     * Returns the single argument of the given feature name, without any quotes,
+     * or empty if it is not a valid query, attribute or constant feature name
+     */
+    public static Optional<String> argumentOf(String feature) {
+        return canonicalizeIfValid(feature).map(f -> {
+            int startParenthesis = f.indexOf("(");
+            int endParenthesis = f.indexOf(")");
+            String possiblyQuotedArgument = f.substring(startParenthesis + 1, endParenthesis);
+            if (possiblyQuotedArgument.startsWith("\""))
+                return possiblyQuotedArgument.substring(1, possiblyQuotedArgument.length() - 1);
+            else
+                return possiblyQuotedArgument;
+        });
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index b185680d41c..135d983c1ca 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -16,7 +16,6 @@ import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
 import com.yahoo.searchlib.rankingexpression.FeatureList;
 import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.searchlib.rankingexpression.evaluation.FeatureNames;
 import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
 import com.yahoo.searchlib.rankingexpression.evaluation.TypeMapContext;
 import com.yahoo.searchlib.rankingexpression.evaluation.Value;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java
new file mode 100644
index 00000000000..40e9db1413f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java
@@ -0,0 +1,32 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.evaluation.TypeContext;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A context which only contains type information.
+ *
+ * @author bratseth
+ */
+public class TypeMapContext implements TypeContext {
+
+    private final Map<String, TensorType> featureTypes = new HashMap<>();
+
+    public void setType(String name, TensorType type) {
+        featureTypes.put(FeatureNames.canonicalize(name), type);
+    }
+
+    @Override
+    public TensorType getType(String name) {
+        return featureTypes.get(FeatureNames.canonicalize(name));
+    }
+
+    /** Returns an unmodifiable map of the bindings in this */
+    public Map<String, TensorType> bindings() { return Collections.unmodifiableMap(featureTypes); }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
index 3ee64094274..c121cdfdd9a 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -6,6 +6,7 @@ import com.yahoo.search.query.profile.types.FieldDescription;
 import com.yahoo.search.query.profile.types.FieldType;
 import com.yahoo.search.query.profile.types.QueryProfileType;
 import com.yahoo.search.query.profile.types.TensorFieldType;
+import com.yahoo.searchdefinition.FeatureNames;
 import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
@@ -14,7 +15,6 @@ import com.yahoo.searchdefinition.document.ImportedField;
 import com.yahoo.searchdefinition.document.ImportedFields;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.processing.Processor;
-import com.yahoo.searchlib.rankingexpression.evaluation.FeatureNames;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
 
 import java.util.Map;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java
new file mode 100644
index 00000000000..1f60ad870ec
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java
@@ -0,0 +1,58 @@
+/*
+ * // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+ *
+ *
+ */
+package com.yahoo.searchdefinition;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Tests rank feature names.
+ *
+ * @author bratseth
+ */
+public class FeatureNamesTestCase {
+
+    @Test
+    public void testCanonicalization() {
+        assertFalse(FeatureNames.canonicalizeIfValid("foo").isPresent());
+        assertEquals("query(bar)", FeatureNames.canonicalize("query(bar)"));
+        assertEquals("query(bar)", FeatureNames.canonicalize("query('bar')"));
+        assertEquals("constant(bar)", FeatureNames.canonicalize("constant(\"bar\")"));
+        assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query(ba.r)"));
+        assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query('ba.r')"));
+        assertEquals("attribute(\"ba.r\")", FeatureNames.canonicalize("attribute(\"ba.r\")"));
+    }
+
+    @Test
+    public void testArgument() {
+        assertFalse(FeatureNames.argumentOf("foo(bar)").isPresent());
+        assertFalse(FeatureNames.argumentOf("foo(bar.baz)").isPresent());
+        assertEquals("bar", FeatureNames.argumentOf("query(bar)").get());
+        assertEquals("bar.baz", FeatureNames.argumentOf("query(bar.baz)").get());
+        assertEquals("bar", FeatureNames.argumentOf("attribute(bar)").get());
+        assertEquals("bar.baz", FeatureNames.argumentOf("attribute(bar.baz)").get());
+        assertEquals("bar", FeatureNames.argumentOf("constant(bar)").get());
+        assertEquals("bar.baz", FeatureNames.argumentOf("constant(bar.baz)").get());
+    }
+
+    @Test
+    public void testConstantFeature() {
+        assertEquals("constant(\"foo/bar\")", FeatureNames.asConstantFeature("foo/bar"));
+    }
+
+    @Test
+    public void testAttributeFeature() {
+        assertEquals("attribute(foo)", FeatureNames.asAttributeFeature("foo"));
+    }
+
+    @Test
+    public void testQueryFeature() {
+        assertEquals("query(\"foo.bar\")", FeatureNames.asQueryFeature("foo.bar"));
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index 6458f3b9a6d..28e1f310078 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -100,7 +100,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
             String rankProperty = rankPropertyExpression.getFirst();
             if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) {
                 String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ",""));
-                System.out.println("--> " + rankExpression);
+                System.out.println("expression is --> " + rankExpression);
                 return rankExpression.equals(transformedExpression);
             }
         }