Publish

5 files changed, 249 insertions, 0 deletions

diff --git a/searchlib/src/tests/rankingexpression/feature_name_extractor/.gitignore b/searchlib/src/tests/rankingexpression/feature_name_extractor/.gitignore
new file mode 100644
index 00000000000..88c86c1720e
--- /dev/null
+++ b/searchlib/src/tests/rankingexpression/feature_name_extractor/.gitignore
@@ -0,0 +1 @@
+searchlib_feature_name_extractor_test_app
diff --git a/searchlib/src/tests/rankingexpression/feature_name_extractor/CMakeLists.txt b/searchlib/src/tests/rankingexpression/feature_name_extractor/CMakeLists.txt
new file mode 100644
index 00000000000..b1b81efd840
--- /dev/null
+++ b/searchlib/src/tests/rankingexpression/feature_name_extractor/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_feature_name_extractor_test_app
+    SOURCES
+    feature_name_extractor_test.cpp
+    DEPENDS
+    searchlib
+)
+vespa_add_test(NAME searchlib_feature_name_extractor_test_app COMMAND searchlib_feature_name_extractor_test_app)
diff --git a/searchlib/src/tests/rankingexpression/feature_name_extractor/FILES b/searchlib/src/tests/rankingexpression/feature_name_extractor/FILES
new file mode 100644
index 00000000000..6f6f6c1df43
--- /dev/null
+++ b/searchlib/src/tests/rankingexpression/feature_name_extractor/FILES
@@ -0,0 +1 @@
+feature_name_extractor_test.cpp
diff --git a/searchlib/src/tests/rankingexpression/feature_name_extractor/feature_name_extractor_test.cpp b/searchlib/src/tests/rankingexpression/feature_name_extractor/feature_name_extractor_test.cpp
new file mode 100644
index 00000000000..12ce67a586a
--- /dev/null
+++ b/searchlib/src/tests/rankingexpression/feature_name_extractor/feature_name_extractor_test.cpp
@@ -0,0 +1,79 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/features/rankingexpression/feature_name_extractor.h>
+
+using search::features::rankingexpression::FeatureNameExtractor;
+
+void verify_extract(const vespalib::string &input,
+                    const vespalib::string &expect_symbol,
+                    const vespalib::string &expect_after)
+{
+    FeatureNameExtractor extractor;
+    const char *pos_in = input.data();
+    const char *end_in = input.data() + input.size();
+    vespalib::string symbol_out;
+    const char *pos_out = nullptr;
+    extractor.extract_symbol(pos_in, end_in, pos_out, symbol_out);
+    ASSERT_TRUE(pos_out != nullptr);
+    vespalib::string after(pos_out, end_in);
+    EXPECT_EQUAL(expect_symbol, symbol_out);
+    EXPECT_EQUAL(expect_after, after);
+}
+
+TEST("require that basic names are extracted correctly") {
+    TEST_DO(verify_extract("foo+", "foo", "+"));
+    TEST_DO(verify_extract("foo.out+", "foo.out", "+"));
+    TEST_DO(verify_extract("foo(p1,p2)+", "foo(p1,p2)", "+"));
+    TEST_DO(verify_extract("foo(p1,p2).out+", "foo(p1,p2).out", "+"));
+}
+
+TEST("require that special characters are allowed in prefix and suffix") {
+    TEST_DO(verify_extract("_@$+", "_@$", "+"));
+    TEST_DO(verify_extract("_@$.$@_+", "_@$.$@_", "+"));
+    TEST_DO(verify_extract("_@$(p1,p2)+", "_@$(p1,p2)", "+"));
+    TEST_DO(verify_extract("_@$(p1,p2).$@_+", "_@$(p1,p2).$@_", "+"));
+}
+
+TEST("require that dot is only allowed in suffix") {
+    TEST_DO(verify_extract("foo.bar+", "foo.bar", "+"));
+    TEST_DO(verify_extract("foo.bar.out+", "foo.bar.out", "+"));
+    TEST_DO(verify_extract("foo.bar(p1,p2)+", "foo.bar", "(p1,p2)+"));
+    TEST_DO(verify_extract("foo.bar(p1,p2).out+", "foo.bar", "(p1,p2).out+"));
+    TEST_DO(verify_extract("foo(p1,p2).out.bar+", "foo(p1,p2).out.bar", "+"));
+}
+
+TEST("require that parameters can be nested") {
+    TEST_DO(verify_extract("foo(p1(a,b),p2(c,d(e,f))).out+", "foo(p1(a,b),p2(c,d(e,f))).out", "+"));
+}
+
+TEST("require that space is allowed among parameters") {
+    TEST_DO(verify_extract("foo( p1 ( a , b ) ).out+", "foo( p1 ( a , b ) ).out", "+"));
+}
+
+TEST("require that space is now allowed outside parameters") {
+    TEST_DO(verify_extract("foo +", "foo", " +"));
+    TEST_DO(verify_extract("foo . out+", "foo", " . out+"));
+    TEST_DO(verify_extract("foo. out+", "foo.", " out+"));
+    TEST_DO(verify_extract("foo (p1,p2)+", "foo", " (p1,p2)+"));
+    TEST_DO(verify_extract("foo(p1,p2) +", "foo(p1,p2)", " +"));
+    TEST_DO(verify_extract("foo(p1,p2) .out+", "foo(p1,p2)", " .out+"));
+    TEST_DO(verify_extract("foo(p1,p2).out +", "foo(p1,p2).out", " +"));
+}
+
+TEST("require that parameters can be scientific numbers") {
+    TEST_DO(verify_extract("foo(1.3E+3,-1.9e-10).out+", "foo(1.3E+3,-1.9e-10).out", "+"));
+}
+
+TEST("require that quoted parenthesis are not counted") {
+    TEST_DO(verify_extract("foo(a,b,\")\").out+", "foo(a,b,\")\").out", "+"));
+}
+
+TEST("require that escaped quotes does not unquote") {
+    TEST_DO(verify_extract("foo(a,b,\"\\\")\").out+", "foo(a,b,\"\\\")\").out", "+"));
+}
+
+TEST("require that escaped escape does not hinder unquote") {
+    TEST_DO(verify_extract("foo(a,b,\"\\\\\")\").out+", "foo(a,b,\"\\\\\")", "\").out+"));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/rankingexpression/rankingexpressionlist b/searchlib/src/tests/rankingexpression/rankingexpressionlist
new file mode 100644
index 00000000000..2ff1350025b
--- /dev/null
+++ b/searchlib/src/tests/rankingexpression/rankingexpressionlist
@@ -0,0 +1,160 @@
+# This file is a list of semicolon separated strings. The first string is the expression to be parsed, whereas all
+# following strings are allowed ways to print the parsed expression. If no alternatives are given, the expression can be
+# printed as the original. Note that all strings are trimmed before they are parsed / compared.
+  1
+1.0; 1.0; 1
+1e1; 1e1; 10
+1e-1; 1e-1; 0.1
+1.0e1; 1.0e1; 10
+1.0e-1; 1.0e-1; 0.1
+-1; -1
+1 + -1; 1 + -1; 1 - 1
+-1 + 1; -1 + 1
+tan(10)
+1
+ 1
+ 1 + 2
+ 1 - 2
+ 1 * 2
+ 1 / 2
+ 1 + 2 - 3
+ 1 + 2 - 3 * 4
+ 1 + 2 - 3 * 4 / 5
+1+2-3*4/5; 1 + 2 - 3 * 4 / 5
+(1)
+(1)+ 2; (1) + 2
+(1)+(2); (1) + (2)
+(1)+(2)-3; (1) + (2) - 3
+(1)+(2)-(3); (1) + (2) - (3)
+(1)+(2)-(3)*4; (1) + (2) - (3) * 4
+(1)+(2)-(3)*(4); (1) + (2) - (3) * (4)
+(1)+(2)-(3)*(4)/5; (1) + (2) - (3) * (4) / 5
+(1)+(2)-(3)*(4)/(5); (1) + (2) - (3) * (4) / (5)
+ 1 +(2)-(3)*(4)/(5); 1 + (2) - (3) * (4) / (5)
+ 1 + 2 -(3)*(4)/(5); 1 + 2 - (3) * (4) / (5)
+ 1 + 2 - 3 *(4)/(5); 1 + 2 - 3 * (4) / (5)
+ 1 + 2 - 3 * 4 /(5); 1 + 2 - 3 * 4 / (5)
+ 1 + 2 - 3 * 4 / 5 ; 1 + 2 - 3 * 4 / 5
+(1 + 2)
+(1 + 2)- 3; (1 + 2) - 3
+(1 + 2 - 3)
+(1 + 2 - 3)* 4; (1 + 2 - 3) * 4
+(1 + 2 - 3 * 4)
+(1 + 2 - 3 * 4)/ 5; (1 + 2 - 3 * 4) / 5
+(1 + 2 - 3 * 4 / 5)
+ 1 +(2 - 3 * 4 / 5); 1 + (2 - 3 * 4 / 5)
+ 1 + 2 -(3 * 4 / 5); 1 + 2 - (3 * 4 / 5)
+ 1 + 2 - 3 *(4 / 5); 1 + 2 - 3 * (4 / 5)
+1+2-3*(4/5); 1 + 2 - 3 * (4 / 5)
+log(1)
+log( 1 ); log(1)
+log( 1 + 2 ); log(1 + 2)
+log( 1 + 2 - 3 ); log(1 + 2 - 3)
+log( 1 + 2 - 3 * 4 ); log(1 + 2 - 3 * 4)
+log( 1 + 2 - 3 * 4 / 5 ); log(1 + 2 - 3 * 4 / 5)
+log((1 + 2)- 3 * 4 / 5 ); log((1 + 2) - 3 * 4 / 5)
+log( 1 +(2 - 3)* 4 / 5 ); log(1 + (2 - 3) * 4 / 5)
+log( 1 + 2 -(3 * 4)/ 5 ); log(1 + 2 - (3 * 4) / 5)
+log( 1 + 2 - 3 *(4 / 5)); log(1 + 2 - 3 * (4 / 5))
+log(1+2-3*4/5); log(1 + 2 - 3 * 4 / 5)
+""; ""
+"foo"
+"foo\""
+(1+"foo"); (1 + "foo")
+if("foo" == "bar", 1, 2); if ("foo" == "bar", 1, 2)
+cosh(1); cosh(1)
+cosh (1); cosh(1)
+cosh ( 1 ); cosh(1)
+cosh ( foo ); cosh(foo)
+cosh ( foo.out ); cosh(foo.out)
+cosh ( foo ( bar ) . out ); cosh(foo(bar).out)
+sin(10)
+cos(10)
+tan(10)
+acos(10)
+asin(10)
+atan(10)
+cosh(10)
+sinh(10)
+tanh(10)
+exp(10)
+log(10)
+log10(10)
+sqrt(10)
+ceil(10)
+fabs(10)
+floor(10)
+atan2(10, 20); atan2(10,20)
+ldexp(10, 20); ldexp(10,20)
+pow(10, 20); pow(10,20)
+fmod(10, 20); fmod(10,20)
+min(0, 1); min(0,1)
+max(1, 0); max(1,0)
+if(1<2,3,4); if (1 < 2, 3, 4)
+if(1>2,3,4); if (1 > 2, 3, 4)
+if(1==2,3,4); if (1 == 2, 3, 4)
+if(1~=2,3,4); if (1 ~= 2, 3, 4)
+if(1<=2,3,4); if (1 <= 2, 3, 4)
+if(1>=2,3,4); if (1 >= 2, 3, 4)
+if(1>=2,3,4,0.3); if (1 >= 2, 3, 4, 0.3)
+if(1>=2,3,4,0.5); if (1 >= 2, 3, 4, 0.5)
+if (1   <   2,   3,   4); if (1 < 2, 3, 4)
+if (1+2 <   3,   4,   5); if (1 + 2 < 3, 4, 5)
+if (1   < 2+3,   4,   5); if (1 < 2 + 3, 4, 5)
+if (1   <   2, 3+4,   5); if (1 < 2, 3 + 4, 5)
+if (1   <   2,   3, 4+5); if (1 < 2, 3, 4 + 5)
+if (foo in [bar], 6, 9); if (foo in [bar], 6, 9)
+if (foo in [bar,baz], 6, 9); if (foo in [bar, baz], 6, 9)
+if (foo in [bar,baz,cox], 6, 9); if (foo in [bar, baz, cox], 6, 9)
+if (foo in [bar], 6, 9)
+if (foo in [bar, baz], 6, 9)
+if (foo in [bar, baz, cox], 6, 9)
+if (foo in [ bar ], 6, 9); if (foo in [bar], 6, 9)
+if (foo in [ bar,  baz ], 6, 9); if (foo in [bar, baz], 6, 9)
+if (foo in [ bar,  baz,  cox ], 6, 9); if (foo in [bar, baz, cox], 6, 9)
+feature;                                    feature
+fe@ture;                                    fe@ture
+featur@;                                    featur@
+fe$ture;                                    fe$ture
+featur$;                                    featur$
+feature.out;                                feature.out
+feature .out;                               feature.out
+feature . out;                              feature.out
+feature.out.out;                            feature.out.out
+feature.if
+feature.in
+feature(arg1);                              feature(arg1)
+feature (arg1);                             feature(arg1)
+feature ( arg1);                            feature(arg1)
+feature ( arg1 );                           feature(arg1)
+feature(arg1,arg2);                         feature(arg1,arg2)
+feature(arg1 ,arg2);                        feature(arg1,arg2)
+feature(arg1 , arg2);                       feature(arg1,arg2)
+feature(arg1 , arg2).out;                   feature(arg1,arg2).out
+feature(arg1 , arg2) .  out;                feature(arg1,arg2).out
+feature("\",difficult","\")arguments\\").out
+feature(arg1,arg2).out;                     feature(arg1,arg2).out
+feature(if)
+feature(in)
+feature(cos)
+feature("cos(1,2)")
+feature(cos,sin,tan,cosh,sinh,tanh,acos,asin,atan,exp,log10,log,sqrt,ceil,fabs,floor)
+feature(cos,"sin(1,2)",3)
+rankingExpression(foo@92c9e83e1b665d2c.fe5dbbcea5ce7e29).rankingScript
+rankingExpression(foo@92c9e83e1b665d2c.2e5dbbcea5ce7e29).rankingScript
+mysum ( mysum(4,  4), value( 4 ), value(4) ); mysum(mysum(4,4),value(4),value(4))
+"\\"
+"\""
+"\f"
+"\female"
+"\n"
+"\nude"
+"\r"
+"fa\rt"
+"\t"
+"fe\tish"
+"\x10081977"
+"10\x081977"
+"1008\x1977"
+"100819\x77"
+if(1.09999~=1.1,2,3); if (1.09999 ~= 1.1, 2, 3)