diff options
Diffstat (limited to 'config-model/src/test/java/com/yahoo')
28 files changed, 180 insertions, 173 deletions
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 03fa92f5cb9..07a36832094 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -5,10 +5,12 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.io.IOException; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,9 +29,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT new DerivedConfiguration(search, registry, new QueryProfileRegistry(), new ImportedModels()); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { - e.printStackTrace(); - assertTrue(e.getCause().getMessage().contains("Could not read ranking expression file")); - assertTrue(e.getCause().getMessage().contains("wrongending.expr.expression")); + String message = Exceptions.toMessageString(e); + assertTrue(message.contains("Could not read ranking expression file")); + assertTrue(message.contains("wrongending.expr.expression")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index 28559f351ac..02ec597c3ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -4,6 +4,8 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; @@ -17,6 +19,7 @@ import static org.junit.Assert.assertNull; * @author Ulf Lilleengen */ public class RankProfileRegistryTest { + private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles"; @Test @@ -43,11 +46,11 @@ public class RankProfileRegistryTest { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { - assertNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); - RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry); - rankProfileWithAddedMacro.addMacro("foo", true); - rankProfileRegistry.add(rankProfileWithAddedMacro); - assertNotNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); + assertNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); + RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, search, rankProfileRegistry); + rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); + rankProfileRegistry.add(rankProfileWithAddedFunction); + assertNotNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index a524a26cbef..150469cc928 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -59,7 +59,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -76,7 +76,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedModels()); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); - assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); + assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString()); List<Pair<String, String>> rankProperties = new RawRankProfile(child2, queryProfileRegistry, new ImportedModels(), @@ -101,7 +101,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " c: 7 \n" + " }\n" + - " macro c() {\n" + + " function c() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -114,7 +114,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase fail("Should have caused an exception"); } catch (IllegalArgumentException e) { - assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and macro named 'c'", + assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'", Exceptions.toMessageString(e)); } } @@ -132,7 +132,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" + " }\n" + " }\n" + @@ -141,8 +141,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -161,7 +160,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " myValue: -9.21034037\n" + " }\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, myValue)\n" + " }\n" + " }\n" + @@ -170,14 +169,13 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test - public void testConstantDivisorInMacro() throws ParseException { + public void testConstantDivisorInFunction() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -186,7 +184,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: k1 + (k2 + k3) / 100000000.0\n\n" + " }\n" + " }\n" + @@ -196,7 +194,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 100000000.0", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } @Test @@ -212,7 +210,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" + " }\n" + " }\n" + @@ -222,7 +220,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index b13ffabda77..e507a6c48e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase { @Test - public void testMacroInliningPreserveArithemticOrdering() throws ParseException { + public void testFunctionInliningPreserveArithemticOrdering() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -44,18 +44,18 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: p1 * add\n" + " }\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 3 + attribute(a) + attribute(b) * mul3\n" + " }\n" + - " macro inline mul3() {\n" + + " function inline mul3() {\n" + " expression: attribute(a) * 3 + singleif\n" + " }\n" + - " macro inline singleif() {\n" + + " function inline singleif() {\n" + " expression: if (p1 < attribute(a), 1, 2) == 0\n" + " }\n" + " }\n" + " rank-profile child inherits parent {\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 9 + attribute(a)\n" + " }\n" + " }\n" + @@ -95,7 +95,7 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " second-phase {\n" + " expression: p2 * foo\n" + " }\n" + - " macro inline foo() {\n" + + " function inline foo() {\n" + " expression: 3 + p1 + p2\n" + " }\n" + " }\n" + @@ -106,16 +106,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro bar() {\n" + + " function bar() {\n" + " expression: p2*p1\n" + " }\n" + - " macro inline baz() {\n" + + " function inline baz() {\n" + " expression: p2+p1+boz\n" + " }\n" + - " macro inline boz() {\n" + + " function inline boz() {\n" + " expression: 3.0\n" + " }\n" + - " macro inline arg(a1) {\n" + + " function inline arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -162,16 +162,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: A + C + D\n" + " }\n" + - " macro inline D() {\n" + + " function inline D() {\n" + " expression: B + 1\n" + " }\n" + - " macro C() {\n" + + " function C() {\n" + " expression: A + B\n" + " }\n" + - " macro inline B() {\n" + + " function inline B() {\n" + " expression: attribute(b)\n" + " }\n" + - " macro inline A() {\n" + + " function inline A() {\n" + " expression: attribute(a)\n" + " }\n" + " }\n" + @@ -187,8 +187,8 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase } /** - * Expression evaluation has no stack so macro arguments are bound at config time creating a separate version of - * each macro for each binding, using hashes to name the bound variants of the macro. + * Expression evaluation has no stack so function arguments are bound at config time creating a separate version of + * each function for each binding, using hashes to name the bound variants of the function. * This method censors those hashes for string comparison. */ private String censorBindingHash(String s) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index df9a40d29e2..17bebcba70e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -29,7 +29,7 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: foo\n" + " }\n" + " }\n" + @@ -61,10 +61,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(5)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: foo + a1*2\n" + " }\n" + " }\n" + @@ -96,10 +96,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(foo)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -131,10 +131,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -159,10 +159,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3) + bar(a1)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -183,10 +183,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(bar(2))\n" + " }\n" + - " macro foo(x) {\n" + + " function foo(x) {\n" + " expression: x * x\n" + " }\n" + - " macro bar(x) {\n" + + " function bar(x) {\n" + " expression: x + x\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 1ece2355a92..e15d4075b19 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase { @Test - public void testBasicMacroShadowing() throws ParseException { + public void testBasicFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -31,7 +31,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -57,7 +57,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMultiLevelMacroShadowing() throws ParseException { + public void testMultiLevelFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -69,13 +69,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro tan(x) {\n" + + " function tan(x) {\n" + " expression: x * x\n" + " }\n" + - " macro cos(x) {\n" + + " function cos(x) {\n" + " expression: tan(x)\n" + " }\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: cos(x)\n" + " }\n" + " first-phase {\n" + @@ -113,7 +113,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMacroShadowingArguments() throws ParseException { + public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -125,7 +125,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -168,13 +168,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro relu(x) {\n" + // relu is a built in function, redefined here + " function relu(x) {\n" + // relu is a built in function, redefined here " expression: max(1.0, x)\n" + " }\n" + - " macro hidden_layer() {\n" + + " function hidden_layer() {\n" + " expression: relu(sum(query(q) * constant(W_hidden), input) + constant(b_input))\n" + " }\n" + - " macro final_layer() {\n" + + " function final_layer() {\n" + " expression: sigmoid(sum(hidden_layer * constant(W_final), hidden) + constant(b_final))\n" + " }\n" + " second-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 3fe3a7c3de1..5e649c2e551 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -5,9 +5,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Ignore; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -29,9 +31,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success - // TODO: Where's the "com.yahoo.searchdefinition.parser.ParseException:" nonsense coming from? - assertTrue("Got unexpected error message: " + e.getCause().getMessage(), - e.getCause().getMessage().startsWith("com.yahoo.searchdefinition.parser.ParseException: Could not parse ranking expression '" + expression + "'")); + assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 8cdfdd51637..82c03c02f61 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -34,20 +34,19 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { assertEquals("simple",search.getName()); assertTrue(search.hasDocument()); - SDDocumentType document=search.getDocument(); - assertEquals("simple",document.getName()); - assertEquals(12,document.getFieldCount()); + SDDocumentType document = search.getDocument(); + assertEquals("simple", document.getName()); + assertEquals(12, document.getFieldCount()); SDField field; Attribute attribute; - new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true); + new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); // First field field=(SDField) document.getField("title"); assertEquals(DataType.STRING,field.getDataType()); - assertEquals("{ summary | index; }", - field.getIndexingScript().toString()); + assertEquals("{ summary | index; }", field.getIndexingScript().toString()); assertTrue(!search.getIndex("default").isPrefix()); assertTrue(search.getIndex("title").isPrefix()); Iterator<String> titleAliases=search.getIndex("title").aliasIterator(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 3ef8591d203..f87a26c6f79 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -31,7 +31,7 @@ public class IdTestCase extends AbstractExportingTestCase { uri.parseIndexingScript("{ summary | index }"); document.addField(uri); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); assertNull(document.getField("uri")); assertNull(document.getField("Uri")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index fad80164135..2bae285301c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -41,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); // Check attribute fields diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index 9d19d4b154c..ba19a8312f6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -69,14 +69,14 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { assertTrue(!transforms.hasNext()); } @Test - public void testPositionDeriving() throws IOException, ParseException { + public void testPositionDeriving() { Search search = new Search("store", null); SDDocumentType document = new SDDocumentType("store"); search.addDocument(document); String fieldName = "location"; SDField field = document.addField(fieldName, PositionDataType.INSTANCE); field.parseIndexingScript("{ attribute | summary }"); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); SummaryMap summaryMap = new SummaryMap(search, new Summaries(search, new BaseDeployLogger())); Iterator transforms = summaryMap.resultTransformIterator(); @@ -141,7 +141,7 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { } @Test - public void testFailOnSummaryFieldSourceCollision() throws IOException, ParseException { + public void testFailOnSummaryFieldSourceCollision() { try { Search search = SearchBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 17cc9aaae4c..8941b07101d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -33,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase { a.parseIndexingScript("{ index }"); document.addField(a); - new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index a0556a156b5..48adc0eefc5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -36,7 +36,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( search,null,null,null); - processor.process(true); + processor.process(true, false); SummaryField summaryField = search.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); SummaryTransform actualTransform = summaryField.getTransform(); assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java index 8f1cc9c3de8..3a0fedfd550 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java @@ -24,7 +24,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { public void testInvalidAttributeProperties() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties1.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail("attribute property should not be set"); } catch (RuntimeException e) { // empty @@ -34,7 +34,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { @Test public void testValidAttributeProperties() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties2.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index 4dee939e3da..1ab8b054cb7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -24,7 +24,7 @@ public class BoldingTestCase extends SearchDefinitionTestCase { public void testBoldingNonString() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/processing/boldnonstring.sd"); - new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail(); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("'bolding: on' for non-text field")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 7230f176048..de08bf66548 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -188,7 +188,7 @@ public class ImportedFieldsResolverTestCase { private static ImportedFields resolve(Search search) { assertNotNull(search.temporaryImportedFields().get()); assertFalse(search.importedFields().isPresent()); - new ImportedFieldsResolver(search, null, null, null).process(true); + new ImportedFieldsResolver(search, null, null, null).process(true, false); assertFalse(search.temporaryImportedFields().isPresent()); assertNotNull(search.importedFields().get()); return search.importedFields().get(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index 2af3477fa62..e078d91f248 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -155,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { sdoc.addField(unprocessedField); Search search = new Search("test", null); search.addDocument(sdoc); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); return unprocessedField.getIndexingScript(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index 1e0270f293d..29bba224f46 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -19,11 +19,12 @@ import static org.junit.Assert.assertTrue; * @author baldersheim */ public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { + @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/integerindex2attribute.sd"); search.process(); - new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); SDField f; f = search.getConcreteField("s1"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index f67c85e2881..cff9abb08ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -72,9 +72,9 @@ class RankProfileSearchFixture { assertEquals(expValue, rankPropertyList.get(0).getValue()); } - public void assertMacro(String expexctedExpression, String macroName, String rankProfile) { + public void assertFunction(String expexctedExpression, String functionName, String rankProfile) { assertEquals(expexctedExpression, - compiledRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString()); + compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString()); } public RankProfile compileRankProfile(String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java index d8eb4368b57..0d8cbbf2e6a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java @@ -109,7 +109,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testMacroInvocationTypes() throws Exception { + public void testFunctionInvocationTypes() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString(joinLines( @@ -123,7 +123,7 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro macro1(attribute_to_use) {", + " function macro1(attribute_to_use) {", " expression: attribute(attribute_to_use)", " }", " summary-features {", @@ -143,7 +143,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testTensorMacroInvocationTypes_Nested() throws Exception { + public void testTensorFunctionInvocationTypes_Nested() throws Exception { SearchBuilder builder = new SearchBuilder(); builder.importString(joinLines( "search test {", @@ -156,16 +156,16 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro return_a() {", + " function return_a() {", " expression: return_first(attribute(a), attribute(b))", " }", - " macro return_b() {", + " function return_b() {", " expression: return_second(attribute(a), attribute(b))", " }", - " macro return_first(e1, e2) {", + " function return_first(e1, e2) {", " expression: e1", " }", - " macro return_second(e1, e2) {", + " function return_second(e1, e2) {", " expression: return_first(e2, e1)", " }", " summary-features {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index b046d60f948..8944409e1e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -128,7 +128,7 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testOnnxReferenceMissingMacro() throws ParseException { + public void testOnnxReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -145,14 +145,14 @@ public class RankingExpressionWithOnnxTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testOnnxReferenceWithWrongMacroType() { + public void testOnnxReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "onnx('mnist_softmax.onnx')"); @@ -163,7 +163,7 @@ public class RankingExpressionWithOnnxTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -213,13 +213,13 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstant() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstant() throws IOException { String rankProfile = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_Variable() {\n" + + " function " + name + "_Variable() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -234,7 +234,7 @@ public class RankingExpressionWithOnnxTestCase { search.compileRankProfile("my_profile", applicationDir.append("models")); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get( name + "_Variable")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -247,7 +247,7 @@ public class RankingExpressionWithOnnxTestCase { RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get( name + "_Variable")); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -275,19 +275,19 @@ public class RankingExpressionWithOnnxTestCase { } } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 14632a568ea..cba931e81f0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -160,7 +160,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceMissingMacro() throws ParseException { + public void testTensorFlowReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -177,14 +177,14 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testTensorFlowReferenceWithWrongMacroType() { + public void testTensorFlowReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "tensorflow('mnist_softmax/saved')"); @@ -195,7 +195,7 @@ public class RankingExpressionWithTensorFlowTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -261,13 +261,13 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstantAndInheritance() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException { String rankProfiles = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_layer_Variable_read() {\n" + + " function " + name + "_layer_Variable_read() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -285,7 +285,7 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -300,7 +300,7 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); } finally { @@ -317,11 +317,11 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testMacroGeneration() { + public void testFunctionGeneration() { final String name = "mnist_saved"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", "tensorflow('mnist/saved')", @@ -330,8 +330,8 @@ public class RankingExpressionWithTensorFlowTestCase { "input", new StoringApplicationPackage(applicationDir)); search.assertFirstPhaseExpression(expression, "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); } @Test @@ -339,7 +339,7 @@ public class RankingExpressionWithTensorFlowTestCase { final String name = "mnist_saved"; final String rankProfiles = " rank-profile my_profile {\n" + - " macro input() {\n" + + " function input() {\n" + " expression: tensor(d0[1],d1[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -349,9 +349,9 @@ public class RankingExpressionWithTensorFlowTestCase { " rank-profile my_profile_child inherits my_profile {\n" + " }"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWithUncompiled(rankProfiles, new StoringApplicationPackage(applicationDir)); search.compileRankProfile("my_profile", applicationDir.append("models")); @@ -359,10 +359,10 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(expression, "my_profile"); search.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); // At this point the expression is stored - copy application to another location which do not have a models dir Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); @@ -377,10 +377,10 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -429,19 +429,19 @@ public class RankingExpressionWithTensorFlowTestCase { new StoringApplicationPackage(applicationDir)); } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 0866d3192cf..48f2bf43e81 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -95,10 +95,10 @@ public class RankingExpressionWithTensorTestCase { } @Test - public void requireThatConstantTensorsCanBeUsedInMacro() throws ParseException { + public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + - " macro my_macro() {\n" + + " function my_macro() {\n" + " expression: sum(my_tensor)\n" + " }\n" + " first-phase {\n" + @@ -112,7 +112,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); - f.assertMacro("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); + f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 86127a260c5..fd048737b43 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -20,24 +20,24 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test - public void testMacros() throws IOException, ParseException { + public void testFunctions() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", rankProfileRegistry, new QueryProfileRegistry()).getSearch(); - final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); - macrosRankProfile.parseExpressions(); - final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros(); - assertEquals(2, macros.get("titlematch$").getFormalParams().size()); - assertEquals("var1", macros.get("titlematch$").getFormalParams().get(0)); - assertEquals("var2", macros.get("titlematch$").getFormalParams().get(1)); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getTextualExpression().trim()); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString()); - assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); - assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); - assertEquals(0, macros.get("artistmatch").getFormalParams().size()); + RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros"); + Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions(); + assertEquals(2, functions.get("titlematch$").function().arguments().size()); + assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); + assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); + assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", + functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); + assertEquals("78 + closeness(distance)", + functions.get("artistmatch").function().getBody().getRoot().toString()); + assertEquals(0, functions.get("artistmatch").function().arguments().size()); - List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, + List<Pair<String, String>> rankProperties = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(), new ImportedModels(), new AttributeFields(search)).configProperties(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java index 8a07e99101c..b39c48b67bf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -16,10 +16,10 @@ import static org.junit.Assert.assertTrue; /** * @author lesters */ -public class ReservedMacroNamesTestCase { +public class ReservedRankingExpressionFunctionNamesTestCase { @Test - public void requireThatMacrosWithReservedNamesIssueAWarning() throws ParseException { + public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { TestDeployLogger deployLogger = new TestDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); @@ -32,10 +32,10 @@ public class ReservedMacroNamesTestCase { " }\n" + " \n" + " rank-profile test_rank_profile {\n" + - " macro not_a_reserved_name(x) {\n" + + " function not_a_reserved_name(x) {\n" + " expression: x + x\n" + " }\n" + - " macro sigmoid(x) {\n" + + " function sigmoid(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -43,7 +43,7 @@ public class ReservedMacroNamesTestCase { " }\n" + " }\n" + " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index 85bee61c1b4..d0c1bf8b0ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -12,6 +12,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { @@ -20,41 +21,44 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTe Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); } } + @Test public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidimplicitsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); } } + @Test public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidselfreferringsummary.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); } } + @Test public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/documentidinsummary.sd"); search.process(); BaseDeployLogger deployLogger = new BaseDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true); + new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); assertEquals("documentid", search.getSummary("withid").getSummaryField("w").getSingleSource()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 76c50821cb9..8e721dbe503 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -113,7 +113,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { } @Test - public void requireThatMaxAndMinWithTensorsReturnedFromMacrosAreReplaced() throws ParseException { + public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException { assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", "max(returns_tensor,x)"); assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", @@ -171,7 +171,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:0 }\n" + " }\n" + " }\n" + - " macro base_tensor() {\n" + + " function base_tensor() {\n" + " expression: constant(base_constant_tensor)\n" + " }\n" + " }\n" + @@ -181,19 +181,19 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:1 }\n" + " }\n" + " }\n" + - " macro returns_tensor_with_arg(arg1) {\n" + + " function returns_tensor_with_arg(arg1) {\n" + " expression: 2.0 * arg1\n" + " }\n" + - " macro wraps_returns_tensor() {\n" + + " function wraps_returns_tensor() {\n" + " expression: returns_tensor\n" + " }\n" + - " macro returns_tensor() {\n" + + " function returns_tensor() {\n" + " expression: attribute(tensor_field_2)\n" + " }\n" + - " macro tensor_inheriting() {\n" + + " function tensor_inheriting() {\n" + " expression: base_tensor\n" + " }\n" + - " macro testexpression() {\n" + + " function testexpression() {\n" + " expression: " + expression + "\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index ab3bb113727..d0b6524a7e1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -40,7 +40,7 @@ public class ValidateFieldTypesTest { exceptionRule.expectMessage( "For search '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); - validator.process(true); + validator.process(true, false); } private static Search createSearchWithDocument(String documentName) { |