From ef8e4859e80c58035c3416c1fa8f24fd2d5af21d Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sun, 16 Sep 2018 18:40:46 +0200 Subject: Refactor: Initialize all macros state at once --- .../com/yahoo/searchdefinition/RankProfile.java | 70 +++++++++++++--------- 1 file changed, 42 insertions(+), 28 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b7e1f9d4538..2affcf5ca7d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -95,11 +95,11 @@ public class RankProfile implements Serializable, Cloneable { /** The properties of this - a multimap */ private Map> rankProperties = new LinkedHashMap<>(); - private Boolean ignoreDefaultRankFeatures=null; + private Boolean ignoreDefaultRankFeatures = null; - private String secondPhaseRankingString=null; + private String secondPhaseRankingString = null; - private String firstPhaseRankingString=null; + private String firstPhaseRankingString = null; private Map macros= new LinkedHashMap<>(); @@ -339,8 +339,8 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getFirstPhaseRanking() { - if (firstPhaseRanking!=null) return firstPhaseRanking; - if (getInherited()!=null) return getInherited().getFirstPhaseRanking(); + if (firstPhaseRanking != null) return firstPhaseRanking; + if (getInherited() != null) return getInherited().getFirstPhaseRanking(); return null; } @@ -353,13 +353,13 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getSecondPhaseRanking() { - if (secondPhaseRanking!=null) return secondPhaseRanking; - if (getInherited()!=null) return getInherited().getSecondPhaseRanking(); + if (secondPhaseRanking != null) return secondPhaseRanking; + if (getInherited() != null) return getInherited().getSecondPhaseRanking(); return null; } public void setSecondPhaseRanking(RankingExpression rankingExpression) { - this.secondPhaseRanking=rankingExpression; + this.secondPhaseRanking = rankingExpression; } /** @@ -412,8 +412,8 @@ public class RankProfile implements Serializable, Cloneable { } public void addRankFeature(ReferenceNode feature) { - if (rankFeatures==null) - rankFeatures=new LinkedHashSet<>(); + if (rankFeatures == null) + rankFeatures = new LinkedHashSet<>(); rankFeatures.add(feature); } @@ -548,9 +548,24 @@ public class RankProfile implements Serializable, Cloneable { return null; } - /** Creates a new (empty) macro and returns it */ - public Macro addMacro(String name, boolean inline) { - Macro macro = new Macro(name, inline); + /** Adds a new macro and returns it */ + public Macro addMacro(String name, RankingExpression expression, boolean inline) { + return addMacro(name, Collections.emptyList(), expression, inline); + } + + /** Adds a new macro and returns it */ + public Macro addMacro(String name, List arguments, String expression, boolean inline) { + try { + return addMacro(name, arguments, parseRankingExpression(name, expression), inline); + } + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse macro '" + name + "'", e); + } + } + + /** Adds a new macro and returns it */ + public Macro addMacro(String name, List arguments, RankingExpression expression, boolean inline) { + Macro macro = new Macro(name, arguments, expression, inline); macros.put(name, macro); return macro; } @@ -622,10 +637,6 @@ public class RankProfile implements Serializable, Cloneable { } } - /** - * Passes the contents of macros on to parser. Then put all the implied rank properties - * from those macros into the profile's props map. - */ private void parseMacros() throws ParseException { for (Map.Entry e : getMacros().entrySet()) { String macroName = e.getKey(); @@ -650,15 +661,15 @@ public class RankProfile implements Serializable, Cloneable { setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); } - private RankingExpression parseRankingExpression(String expressionName, String exp) throws ParseException { - if (exp.trim().length() == 0) + private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { + if (expression.trim().length() == 0) throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); - try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, exp.trim())) { + try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) { return new RankingExpression(expressionName, rankingExpressionReader); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { - ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() + + ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + "' in " + getName()+ ", " + expressionName + "."); throw (ParseException)exception.initCause(e); } @@ -936,7 +947,7 @@ public class RankProfile implements Serializable, Cloneable { @Override public int hashCode() { - return name.hashCode() + 17*value.hashCode(); + return name.hashCode() + 17 * value.hashCode(); } @Override @@ -960,23 +971,26 @@ public class RankProfile implements Serializable, Cloneable { private final String name; private String textualExpression = null; - private RankingExpression expression = null; - private List formalParams = new ArrayList<>(); + private RankingExpression expression; + private List arguments; /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ private final boolean inline; - public Macro(String name, boolean inline) { + public Macro(String name, List arguments, RankingExpression expression, boolean inline) { this.name = name; + this.arguments = arguments; + this.textualExpression = expression.getRoot().toString(); + this.expression = expression; this.inline = inline; } public void addParam(String name) { - formalParams.add(name); + arguments.add(name); } public List getFormalParams() { - return formalParams; + return arguments; } public String getTextualExpression() { @@ -1000,7 +1014,7 @@ public class RankProfile implements Serializable, Cloneable { } public boolean getInline() { - return inline && formalParams.size() == 0; // only inline no-arg macros; + return inline && arguments.size() == 0; // only inline no-arg macros; } public ExpressionFunction asExpressionFunction() { -- cgit v1.2.3 From b5531d333794b647e8a56ee88dd07e0ccf860404 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sun, 16 Sep 2018 18:47:05 +0200 Subject: Refactory: Remove mutations --- .../com/yahoo/searchdefinition/RankProfile.java | 31 ++-------------------- .../expressiontransforms/MacroShadower.java | 2 +- .../java/com/yahoo/vespa/model/VespaModel.java | 2 +- .../processing/RankingExpressionsTestCase.java | 11 ++++---- 4 files changed, 9 insertions(+), 37 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 2affcf5ca7d..1b9580d48ef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -631,24 +631,11 @@ public class RankProfile implements Serializable, Cloneable { public void parseExpressions() { try { parseRankingExpressions(); - parseMacros(); } catch (ParseException e) { throw new IllegalArgumentException(e); } } - private void parseMacros() throws ParseException { - for (Map.Entry e : getMacros().entrySet()) { - String macroName = e.getKey(); - Macro macro = e.getValue(); - if (macro.getRankingExpression() == null) { - RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); - macro.setRankingExpression(expr); - macro.setTextualExpression(expr.getRoot().toString()); - } - } - } - /** * Passes ranking expressions on to parser * @@ -970,7 +957,6 @@ public class RankProfile implements Serializable, Cloneable { public static class Macro implements Serializable, Cloneable { private final String name; - private String textualExpression = null; private RankingExpression expression; private List arguments; @@ -980,27 +966,14 @@ public class RankProfile implements Serializable, Cloneable { public Macro(String name, List arguments, RankingExpression expression, boolean inline) { this.name = name; this.arguments = arguments; - this.textualExpression = expression.getRoot().toString(); this.expression = expression; this.inline = inline; } - public void addParam(String name) { - arguments.add(name); - } - - public List getFormalParams() { + public List getArguments() { return arguments; } - public String getTextualExpression() { - return textualExpression; - } - - public void setTextualExpression(String textualExpression) { - this.textualExpression = textualExpression; - } - public void setRankingExpression(RankingExpression expr) { this.expression=expr; } @@ -1018,7 +991,7 @@ public class RankProfile implements Serializable, Cloneable { } public ExpressionFunction asExpressionFunction() { - return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression()); + return new ExpressionFunction(getName(), getArguments(), getRankingExpression()); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java index 758d2b2a87d..a1d7f2ddd2a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java @@ -49,7 +49,7 @@ public class MacroShadower extends ExpressionTransformer entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), entry.getValue(), false).setRankingExpression(entry.getValue()); + profile.addMacro(entry.getKey(), entry.getValue(), false); } } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 86127a260c5..deff4b20139 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -28,14 +28,13 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); macrosRankProfile.parseExpressions(); final Map macros = macrosRankProfile.getMacros(); - assertEquals(2, macros.get("titlematch$").getFormalParams().size()); - assertEquals("var1", macros.get("titlematch$").getFormalParams().get(0)); - assertEquals("var2", macros.get("titlematch$").getFormalParams().get(1)); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getTextualExpression().trim()); + assertEquals(2, macros.get("titlematch$").getArguments().size()); + assertEquals("var1", macros.get("titlematch$").getArguments().get(0)); + assertEquals("var2", macros.get("titlematch$").getArguments().get(1)); assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString()); assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); - assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); - assertEquals(0, macros.get("artistmatch").getFormalParams().size()); + assertEquals("78 + closeness(distance)", macros.get("artistmatch").getRankingExpression().getRoot().toString()); + assertEquals(0, macros.get("artistmatch").getArguments().size()); List> rankProperties = new RawRankProfile(macrosRankProfile, new QueryProfileRegistry(), -- cgit v1.2.3 From 688812e215e1e85d7d486344dc433fd24caa43c2 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 08:15:23 +0200 Subject: Refactor: Use ExpressionFunction --- .../com/yahoo/searchdefinition/RankProfile.java | 41 +++++++--------------- 1 file changed, 13 insertions(+), 28 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 1b9580d48ef..9afced44b1d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -751,10 +751,9 @@ public class RankProfile implements Serializable, Cloneable { ExpressionTransforms expressionTransforms) { Map compiledMacros = new LinkedHashMap<>(); for (Map.Entry entry : macros.entrySet()) { - Macro macro = entry.getValue().clone(); - RankingExpression exp = compile(macro.getRankingExpression(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - macro.setRankingExpression(exp); - compiledMacros.put(entry.getKey(), macro); + Macro macro = entry.getValue(); + RankingExpression compiled = compile(macro.getRankingExpression(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); + compiledMacros.put(entry.getKey(), macro.withExpression(compiled)); } return compiledMacros; } @@ -954,59 +953,45 @@ public class RankProfile implements Serializable, Cloneable { /** * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro */ - public static class Macro implements Serializable, Cloneable { + public static class Macro { - private final String name; - private RankingExpression expression; - private List arguments; + private final ExpressionFunction function; /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ private final boolean inline; public Macro(String name, List arguments, RankingExpression expression, boolean inline) { - this.name = name; - this.arguments = arguments; - this.expression = expression; + this.function = new ExpressionFunction(name, arguments, expression); this.inline = inline; } public List getArguments() { - return arguments; + return function.arguments(); } - public void setRankingExpression(RankingExpression expr) { - this.expression=expr; + public Macro withExpression(RankingExpression expression) { + return new Macro(function.getName(), function.arguments(), expression, inline); } public RankingExpression getRankingExpression() { - return expression; + return function.getBody(); } public String getName() { - return name; + return function.getName(); } public boolean getInline() { - return inline && arguments.size() == 0; // only inline no-arg macros; + return inline && function.arguments().isEmpty(); // only inline no-arg macros; } public ExpressionFunction asExpressionFunction() { return new ExpressionFunction(getName(), getArguments(), getRankingExpression()); } - @Override - public Macro clone() { - try { - return (Macro)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Won't happen", e); - } - } - @Override public String toString() { - return "macro " + getName() + ": " + expression; + return "macro " + getName() + ": " + getRankingExpression(); } } -- cgit v1.2.3 From 5a6f9b703f228a6bb5565d5e680f22a3e3059ff0 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 09:11:30 +0200 Subject: Refactor: Expose function --- .../main/java/com/yahoo/searchdefinition/RankProfile.java | 14 +++++++------- .../com/yahoo/searchdefinition/derived/RawRankProfile.java | 2 +- .../expressiontransforms/MacroInliner.java | 2 +- .../main/java/com/yahoo/vespa/model/ml/ConvertedModel.java | 12 ++++++------ .../RankingExpressionConstantsTestCase.java | 12 ++++++------ .../processing/RankProfileSearchFixture.java | 2 +- .../processing/RankingExpressionsTestCase.java | 4 ++-- 7 files changed, 24 insertions(+), 24 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 9afced44b1d..98238d886db 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -740,7 +740,7 @@ public class RankProfile implements Serializable, Cloneable { } private Map getInlineMacros() { - return getMacros().entrySet().stream().filter(x -> x.getValue().getInline()) + return getMacros().entrySet().stream().filter(x -> x.getValue().inline()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } @@ -752,7 +752,7 @@ public class RankProfile implements Serializable, Cloneable { Map compiledMacros = new LinkedHashMap<>(); for (Map.Entry entry : macros.entrySet()) { Macro macro = entry.getValue(); - RankingExpression compiled = compile(macro.getRankingExpression(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); + RankingExpression compiled = compile(macro.function().getBody(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); compiledMacros.put(entry.getKey(), macro.withExpression(compiled)); } return compiledMacros; @@ -783,7 +783,7 @@ public class RankProfile implements Serializable, Cloneable { */ public TypeContext typeContext(QueryProfileRegistry queryProfiles) { MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() - .map(Macro::asExpressionFunction) + .map(Macro::function) .collect(Collectors.toList())); // Add small and large constants, respectively @@ -981,17 +981,17 @@ public class RankProfile implements Serializable, Cloneable { return function.getName(); } - public boolean getInline() { + public boolean inline() { return inline && function.arguments().isEmpty(); // only inline no-arg macros; } - public ExpressionFunction asExpressionFunction() { - return new ExpressionFunction(getName(), getArguments(), getRankingExpression()); + public ExpressionFunction function() { + return function; } @Override public String toString() { - return "macro " + getName() + ": " + getRankingExpression(); + return "macro " + getName() + ": " + function().getBody(); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index ffd7f74cfe0..d3ba7d4b613 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -182,7 +182,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (macros.isEmpty()) return; Map expressionMacros = new LinkedHashMap<>(); for (Map.Entry macro : macros.entrySet()) { - expressionMacros.put(macro.getKey(), macro.getValue().asExpressionFunction()); + expressionMacros.put(macro.getKey(), macro.getValue().function()); } Map macroProperties = new LinkedHashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java index 6aef39db4da..b5642a5426f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java @@ -26,7 +26,7 @@ public class MacroInliner extends ExpressionTransformer transformGeneratedMacro(store, constantsReplacedByMacros, k, - profile.getMacros().get(k).getRankingExpression())); + profile.getMacros().get(k).function().getBody())); return expressions; } @@ -252,7 +252,7 @@ public class ConvertedModel { Tensor constantValue) { RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName); if (macroOverridingConstant != null) { - TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles)); + TensorType macroType = macroOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); if ( ! macroType.equals(constantValue.type())) throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + typeMismatchExplanation(constantValue.type(), macroType)); @@ -278,10 +278,10 @@ public class ConvertedModel { private static void addGeneratedMacroToProfile(RankProfile profile, String macroName, RankingExpression expression) { if (profile.getMacros().containsKey(macroName)) { - if ( ! profile.getMacros().get(macroName).getRankingExpression().equals(expression)) + if ( ! profile.getMacros().get(macroName).function().getBody().equals(expression)) throw new IllegalArgumentException("Generated macro '" + macroName + "' already exists in " + profile + " - with a different definition" + - ": Has\n" + profile.getMacros().get(macroName).getRankingExpression() + + ": Has\n" + profile.getMacros().get(macroName).function().getBody() + "\nwant to add " + expression + "\n"); return; } @@ -309,7 +309,7 @@ public class ConvertedModel { // phase and summary features), as it may only resolve correctly given those bindings // Or, probably better, annotate the macros with type constraints here and verify during general // type verification - TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); + TensorType actualType = macro.function().getBody().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) throw new IllegalArgumentException("Model refers input '" + macroName + "' of type " + requiredType + @@ -356,7 +356,7 @@ public class ConvertedModel { throw new IllegalArgumentException("Model refers to generated macro '" + macroName + "but this macro is not present in " + profile); } - RankingExpression macroExpression = macro.getRankingExpression(); + RankingExpression macroExpression = macro.function().getBody(); macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index a524a26cbef..bceafc8e415 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -76,7 +76,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedModels()); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); - assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); + assertEquals("foo: 14.0", child2.getMacros().get("foo").function().getBody().toString()); List> rankProperties = new RawRankProfile(child2, queryProfileRegistry, new ImportedModels(), @@ -142,7 +142,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -171,9 +171,9 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -196,7 +196,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 100000000.0", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").function().getBody().getRoot().toString()); } @Test @@ -222,7 +222,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").function().getBody().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index f67c85e2881..749a97bf12f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -74,7 +74,7 @@ class RankProfileSearchFixture { public void assertMacro(String expexctedExpression, String macroName, String rankProfile) { assertEquals(expexctedExpression, - compiledRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString()); + compiledRankProfile(rankProfile).getMacros().get(macroName).function().getBody().getRoot().toString()); } public RankProfile compileRankProfile(String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index deff4b20139..0f8e6072d53 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -31,9 +31,9 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { assertEquals(2, macros.get("titlematch$").getArguments().size()); assertEquals("var1", macros.get("titlematch$").getArguments().get(0)); assertEquals("var2", macros.get("titlematch$").getArguments().get(1)); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString()); + assertEquals("var1 * var2 + 890", macros.get("titlematch$").function().getBody().getRoot().toString()); assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); - assertEquals("78 + closeness(distance)", macros.get("artistmatch").getRankingExpression().getRoot().toString()); + assertEquals("78 + closeness(distance)", macros.get("artistmatch").function().getBody().getRoot().toString()); assertEquals(0, macros.get("artistmatch").getArguments().size()); List> rankProperties = new RawRankProfile(macrosRankProfile, -- cgit v1.2.3 From 347db07eb16da7ff3a4b2cac676caa932e8136dc Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 09:47:39 +0200 Subject: Refactor: Construct using ExpressionFunction --- .../com/yahoo/searchdefinition/RankProfile.java | 43 ++++++---------------- .../expressiontransforms/MacroShadower.java | 4 +- .../java/com/yahoo/vespa/model/VespaModel.java | 5 ++- .../com/yahoo/vespa/model/ml/ConvertedModel.java | 3 +- .../searchdefinition/RankProfileRegistryTest.java | 3 +- .../processing/RankingExpressionsTestCase.java | 8 ++-- .../rankingexpression/ExpressionFunction.java | 11 ++++++ 7 files changed, 35 insertions(+), 42 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 98238d886db..200eac0cdad 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -548,15 +548,10 @@ public class RankProfile implements Serializable, Cloneable { return null; } - /** Adds a new macro and returns it */ - public Macro addMacro(String name, RankingExpression expression, boolean inline) { - return addMacro(name, Collections.emptyList(), expression, inline); - } - /** Adds a new macro and returns it */ public Macro addMacro(String name, List arguments, String expression, boolean inline) { try { - return addMacro(name, arguments, parseRankingExpression(name, expression), inline); + return addMacro(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); } catch (ParseException e) { throw new IllegalArgumentException("Could not parse macro '" + name + "'", e); @@ -564,9 +559,9 @@ public class RankProfile implements Serializable, Cloneable { } /** Adds a new macro and returns it */ - public Macro addMacro(String name, List arguments, RankingExpression expression, boolean inline) { - Macro macro = new Macro(name, arguments, expression, inline); - macros.put(name, macro); + public Macro addMacro(ExpressionFunction function, boolean inline) { + Macro macro = new Macro(function, inline); + macros.put(function.getName(), macro); return macro; } @@ -753,7 +748,7 @@ public class RankProfile implements Serializable, Cloneable { for (Map.Entry entry : macros.entrySet()) { Macro macro = entry.getValue(); RankingExpression compiled = compile(macro.function().getBody(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - compiledMacros.put(entry.getKey(), macro.withExpression(compiled)); + compiledMacros.put(entry.getKey(), macro.withBody(compiled)); } return compiledMacros; } @@ -951,7 +946,7 @@ public class RankProfile implements Serializable, Cloneable { } /** - * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro + * A function in a rank profile */ public static class Macro { @@ -960,38 +955,24 @@ public class RankProfile implements Serializable, Cloneable { /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ private final boolean inline; - public Macro(String name, List arguments, RankingExpression expression, boolean inline) { - this.function = new ExpressionFunction(name, arguments, expression); + public Macro(ExpressionFunction function, boolean inline) { + this.function = function; this.inline = inline; } - public List getArguments() { - return function.arguments(); - } - - public Macro withExpression(RankingExpression expression) { - return new Macro(function.getName(), function.arguments(), expression, inline); - } - - public RankingExpression getRankingExpression() { - return function.getBody(); - } - - public String getName() { - return function.getName(); - } + public ExpressionFunction function() { return function; } public boolean inline() { return inline && function.arguments().isEmpty(); // only inline no-arg macros; } - public ExpressionFunction function() { - return function; + public Macro withBody(RankingExpression expression) { + return new Macro(function.withBody(expression), inline); } @Override public String toString() { - return "macro " + getName() + ": " + function().getBody(); + return "function " + function; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java index a1d7f2ddd2a..13f31c0254e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java @@ -49,10 +49,8 @@ public class MacroShadower extends ExpressionTransformer entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), entry.getValue(), false); + profile.addMacro(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } @@ -248,7 +249,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); for (Map.Entry entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), entry.getValue(), false); + profile.addMacro(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 390fd32f0ba..fb1ca9a0822 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -13,6 +13,7 @@ import com.yahoo.searchdefinition.FeatureNames; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; @@ -285,7 +286,7 @@ public class ConvertedModel { "\nwant to add " + expression + "\n"); return; } - profile.addMacro(macroName, expression, false); // TODO: Inline if only used once + profile.addMacro(new ExpressionFunction(macroName, expression), false); // TODO: Inline if only used once } /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index e8bc4f44738..7b262bdc4d0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; @@ -46,7 +47,7 @@ public class RankProfileRegistryTest { for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { assertNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry); - rankProfileWithAddedMacro.addMacro("foo", RankingExpression.from("1+2"), true); + rankProfileWithAddedMacro.addMacro(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); rankProfileRegistry.add(rankProfileWithAddedMacro); assertNotNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 0f8e6072d53..132807bf8b8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -28,13 +28,13 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); macrosRankProfile.parseExpressions(); final Map macros = macrosRankProfile.getMacros(); - assertEquals(2, macros.get("titlematch$").getArguments().size()); - assertEquals("var1", macros.get("titlematch$").getArguments().get(0)); - assertEquals("var2", macros.get("titlematch$").getArguments().get(1)); + assertEquals(2, macros.get("titlematch$").function().arguments().size()); + assertEquals("var1", macros.get("titlematch$").function().arguments().get(0)); + assertEquals("var2", macros.get("titlematch$").function().arguments().get(1)); assertEquals("var1 * var2 + 890", macros.get("titlematch$").function().getBody().getRoot().toString()); assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); assertEquals("78 + closeness(distance)", macros.get("artistmatch").function().getBody().getRoot().toString()); - assertEquals(0, macros.get("artistmatch").getArguments().size()); + assertEquals(0, macros.get("artistmatch").function().arguments().size()); List> rankProperties = new RawRankProfile(macrosRankProfile, new QueryProfileRegistry(), diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 432fab00b2b..da34ab8822d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -8,6 +8,7 @@ import com.yahoo.text.Utf8; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.List; @@ -25,6 +26,16 @@ public class ExpressionFunction { private final ImmutableList arguments; private final RankingExpression body; + /** + * Constructs a new function with no arguments + * + * @param name the name of this function + * @param body the ranking expression that defines this function + */ + public ExpressionFunction(String name, RankingExpression body) { + this(name, Collections.emptyList(), body); + } + /** * Constructs a new function * -- cgit v1.2.3 From e2bc647f01162d7b5f0886337f308a9be6629a40 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 09:53:03 +0200 Subject: Refactor: Macro -> RankingExpressionFunction --- .../com/yahoo/searchdefinition/RankProfile.java | 60 +++++++++--------- .../searchdefinition/derived/RawRankProfile.java | 4 +- .../expressiontransforms/MacroInliner.java | 6 +- .../expressiontransforms/MacroShadower.java | 6 +- .../RankProfileTransformContext.java | 6 +- .../com/yahoo/vespa/model/ml/ConvertedModel.java | 18 +++--- .../processing/RankingExpressionsTestCase.java | 2 +- .../processing/ReservedMacroNamesTestCase.java | 72 ---------------------- ...rvedRankingExpressionFunctionNamesTestCase.java | 72 ++++++++++++++++++++++ 9 files changed, 122 insertions(+), 124 deletions(-) delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java create mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 200eac0cdad..509fa243bc3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -101,7 +101,7 @@ public class RankProfile implements Serializable, Cloneable { private String firstPhaseRankingString = null; - private Map macros= new LinkedHashMap<>(); + private Map macros= new LinkedHashMap<>(); private Set filterFields = new HashSet<>(); @@ -549,7 +549,7 @@ public class RankProfile implements Serializable, Cloneable { } /** Adds a new macro and returns it */ - public Macro addMacro(String name, List arguments, String expression, boolean inline) { + public RankingExpressionFunction addMacro(String name, List arguments, String expression, boolean inline) { try { return addMacro(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); } @@ -559,20 +559,20 @@ public class RankProfile implements Serializable, Cloneable { } /** Adds a new macro and returns it */ - public Macro addMacro(ExpressionFunction function, boolean inline) { - Macro macro = new Macro(function, inline); - macros.put(function.getName(), macro); - return macro; + public RankingExpressionFunction addMacro(ExpressionFunction function, boolean inline) { + RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); + macros.put(function.getName(), rankingExpressionFunction); + return rankingExpressionFunction; } /** Returns an unmodifiable view of the macros in this */ - public Map getMacros() { + public Map getMacros() { if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap(); if (macros.size() == 0) return getInherited().getMacros(); if (getInherited() == null) return Collections.unmodifiableMap(macros); // Neither is null - Map allMacros = new LinkedHashMap<>(getInherited().getMacros()); + Map allMacros = new LinkedHashMap<>(getInherited().getMacros()); allMacros.putAll(macros); return Collections.unmodifiableMap(allMacros); @@ -717,7 +717,7 @@ public class RankProfile implements Serializable, Cloneable { ExpressionTransforms expressionTransforms = new ExpressionTransforms(); // Macro compiling first pass: compile inline macros without resolving other macros - Map inlineMacros = compileMacros(getInlineMacros(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); + Map inlineMacros = compileMacros(getInlineMacros(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); // Macro compiling second pass: compile all macros and insert previously compiled inline macros macros = compileMacros(getMacros(), queryProfiles, importedModels, inlineMacros, expressionTransforms); @@ -726,29 +726,29 @@ public class RankProfile implements Serializable, Cloneable { secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); } - private void checkNameCollisions(Map macros, Map constants) { - for (Map.Entry macroEntry : macros.entrySet()) { + private void checkNameCollisions(Map macros, Map constants) { + for (Map.Entry macroEntry : macros.entrySet()) { if (constants.get(macroEntry.getKey()) != null) throw new IllegalArgumentException("Cannot have both a constant and macro named '" + macroEntry.getKey() + "'"); } } - private Map getInlineMacros() { + private Map getInlineMacros() { return getMacros().entrySet().stream().filter(x -> x.getValue().inline()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private Map compileMacros(Map macros, - QueryProfileRegistry queryProfiles, - ImportedModels importedModels, - Map inlineMacros, - ExpressionTransforms expressionTransforms) { - Map compiledMacros = new LinkedHashMap<>(); - for (Map.Entry entry : macros.entrySet()) { - Macro macro = entry.getValue(); - RankingExpression compiled = compile(macro.function().getBody(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - compiledMacros.put(entry.getKey(), macro.withBody(compiled)); + private Map compileMacros(Map macros, + QueryProfileRegistry queryProfiles, + ImportedModels importedModels, + Map inlineMacros, + ExpressionTransforms expressionTransforms) { + Map compiledMacros = new LinkedHashMap<>(); + for (Map.Entry entry : macros.entrySet()) { + RankingExpressionFunction rankingExpressionFunction = entry.getValue(); + RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); + compiledMacros.put(entry.getKey(), rankingExpressionFunction.withBody(compiled)); } return compiledMacros; } @@ -757,7 +757,7 @@ public class RankProfile implements Serializable, Cloneable { QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map constants, - Map inlineMacros, + Map inlineMacros, ExpressionTransforms expressionTransforms) { if (expression == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, @@ -778,7 +778,7 @@ public class RankProfile implements Serializable, Cloneable { */ public TypeContext typeContext(QueryProfileRegistry queryProfiles) { MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() - .map(Macro::function) + .map(RankingExpressionFunction::function) .collect(Collectors.toList())); // Add small and large constants, respectively @@ -945,17 +945,15 @@ public class RankProfile implements Serializable, Cloneable { } - /** - * A function in a rank profile - */ - public static class Macro { + /** A function in a rank profile */ + public static class RankingExpressionFunction { private final ExpressionFunction function; /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ private final boolean inline; - public Macro(ExpressionFunction function, boolean inline) { + public RankingExpressionFunction(ExpressionFunction function, boolean inline) { this.function = function; this.inline = inline; } @@ -966,8 +964,8 @@ public class RankProfile implements Serializable, Cloneable { return inline && function.arguments().isEmpty(); // only inline no-arg macros; } - public Macro withBody(RankingExpression expression) { - return new Macro(function.withBody(expression), inline); + public RankingExpressionFunction withBody(RankingExpression expression) { + return new RankingExpressionFunction(function.withBody(expression), inline); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index d3ba7d4b613..98d147aca79 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -178,10 +178,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer { derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros()); } - private void derivePropertiesAndSummaryFeaturesFromMacros(Map macros) { + private void derivePropertiesAndSummaryFeaturesFromMacros(Map macros) { if (macros.isEmpty()) return; Map expressionMacros = new LinkedHashMap<>(); - for (Map.Entry macro : macros.entrySet()) { + for (Map.Entry macro : macros.entrySet()) { expressionMacros.put(macro.getKey(), macro.getValue().function()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java index b5642a5426f..155568825e0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java @@ -24,9 +24,9 @@ public class MacroInliner extends ExpressionTransformer inlineMacros; + private final Map inlineMacros; private final Map rankProperties = new HashMap<>(); public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map constants, - Map inlineMacros) { + Map inlineMacros) { super(constants); this.rankProfile = rankProfile; this.queryProfiles = queryProfiles; @@ -38,7 +38,7 @@ public class RankProfileTransformContext extends TransformContext { public RankProfile rankProfile() { return rankProfile; } public QueryProfileRegistry queryProfiles() { return queryProfiles; } public ImportedModels importedModels() { return importedModels; } - public Map inlineMacros() { return inlineMacros; } + public Map inlineMacros() { return inlineMacros; } public Map rankProperties() { return rankProperties; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index fb1ca9a0822..ffc3c53fdb6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -251,9 +251,9 @@ public class ConvertedModel { Set constantsReplacedByMacros, String constantName, Tensor constantValue) { - RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName); - if (macroOverridingConstant != null) { - TensorType macroType = macroOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); + RankProfile.RankingExpressionFunction rankingExpressionFunctionOverridingConstant = profile.getMacros().get(constantName); + if (rankingExpressionFunctionOverridingConstant != null) { + TensorType macroType = rankingExpressionFunctionOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); if ( ! macroType.equals(constantValue.type())) throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + typeMismatchExplanation(constantValue.type(), macroType)); @@ -301,8 +301,8 @@ public class ConvertedModel { TensorType requiredType = model.requiredMacros().get(macroName); if (requiredType == null) continue; // Not a required macro - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getMacros().get(macroName); + if (rankingExpressionFunction == null) throw new IllegalArgumentException("Model refers input '" + macroName + "' of type " + requiredType + " but this macro is not present in " + profile); @@ -310,7 +310,7 @@ public class ConvertedModel { // phase and summary features), as it may only resolve correctly given those bindings // Or, probably better, annotate the macros with type constraints here and verify during general // type verification - TensorType actualType = macro.function().getBody().getRoot().type(profile.typeContext(queryProfiles)); + TensorType actualType = rankingExpressionFunction.function().getBody().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) throw new IllegalArgumentException("Model refers input '" + macroName + "' of type " + requiredType + @@ -352,12 +352,12 @@ public class ConvertedModel { for (String macroName : macroNames) { if ( ! model.macros().containsKey(macroName)) continue; - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) { + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getMacros().get(macroName); + if (rankingExpressionFunction == null) { throw new IllegalArgumentException("Model refers to generated macro '" + macroName + "but this macro is not present in " + profile); } - RankingExpression macroExpression = macro.function().getBody(); + RankingExpression macroExpression = rankingExpressionFunction.function().getBody(); macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 132807bf8b8..76a7a01a3aa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -27,7 +27,7 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { new QueryProfileRegistry()).getSearch(); final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); macrosRankProfile.parseExpressions(); - final Map macros = macrosRankProfile.getMacros(); + final Map macros = macrosRankProfile.getMacros(); assertEquals(2, macros.get("titlematch$").function().arguments().size()); assertEquals("var1", macros.get("titlematch$").function().arguments().get(0)); assertEquals("var2", macros.get("titlematch$").function().arguments().get(1)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java deleted file mode 100644 index 8a07e99101c..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.parser.ParseException; -import org.junit.Test; - -import java.util.logging.Level; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author lesters - */ -public class ReservedMacroNamesTestCase { - - @Test - public void requireThatMacrosWithReservedNamesIssueAWarning() throws ParseException { - TestDeployLogger deployLogger = new TestDeployLogger(); - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SearchBuilder builder = new SearchBuilder(rankProfileRegistry); - builder.importString( - "search test {\n" + - " document test { \n" + - " field a type string { \n" + - " indexing: index \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile test_rank_profile {\n" + - " macro not_a_reserved_name(x) {\n" + - " expression: x + x\n" + - " }\n" + - " macro sigmoid(x) {\n" + - " expression: x * x\n" + - " }\n" + - " first-phase {\n" + - " expression: sigmoid(2) + not_a_reserved_name(1)\n" + - " }\n" + - " }\n" + - " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + - " macro sin(x) {\n" + - " expression: x * x\n" + - " }\n" + - " first-phase {\n" + - " expression: sigmoid(2) + sin(1)\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true, deployLogger); - - assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile")); - assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile_2")); - assertTrue(deployLogger.log.contains("sin") && deployLogger.log.contains("test_rank_profile_2")); - assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile")); - assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile_2")); - - } - - public static class TestDeployLogger implements DeployLogger { - public String log = ""; - @Override - public void log(Level level, String message) { - log += message; - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java new file mode 100644 index 00000000000..56639e6b9a8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -0,0 +1,72 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.util.logging.Level; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lesters + */ +public class ReservedRankingExpressionFunctionNamesTestCase { + + @Test + public void requireThatMacrosWithReservedNamesIssueAWarning() throws ParseException { + TestDeployLogger deployLogger = new TestDeployLogger(); + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importString( + "search test {\n" + + " document test { \n" + + " field a type string { \n" + + " indexing: index \n" + + " }\n" + + " }\n" + + " \n" + + " rank-profile test_rank_profile {\n" + + " macro not_a_reserved_name(x) {\n" + + " expression: x + x\n" + + " }\n" + + " macro sigmoid(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: sigmoid(2) + not_a_reserved_name(1)\n" + + " }\n" + + " }\n" + + " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + + " macro sin(x) {\n" + + " expression: x * x\n" + + " }\n" + + " first-phase {\n" + + " expression: sigmoid(2) + sin(1)\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true, deployLogger); + + assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile")); + assertTrue(deployLogger.log.contains("sigmoid") && deployLogger.log.contains("test_rank_profile_2")); + assertTrue(deployLogger.log.contains("sin") && deployLogger.log.contains("test_rank_profile_2")); + assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile")); + assertFalse(deployLogger.log.contains("not_a_reserved_name") && deployLogger.log.contains("test_rank_profile_2")); + + } + + public static class TestDeployLogger implements DeployLogger { + public String log = ""; + @Override + public void log(Level level, String message) { + log += message; + } + } + +} -- cgit v1.2.3 From 493adba229cc53c5766b0b63060562d6256abdd8 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 11:23:38 +0200 Subject: Refactor: macro -> function --- .../searchdefinition/MapEvaluationTypeContext.java | 2 +- .../com/yahoo/searchdefinition/RankProfile.java | 104 +++++------ .../searchdefinition/derived/RawRankProfile.java | 46 ++--- .../expressiontransforms/ExpressionTransforms.java | 4 +- .../expressiontransforms/FunctionInliner.java | 32 ++++ .../expressiontransforms/FunctionShadower.java | 59 +++++++ .../expressiontransforms/MacroInliner.java | 32 ---- .../expressiontransforms/MacroShadower.java | 59 ------- .../RankProfileTransformContext.java | 8 +- .../searchdefinition/processing/Processing.java | 2 +- .../processing/ReservedFunctionNames.java | 55 ++++++ .../processing/ReservedMacroNames.java | 55 ------ .../java/com/yahoo/vespa/model/VespaModel.java | 4 +- .../com/yahoo/vespa/model/ml/ConvertedModel.java | 190 ++++++++++----------- config-model/src/main/javacc/SDParser.jj | 6 +- .../searchdefinition/RankProfileRegistryTest.java | 6 +- .../RankingExpressionConstantsTestCase.java | 14 +- .../processing/RankProfileSearchFixture.java | 2 +- .../RankingExpressionWithOnnxTestCase.java | 4 +- .../RankingExpressionWithTensorFlowTestCase.java | 4 +- .../processing/RankingExpressionsTestCase.java | 2 +- .../integration/ml/ImportedModel.java | 4 +- .../integration/ml/DropoutImportTestCase.java | 6 +- .../ml/OnnxMnistSoftmaxImportTestCase.java | 6 +- .../ml/TensorFlowMnistSoftmaxImportTestCase.java | 8 +- .../integration/ml/TestableTensorFlowModel.java | 8 +- 26 files changed, 360 insertions(+), 362 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java create mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java create mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index e094dd0ebcd..0d9ea00bf73 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -70,7 +70,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement currentResolutionCallStack.stream().map(Reference::toString).collect(Collectors.joining(" -> ")) + " -> " + reference); - // A reference to a macro argument? + // A reference to a function argument? Optional binding = boundIdentifier(reference); if (binding.isPresent()) { try { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 509fa243bc3..30737365844 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -101,7 +101,7 @@ public class RankProfile implements Serializable, Cloneable { private String firstPhaseRankingString = null; - private Map macros= new LinkedHashMap<>(); + private Map functions = new LinkedHashMap<>(); private Set filterFields = new HashSet<>(); @@ -548,39 +548,39 @@ public class RankProfile implements Serializable, Cloneable { return null; } - /** Adds a new macro and returns it */ - public RankingExpressionFunction addMacro(String name, List arguments, String expression, boolean inline) { + /** Adds a function and returns it */ + public RankingExpressionFunction addFunction(String name, List arguments, String expression, boolean inline) { try { - return addMacro(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + return addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); } catch (ParseException e) { - throw new IllegalArgumentException("Could not parse macro '" + name + "'", e); + throw new IllegalArgumentException("Could not parse function '" + name + "'", e); } } - /** Adds a new macro and returns it */ - public RankingExpressionFunction addMacro(ExpressionFunction function, boolean inline) { + /** Adds a function and returns it */ + public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); - macros.put(function.getName(), rankingExpressionFunction); + functions.put(function.getName(), rankingExpressionFunction); return rankingExpressionFunction; } - /** Returns an unmodifiable view of the macros in this */ - public Map getMacros() { - if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap(); - if (macros.size() == 0) return getInherited().getMacros(); - if (getInherited() == null) return Collections.unmodifiableMap(macros); + /** Returns an unmodifiable view of the functions in this */ + public Map getFunctions() { + if (functions.size() == 0 && getInherited() == null) return Collections.emptyMap(); + if (functions.size() == 0) return getInherited().getFunctions(); + if (getInherited() == null) return Collections.unmodifiableMap(functions); // Neither is null - Map allMacros = new LinkedHashMap<>(getInherited().getMacros()); - allMacros.putAll(macros); - return Collections.unmodifiableMap(allMacros); + Map allFunctions = new LinkedHashMap<>(getInherited().getFunctions()); + allFunctions.putAll(functions); + return Collections.unmodifiableMap(allFunctions); } public int getKeepRankCount() { - if (keepRankCount>=0) return keepRankCount; - if (getInherited()!=null) return getInherited().getKeepRankCount(); + if (keepRankCount >= 0) return keepRankCount; + if (getInherited() != null) return getInherited().getKeepRankCount(); return -1; } @@ -679,14 +679,13 @@ public class RankProfile implements Serializable, Cloneable { @Override public RankProfile clone() { try { - // Note: This treats RankingExpression in Macros as immutables even though they are not RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; clone.rankProperties = new LinkedHashMap<>(this.rankProperties); - clone.macros = new LinkedHashMap<>(this.macros); + clone.functions = new LinkedHashMap<>(this.functions); clone.filterFields = new HashSet<>(this.filterFields); clone.constants = new HashMap<>(this.constants); return clone; @@ -713,58 +712,59 @@ public class RankProfile implements Serializable, Cloneable { private void compileThis(QueryProfileRegistry queryProfiles, ImportedModels importedModels) { parseExpressions(); - checkNameCollisions(getMacros(), getConstants()); + checkNameCollisions(getFunctions(), getConstants()); ExpressionTransforms expressionTransforms = new ExpressionTransforms(); - // Macro compiling first pass: compile inline macros without resolving other macros - Map inlineMacros = compileMacros(getInlineMacros(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); + // Function compiling first pass: compile inline functions without resolving other functions + Map inlineFunctions = + compileFunctions(getInlineFunctions(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); - // Macro compiling second pass: compile all macros and insert previously compiled inline macros - macros = compileMacros(getMacros(), queryProfiles, importedModels, inlineMacros, expressionTransforms); + // Function compiling second pass: compile all functions and insert previously compiled inline functions + functions = compileFunctions(getFunctions(), queryProfiles, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); } - private void checkNameCollisions(Map macros, Map constants) { - for (Map.Entry macroEntry : macros.entrySet()) { - if (constants.get(macroEntry.getKey()) != null) - throw new IllegalArgumentException("Cannot have both a constant and macro named '" + - macroEntry.getKey() + "'"); + private void checkNameCollisions(Map functions, Map constants) { + for (Map.Entry functionEntry : functions.entrySet()) { + if (constants.get(functionEntry.getKey()) != null) + throw new IllegalArgumentException("Cannot have both a constant and function named '" + + functionEntry.getKey() + "'"); } } - private Map getInlineMacros() { - return getMacros().entrySet().stream().filter(x -> x.getValue().inline()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + private Map getInlineFunctions() { + return getFunctions().entrySet().stream().filter(x -> x.getValue().inline()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private Map compileMacros(Map macros, - QueryProfileRegistry queryProfiles, - ImportedModels importedModels, - Map inlineMacros, - ExpressionTransforms expressionTransforms) { - Map compiledMacros = new LinkedHashMap<>(); - for (Map.Entry entry : macros.entrySet()) { + private Map compileFunctions(Map functions, + QueryProfileRegistry queryProfiles, + ImportedModels importedModels, + Map inlineFunctions, + ExpressionTransforms expressionTransforms) { + Map compiledFunctions = new LinkedHashMap<>(); + for (Map.Entry entry : functions.entrySet()) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); - RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - compiledMacros.put(entry.getKey(), rankingExpressionFunction.withBody(compiled)); + RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withBody(compiled)); } - return compiledMacros; + return compiledFunctions; } private RankingExpression compile(RankingExpression expression, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map constants, - Map inlineMacros, + Map inlineFunctions, ExpressionTransforms expressionTransforms) { if (expression == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, importedModels, constants, - inlineMacros); + inlineFunctions); expression = expressionTransforms.transform(expression, context); for (Map.Entry rankProperty : context.rankProperties().entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); @@ -777,9 +777,9 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public TypeContext typeContext(QueryProfileRegistry queryProfiles) { - MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() - .map(RankingExpressionFunction::function) - .collect(Collectors.toList())); + MapEvaluationTypeContext context = new MapEvaluationTypeContext(getFunctions().values().stream() + .map(RankingExpressionFunction::function) + .collect(Collectors.toList())); // Add small and large constants, respectively getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); @@ -950,7 +950,7 @@ public class RankProfile implements Serializable, Cloneable { private final ExpressionFunction function; - /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ + /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ private final boolean inline; public RankingExpressionFunction(ExpressionFunction function, boolean inline) { @@ -961,7 +961,7 @@ public class RankProfile implements Serializable, Cloneable { public ExpressionFunction function() { return function; } public boolean inline() { - return inline && function.arguments().isEmpty(); // only inline no-arg macros; + return inline && function.arguments().isEmpty(); // only inline no-arg functions; } public RankingExpressionFunction withBody(RankingExpression expression) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 98d147aca79..c041d5c6a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -175,57 +175,57 @@ public class RawRankProfile implements RankProfilesConfig.Producer { rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros()); + derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); } - private void derivePropertiesAndSummaryFeaturesFromMacros(Map macros) { - if (macros.isEmpty()) return; - Map expressionMacros = new LinkedHashMap<>(); - for (Map.Entry macro : macros.entrySet()) { - expressionMacros.put(macro.getKey(), macro.getValue().function()); + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map functions) { + if (functions.isEmpty()) return; + Map expressionFunctions = new LinkedHashMap<>(); + for (Map.Entry function : functions.entrySet()) { + expressionFunctions.put(function.getKey(), function.getValue().function()); } - Map macroProperties = new LinkedHashMap<>(); - macroProperties.putAll(deriveMacroProperties(expressionMacros)); + Map functionProperties = new LinkedHashMap<>(); + functionProperties.putAll(deriveFunctionProperties(expressionFunctions)); if (firstPhaseRanking != null) { - macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } if (secondPhaseRanking != null) { - macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } - for (Map.Entry e : macroProperties.entrySet()) { + for (Map.Entry e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties); - replaceMacroSummaryFeatures(context); + SerializationContext context = new SerializationContext(expressionFunctions.values(), null, functionProperties); + replaceFunctionSummaryFeatures(context); } - private Map deriveMacroProperties(Map eMacros) { - SerializationContext context = new SerializationContext(eMacros); - for (Map.Entry e : eMacros.entrySet()) { + private Map deriveFunctionProperties(Map functions) { + SerializationContext context = new SerializationContext(functions); + for (Map.Entry e : functions.entrySet()) { String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression); } return context.serializedFunctions(); } - private void replaceMacroSummaryFeatures(SerializationContext context) { + private void replaceFunctionSummaryFeatures(SerializationContext context) { if (summaryFeatures == null) return; - Map macroSummaryFeatures = new LinkedHashMap<>(); + Map functionSummaryFeatures = new LinkedHashMap<>(); for (Iterator i = summaryFeatures.iterator(); i.hasNext(); ) { ReferenceNode referenceNode = i.next(); - // Is the feature a macro? + // Is the feature a function? if (context.getFunction(referenceNode.getName()) != null) { context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), referenceNode.toString(new StringBuilder(), context, null, null).toString()); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); - macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); + functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block } } - // Then, replace the summary features that were macros - for (Map.Entry e : macroSummaryFeatures.entrySet()) { + // Then, replace the summary features that were functions + for (Map.Entry e : functionSummaryFeatures.entrySet()) { summaryFeatures.add(e.getValue()); } } @@ -295,7 +295,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { List> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) { - // Could have been set by macro expansion. Set expressions, then skip this property. + // Could have been set by function expansion. Set expressions, then skip this property. try { firstPhaseRanking = new RankingExpression(property.getValue()); } catch (ParseException e) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index a639165d297..cbabfffb7a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -28,8 +28,8 @@ public class ExpressionTransforms { new XgboostFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), - new MacroInliner(), - new MacroShadower(), + new FunctionInliner(), + new FunctionShadower(), new TensorTransformer(), new Simplifier()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java new file mode 100644 index 00000000000..c15ef20a455 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.expressiontransforms; + +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; + +/** + * Inlines functions in ranking expressions + * + * @author bratseth + */ +public class FunctionInliner extends ExpressionTransformer { + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof ReferenceNode) + return transformFeatureNode((ReferenceNode)node, context); + if (node instanceof CompositeNode) + return transformChildren((CompositeNode)node, context); + return node; + } + + private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) { + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineFunctions().get(feature.getName()); + if (rankingExpressionFunction == null) return feature; + return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java new file mode 100644 index 00000000000..74b6471d291 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.expressiontransforms; + +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; + +/** + * Transforms function nodes to reference nodes if a rank profile function shadows a built-in function. + * This has the effect of allowing rank profile functions to redefine built-in functions. + * Another effect is that we can add built-in functions over time + * without fear of breaking existing users' functions with the same name. + * + * However, there is a (largish) caveat. If a user has a function with a certain number + * of arguments, and we add in a built-in function with a different arity, + * this will cause parse errors as the Java parser gives precedence to + * built-in functions. + * + * @author lesters + */ +public class FunctionShadower extends ExpressionTransformer { + + @Override + public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { + String name = expression.getName(); + ExpressionNode node = expression.getRoot(); + ExpressionNode result = transform(node, context); + return new RankingExpression(name, result); + } + + @Override + public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { + if (node instanceof FunctionNode) + return transformFunctionNode((FunctionNode) node, context); + if (node instanceof CompositeNode) + return transformChildren((CompositeNode)node, context); + return node; + } + + private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) { + String name = function.getFunction().toString(); + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().getFunctions().get(name); + if (rankingExpressionFunction == null) { + return transformChildren(function, context); + } + + int functionArity = function.getFunction().arity(); + if (functionArity != rankingExpressionFunction.function().arguments().size()) + return transformChildren(function, context); + + ReferenceNode node = new ReferenceNode(name, function.children(), null); + return transformChildren(node, context); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java deleted file mode 100644 index 155568825e0..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; - -/** - * Inlines macros in ranking expressions - * - * @author bratseth - */ -public class MacroInliner extends ExpressionTransformer { - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof ReferenceNode) - return transformFeatureNode((ReferenceNode)node, context); - if (node instanceof CompositeNode) - return transformChildren((CompositeNode)node, context); - return node; - } - - private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) { - RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineMacros().get(feature.getName()); - if (rankingExpressionFunction == null) return feature; - return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java deleted file mode 100644 index 62315257c2c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; - -/** - * Transforms function nodes to reference nodes if a macro shadows a built-in function. - * This has the effect of allowing macros to redefine built-in functions. - * Another effect is that we can more or less add built-in functions over time - * without fear of breaking existing users' macros with the same name. - * - * However, there is a (largish) caveat. If a user has a macro with a certain number - * of arguments, and we add in a built-in function with a different arity, - * this will cause parse errors as the Java parser gives precedence to - * built-in functions. - * - * @author lesters - */ -public class MacroShadower extends ExpressionTransformer { - - @Override - public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { - String name = expression.getName(); - ExpressionNode node = expression.getRoot(); - ExpressionNode result = transform(node, context); - return new RankingExpression(name, result); - } - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof FunctionNode) - return transformFunctionNode((FunctionNode) node, context); - if (node instanceof CompositeNode) - return transformChildren((CompositeNode)node, context); - return node; - } - - private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) { - String name = function.getFunction().toString(); - RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().getMacros().get(name); - if (rankingExpressionFunction == null) { - return transformChildren(function, context); - } - - int functionArity = function.getFunction().arity(); - if (functionArity != rankingExpressionFunction.function().arguments().size()) - return transformChildren(function, context); - - ReferenceNode node = new ReferenceNode(name, function.children(), null); - return transformChildren(node, context); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index f45ef359338..2fe2dacf2ce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -20,25 +20,25 @@ public class RankProfileTransformContext extends TransformContext { private final RankProfile rankProfile; private final QueryProfileRegistry queryProfiles; private final ImportedModels importedModels; - private final Map inlineMacros; + private final Map inlineFunctions; private final Map rankProperties = new HashMap<>(); public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map constants, - Map inlineMacros) { + Map inlineFunctions) { super(constants); this.rankProfile = rankProfile; this.queryProfiles = queryProfiles; this.importedModels = importedModels; - this.inlineMacros = inlineMacros; + this.inlineFunctions = inlineFunctions; } public RankProfile rankProfile() { return rankProfile; } public QueryProfileRegistry queryProfiles() { return queryProfiles; } public ImportedModels importedModels() { return importedModels; } - public Map inlineMacros() { return inlineMacros; } + public Map inlineFunctions() { return inlineFunctions; } public Map rankProperties() { return rankProperties; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 19025d37f8c..fdef93e21ed 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -74,7 +74,7 @@ public class Processing { RankProfileTypeSettingsProcessor::new, ReferenceFieldsProcessor::new, FastAccessValidator::new, - ReservedMacroNames::new, + ReservedFunctionNames::new, RankingExpressionTypeValidator::new, // These should be last. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java new file mode 100644 index 00000000000..8aa8ff56756 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.google.common.collect.ImmutableSet; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConstants; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Set; +import java.util.logging.Level; + +/** + * Issues a warning if some function has a reserved name. This is not necessarily + * an error, as a rank profile function can shadow a built-in function. + * + * @author lesters + */ +public class ReservedFunctionNames extends Processor { + + private static Set reservedNames = getReservedNames(); + + public ReservedFunctionNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate) { + if ( ! validate) return; + + for (RankProfile rp : rankProfileRegistry.all()) { + for (String functionName : rp.getFunctions().keySet()) { + if (reservedNames.contains(functionName)) { + deployLogger.log(Level.WARNING, "Funcion '" + functionName + "' " + + "in rank profile '" + rp.getName() + "' " + + "has a reserved name. This might mean that the function shadows " + + "the built-in function with the same name." + ); + } + } + } + } + + private static ImmutableSet getReservedNames() { + ImmutableSet.Builder names = ImmutableSet.builder(); + for (String token : RankingExpressionParserConstants.tokenImage) { + String tokenWithoutQuotes = token.substring(1, token.length()-1); + names.add(tokenWithoutQuotes); + } + return names.build(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java deleted file mode 100644 index adcebed9254..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.google.common.collect.ImmutableSet; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Search; -import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConstants; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.Set; -import java.util.logging.Level; - -/** - * Issues a warning if some macro has a reserved name. This is not necessarily - * an error, as a macro can shadow a built-in function. - * - * @author lesters - */ -public class ReservedMacroNames extends Processor { - - private static Set reservedNames = getReservedNames(); - - public ReservedMacroNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate) { - if ( ! validate) return; - - for (RankProfile rp : rankProfileRegistry.all()) { - for (String macroName : rp.getMacros().keySet()) { - if (reservedNames.contains(macroName)) { - deployLogger.log(Level.WARNING, "Macro \"" + macroName + "\" " + - "in rank profile \"" + rp.getName() + "\" " + - "has a reserved name. This might mean that the macro shadows " + - "the built-in function with the same name." - ); - } - } - } - } - - private static ImmutableSet getReservedNames() { - ImmutableSet.Builder names = ImmutableSet.builder(); - for (String token : RankingExpressionParserConstants.tokenImage) { - String tokenWithoutQuotes = token.substring(1, token.length()-1); - names.add(tokenWithoutQuotes); - } - return names.build(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index bc2b2f70eae..4b70b1b5ae2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -237,7 +237,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), model.name(), profile, queryProfiles, model); for (Map.Entry entry : convertedModel.expressions().entrySet()) { - profile.addMacro(new ExpressionFunction(entry.getKey(), entry.getValue()), false); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } @@ -249,7 +249,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); for (Map.Entry entry : convertedModel.expressions().entrySet()) { - profile.addMacro(new ExpressionFunction(entry.getKey(), entry.getValue()), false); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index ffc3c53fdb6..adf5c81283e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -140,7 +140,7 @@ public class ConvertedModel { public ExpressionNode expression(FeatureArguments arguments, RankProfileTransformContext context) { RankingExpression expression = selectExpression(arguments); if (sourceModel.isPresent()) // we can verify - verifyRequiredMacros(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); + verifyRequiredFunctions(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); return expression.getRoot(); } @@ -184,41 +184,41 @@ public class ConvertedModel { QueryProfileRegistry queryProfiles, ModelStore store) { // Add constants - Set constantsReplacedByMacros = new HashSet<>(); + Set constantsReplacedByFunctions = new HashSet<>(); model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, queryProfiles, - constantsReplacedByMacros, k, v)); + constantsReplacedByFunctions, k, v)); - // Add macros - addGeneratedMacros(model, profile); + // Add functions + addGeneratedFunctions(model, profile); // Add expressions Map expressions = new HashMap<>(); for (Pair output : model.outputExpressions()) { addExpression(output.getSecond(), output.getFirst(), - constantsReplacedByMacros, + constantsReplacedByFunctions, model, store, profile, queryProfiles, expressions); } - // Transform and save macro - must come after reading expressions due to optimization transforms - // and must use the macro expression added to the profile, which may differ from the one saved in the model, + // Transform and save function - must come after reading expressions due to optimization transforms + // and must use the function expression added to the profile, which may differ from the one saved in the model, // after rewrite - model.macros().forEach((k, v) -> transformGeneratedMacro(store, constantsReplacedByMacros, k, - profile.getMacros().get(k).function().getBody())); + model.functions().forEach((k, v) -> transformGeneratedFunction(store, constantsReplacedByFunctions, k, + profile.getFunctions().get(k).function().getBody())); return expressions; } private static void addExpression(RankingExpression expression, String expressionName, - Set constantsReplacedByMacros, + Set constantsReplacedByFunctions, ImportedModel model, ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, Map expressions) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); reduceBatchDimensions(expression, model, profile, queryProfiles); store.writeExpression(expressionName, expression); expressions.put(expressionName, expression); @@ -233,8 +233,8 @@ public class ConvertedModel { profile.rankingConstants().add(constant); } - for (Pair macro : store.readMacros()) { - addGeneratedMacroToProfile(profile, macro.getFirst(), macro.getSecond()); + for (Pair function : store.readFunctions()) { + addGeneratedFunctionToProfile(profile, function.getFirst(), function.getSecond()); } return store.readExpressions(); @@ -248,16 +248,16 @@ public class ConvertedModel { private static void transformLargeConstant(ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, - Set constantsReplacedByMacros, + Set constantsReplacedByFunctions, String constantName, Tensor constantValue) { - RankProfile.RankingExpressionFunction rankingExpressionFunctionOverridingConstant = profile.getMacros().get(constantName); + RankProfile.RankingExpressionFunction rankingExpressionFunctionOverridingConstant = profile.getFunctions().get(constantName); if (rankingExpressionFunctionOverridingConstant != null) { - TensorType macroType = rankingExpressionFunctionOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); - if ( ! macroType.equals(constantValue.type())) - throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + - typeMismatchExplanation(constantValue.type(), macroType)); - constantsReplacedByMacros.add(constantName); // will replace constant(constantName) by constantName later + TensorType functionType = rankingExpressionFunctionOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); + if ( ! functionType.equals(constantValue.type())) + throw new IllegalArgumentException("Function '" + constantName + "' replaces the constant with this name. " + + typeMismatchExplanation(constantValue.type(), functionType)); + constantsReplacedByFunctions.add(constantName); // will replace constant(constantName) by constantName later } else { Path constantPath = store.writeLargeConstant(constantName, constantValue); @@ -268,77 +268,75 @@ public class ConvertedModel { } } - private static void transformGeneratedMacro(ModelStore store, - Set constantsReplacedByMacros, - String macroName, - RankingExpression expression) { + private static void transformGeneratedFunction(ModelStore store, + Set constantsReplacedByFunctions, + String functionName, + RankingExpression expression) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); - store.writeMacro(macroName, expression); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); + store.writeFunction(functionName, expression); } - private static void addGeneratedMacroToProfile(RankProfile profile, String macroName, RankingExpression expression) { - if (profile.getMacros().containsKey(macroName)) { - if ( ! profile.getMacros().get(macroName).function().getBody().equals(expression)) - throw new IllegalArgumentException("Generated macro '" + macroName + "' already exists in " + profile + + private static void addGeneratedFunctionToProfile(RankProfile profile, String functionName, RankingExpression expression) { + if (profile.getFunctions().containsKey(functionName)) { + if ( ! profile.getFunctions().get(functionName).function().getBody().equals(expression)) + throw new IllegalArgumentException("Generated function '" + functionName + "' already exists in " + profile + " - with a different definition" + - ": Has\n" + profile.getMacros().get(macroName).function().getBody() + + ": Has\n" + profile.getFunctions().get(functionName).function().getBody() + "\nwant to add " + expression + "\n"); return; } - profile.addMacro(new ExpressionFunction(macroName, expression), false); // TODO: Inline if only used once + profile.addFunction(new ExpressionFunction(functionName, expression), false); // TODO: Inline if only used once } /** - * Verify that the macros referred in the given expression exists in the given rank profile, - * and return tensors of the types specified in requiredMacros. + * Verify that the functions referred in the given expression exists in the given rank profile, + * and return tensors of the types specified in requiredFunctions. */ - private static void verifyRequiredMacros(RankingExpression expression, ImportedModel model, - RankProfile profile, QueryProfileRegistry queryProfiles) { - Set macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - TensorType requiredType = model.requiredMacros().get(macroName); - if (requiredType == null) continue; // Not a required macro - - RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getMacros().get(macroName); + private static void verifyRequiredFunctions(RankingExpression expression, ImportedModel model, + RankProfile profile, QueryProfileRegistry queryProfiles) { + Set functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + TensorType requiredType = model.requiredFunctions().get(functionName); + if (requiredType == null) continue; // Not a required function + + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); if (rankingExpressionFunction == null) - throw new IllegalArgumentException("Model refers input '" + macroName + - "' of type " + requiredType + " but this macro is not present in " + + throw new IllegalArgumentException("Model refers input '" + functionName + + "' of type " + requiredType + " but this function is not present in " + profile); // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second // phase and summary features), as it may only resolve correctly given those bindings - // Or, probably better, annotate the macros with type constraints here and verify during general + // Or, probably better, annotate the functions with type constraints here and verify during general // type verification TensorType actualType = rankingExpressionFunction.function().getBody().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) - throw new IllegalArgumentException("Model refers input '" + macroName + + throw new IllegalArgumentException("Model refers input '" + functionName + "' of type " + requiredType + - " which must be produced by a macro in the rank profile, but " + - "this macro references a feature which is not declared"); + " which must be produced by a function in the rank profile, but " + + "this function references a feature which is not declared"); if ( ! actualType.isAssignableTo(requiredType)) - throw new IllegalArgumentException("Model refers input '" + macroName + "'. " + + throw new IllegalArgumentException("Model refers input '" + functionName + "'. " + typeMismatchExplanation(requiredType, actualType)); } } private static String typeMismatchExplanation(TensorType requiredType, TensorType actualType) { - return "The required type of this is " + requiredType + ", but this macro returns " + actualType + + return "The required type of this is " + requiredType + ", but this function returns " + actualType + (actualType.rank() == 0 ? ". This is often due to missing declaration of query tensor features " + "in query profile types - see the documentation." : ""); } - /** - * Add the generated macros to the rank profile - */ - private static void addGeneratedMacros(ImportedModel model, RankProfile profile) { - model.macros().forEach((k, v) -> addGeneratedMacroToProfile(profile, k, v.copy())); + /** Add the generated functions to the rank profile */ + private static void addGeneratedFunctions(ImportedModel model, RankProfile profile) { + model.functions().forEach((k, v) -> addGeneratedFunctionToProfile(profile, k, v.copy())); } /** * Check if batch dimensions of inputs can be reduced out. If the input - * macro specifies that a single exemplar should be evaluated, we can + * function specifies that a single exemplar should be evaluated, we can * reduce the batch dimension out. */ private static void reduceBatchDimensions(RankingExpression expression, ImportedModel model, @@ -346,19 +344,19 @@ public class ConvertedModel { TypeContext typeContext = profile.typeContext(queryProfiles); TensorType typeBeforeReducing = expression.getRoot().type(typeContext); - // Check generated macros for inputs to reduce - Set macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - if ( ! model.macros().containsKey(macroName)) continue; + // Check generated functions for inputs to reduce + Set functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + if ( ! model.functions().containsKey(functionName)) continue; - RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getMacros().get(macroName); + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); if (rankingExpressionFunction == null) { - throw new IllegalArgumentException("Model refers to generated macro '" + macroName + - "but this macro is not present in " + profile); + throw new IllegalArgumentException("Model refers to generated function '" + functionName + + "but this function is not present in " + profile); } - RankingExpression macroExpression = rankingExpressionFunction.function().getBody(); - macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext)); + RankingExpression functionExpression = rankingExpressionFunction.function().getBody(); + functionExpression.setRoot(reduceBatchDimensionsAtInput(functionExpression.getRoot(), model, typeContext)); } // Check expression for inputs to reduce @@ -377,7 +375,7 @@ public class ConvertedModel { List children = ((TensorFunctionNode)node).children(); if (children.size() == 1 && children.get(0) instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) children.get(0); - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(tensorFunction, typeContext); } } @@ -385,7 +383,7 @@ public class ConvertedModel { } if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) node; - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext); } } @@ -446,47 +444,47 @@ public class ConvertedModel { } /** - * If a constant c is overridden by a macro, we need to replace instances of "constant(c)" by "c" in expressions. + * If a constant c is overridden by a function, we need to replace instances of "constant(c)" by "c" in expressions. * This method does that for the given expression and returns the result. */ - private static RankingExpression replaceConstantsByMacros(RankingExpression expression, - Set constantsReplacedByMacros) { - if (constantsReplacedByMacros.isEmpty()) return expression; + private static RankingExpression replaceConstantsByFunctions(RankingExpression expression, + Set constantsReplacedByFunctions) { + if (constantsReplacedByFunctions.isEmpty()) return expression; return new RankingExpression(expression.getName(), - replaceConstantsByMacros(expression.getRoot(), constantsReplacedByMacros)); + replaceConstantsByFunctions(expression.getRoot(), constantsReplacedByFunctions)); } - private static ExpressionNode replaceConstantsByMacros(ExpressionNode node, Set constantsReplacedByMacros) { + private static ExpressionNode replaceConstantsByFunctions(ExpressionNode node, Set constantsReplacedByFunctions) { if (node instanceof ReferenceNode) { Reference reference = ((ReferenceNode)node).reference(); if (FeatureNames.isSimpleFeature(reference) && reference.name().equals("constant")) { String argument = reference.simpleArgument().get(); - if (constantsReplacedByMacros.contains(argument)) + if (constantsReplacedByFunctions.contains(argument)) return new ReferenceNode(argument); } } if (node instanceof CompositeNode) { // not else: this matches some of the same nodes as the outer if above CompositeNode composite = (CompositeNode)node; return composite.setChildren(composite.children().stream() - .map(child -> replaceConstantsByMacros(child, constantsReplacedByMacros)) + .map(child -> replaceConstantsByFunctions(child, constantsReplacedByFunctions)) .collect(Collectors.toList())); } return node; } - private static void addMacroNamesIn(ExpressionNode node, Set names, ImportedModel model) { + private static void addFunctionNamesIn(ExpressionNode node, Set names, ImportedModel model) { if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode)node; - if (referenceNode.getOutput() == null) { // macro references cannot specify outputs + if (referenceNode.getOutput() == null) { // function references cannot specify outputs names.add(referenceNode.getName()); - if (model.macros().containsKey(referenceNode.getName())) { - addMacroNamesIn(model.macros().get(referenceNode.getName()).getRoot(), names, model); + if (model.functions().containsKey(referenceNode.getName())) { + addFunctionNamesIn(model.functions().get(referenceNode.getName()).getRoot(), names, model); } } } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode)node).children()) - addMacroNamesIn(child, names, model); + addFunctionNamesIn(child, names, model); } } @@ -550,19 +548,19 @@ public class ConvertedModel { return expressions; } - /** Adds this macro expression to the application package so it can be read later. */ - void writeMacro(String name, RankingExpression expression) { - application.getFile(modelFiles.macrosPath()).appendFile(name + "\t" + - expression.getRoot().toString() + "\n"); + /** Adds this function expression to the application package so it can be read later. */ + void writeFunction(String name, RankingExpression expression) { + application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" + + expression.getRoot().toString() + "\n"); } - /** Reads the previously stored macro expressions for these arguments */ - List> readMacros() { + /** Reads the previously stored function expressions for these arguments */ + List> readFunctions() { try { - ApplicationFile file = application.getFile(modelFiles.macrosPath()); + ApplicationFile file = application.getFile(modelFiles.functionsPath()); if ( ! file.exists()) return Collections.emptyList(); - List> macros = new ArrayList<>(); + List> functions = new ArrayList<>(); BufferedReader reader = new BufferedReader(file.createReader()); String line; while (null != (line = reader.readLine())) { @@ -570,13 +568,13 @@ public class ConvertedModel { String name = parts[0]; try { RankingExpression expression = new RankingExpression(parts[0], parts[1]); - macros.add(new Pair<>(name, expression)); + functions.add(new Pair<>(name, expression)); } catch (ParseException e) { throw new IllegalStateException("Could not parse " + name, e); } } - return macros; + return functions; } catch (IOException e) { throw new UncheckedIOException(e); @@ -724,9 +722,9 @@ public class ConvertedModel { return storedModelReplicatedPath().append("constants"); } - /** Path to the macros file */ - public Path macrosPath() { - return storedModelReplicatedPath().append("macros.txt"); + /** Path to the functions file */ + public Path functionsPath() { + return storedModelReplicatedPath().append("functions.txt"); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 2c60ab3771b..21edbb7eb58 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -1887,7 +1887,7 @@ Object rankProfileItem(RankProfile profile) : { } | fieldRankFilter(profile) | firstPhase(profile) | matchPhase(profile) - | macro(profile) + | function(profile) | ignoreRankFeatures(profile) | numThreadsPerSearch(profile) | minHitsPerThread(profile) @@ -1920,7 +1920,7 @@ void inheritsRankProfile(RankProfile profile) : * * @param profile The profile to modify. */ -void macro(RankProfile profile) : +void function(RankProfile profile) : { String name, expression, parameter; List parameters = new ArrayList(); @@ -1933,7 +1933,7 @@ void macro(RankProfile profile) : ( parameter = identifier() { parameters.add(parameter); } )* ] ")" lbrace() expression = expression() ()* ) - { profile.addMacro(name, parameters, expression, inline); } + { profile.addFunction(name, parameters, expression, inline); } } boolean inline() : diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index 7b262bdc4d0..fda4bdeea47 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -45,11 +45,11 @@ public class RankProfileRegistryTest { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { - assertNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); + assertNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry); - rankProfileWithAddedMacro.addMacro(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); + rankProfileWithAddedMacro.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); rankProfileRegistry.add(rankProfileWithAddedMacro); - assertNotNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); + assertNotNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index bceafc8e415..6b08c013a07 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -76,7 +76,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedModels()); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); - assertEquals("foo: 14.0", child2.getMacros().get("foo").function().getBody().toString()); + assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString()); List> rankProperties = new RawRankProfile(child2, queryProfileRegistry, new ImportedModels(), @@ -114,7 +114,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase fail("Should have caused an exception"); } catch (IllegalArgumentException e) { - assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and macro named 'c'", + assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'", Exceptions.toMessageString(e)); } } @@ -142,7 +142,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -171,9 +171,9 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -196,7 +196,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 100000000.0", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").function().getBody().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } @Test @@ -222,7 +222,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").function().getBody().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 749a97bf12f..0075a781950 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -74,7 +74,7 @@ class RankProfileSearchFixture { public void assertMacro(String expexctedExpression, String macroName, String rankProfile) { assertEquals(expexctedExpression, - compiledRankProfile(rankProfile).getMacros().get(macroName).function().getBody().getRoot().toString()); + compiledRankProfile(rankProfile).getFunctions().get(macroName).function().getBody().getRoot().toString()); } public RankProfile compileRankProfile(String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index b046d60f948..23ca59edb1c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -145,7 +145,7 @@ public class RankingExpressionWithOnnxTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } @@ -163,7 +163,7 @@ public class RankingExpressionWithOnnxTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 14632a568ea..d8602d438a8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -177,7 +177,7 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } @@ -195,7 +195,7 @@ public class RankingExpressionWithTensorFlowTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 76a7a01a3aa..d4fd48cfaca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -27,7 +27,7 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { new QueryProfileRegistry()).getSearch(); final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); macrosRankProfile.parseExpressions(); - final Map macros = macrosRankProfile.getMacros(); + final Map macros = macrosRankProfile.getFunctions(); assertEquals(2, macros.get("titlematch$").function().arguments().size()); assertEquals("var1", macros.get("titlematch$").function().arguments().get(0)); assertEquals("var2", macros.get("titlematch$").function().arguments().get(1)); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java index ac5eefcc5b2..8653e05a7d2 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java @@ -80,10 +80,10 @@ public class ImportedModel { * Returns an immutable map of macros that are part of this model. * Note that the macros themselves are *not* copies and *not* immutable - they must be copied before modification. */ - public Map macros() { return Collections.unmodifiableMap(macros); } + public Map functions() { return Collections.unmodifiableMap(macros); } /** Returns an immutable map of the macros that must be provided by the environment running this model */ - public Map requiredMacros() { return Collections.unmodifiableMap(requiredMacros); } + public Map requiredFunctions() { return Collections.unmodifiableMap(requiredMacros); } /** Returns an immutable map of the signatures of this */ public Map signatures() { return Collections.unmodifiableMap(signatures); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java index a63c7346335..1c73f97e929 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java @@ -19,10 +19,10 @@ public class DropoutImportTestCase { TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/files/integration/tensorflow/dropout/saved"); // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("X")); + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("X")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("X")); + model.get().requiredFunctions().get("X")); ImportedModel.Signature signature = model.get().signature("serving_default"); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java index bcfc6ce0a04..6856434e2d0 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java @@ -37,10 +37,10 @@ public class OnnxMnistSoftmaxImportTestCase { assertEquals(10, constant1.size()); // Check required macros (inputs) - assertEquals(1, model.requiredMacros().size()); - assertTrue(model.requiredMacros().containsKey("Placeholder")); + assertEquals(1, model.requiredFunctions().size()); + assertTrue(model.requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.requiredMacros().get("Placeholder")); + model.requiredFunctions().get("Placeholder")); // Check outputs RankingExpression output = model.defaultSignature().outputExpression("add"); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java index dd6c8095e3c..97f4e938a4f 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java @@ -35,13 +35,13 @@ public class TensorFlowMnistSoftmaxImportTestCase { assertEquals(10, constant1.size()); // Check (provided) macros - assertEquals(0, model.get().macros().size()); + assertEquals(0, model.get().functions().size()); // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("Placeholder")); + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("Placeholder")); + model.get().requiredFunctions().get("Placeholder")); // Check signatures assertEquals(1, model.get().signatures().size()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java index 4de3aa5d635..e688a983d27 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java @@ -48,7 +48,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateMacro(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", @@ -62,7 +62,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateMacro(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", tfResult, vespaResult); @@ -98,7 +98,7 @@ public class TestableTensorFlowModel { private void evaluateMacro(Context context, ImportedModel model, String macroName) { if (!context.names().contains(macroName)) { - RankingExpression e = model.macros().get(macroName); + RankingExpression e = model.functions().get(macroName); evaluateMacroDependencies(context, model, e.getRoot()); context.put(macroName, new TensorValue(e.evaluate(context).asTensor())); } @@ -107,7 +107,7 @@ public class TestableTensorFlowModel { private void evaluateMacroDependencies(Context context, ImportedModel model, ExpressionNode node) { if (node instanceof ReferenceNode) { String name = node.toString(); - if (model.macros().containsKey(name)) { + if (model.functions().containsKey(name)) { evaluateMacro(context, model, name); } } -- cgit v1.2.3 From e9bf13c3cbeaa351c973281bc57767d97d8db04a Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 13:38:56 +0200 Subject: Refactor: Eagerly parse first and second phase --- .../com/yahoo/searchdefinition/RankProfile.java | 30 ++++++++++++---------- .../IncorrectRankingExpressionFileRefTestCase.java | 8 +++--- .../RankingExpressionValidationTestCase.java | 6 ++--- 3 files changed, 24 insertions(+), 20 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 30737365844..b687fac6c12 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -362,22 +362,24 @@ public class RankProfile implements Serializable, Cloneable { this.secondPhaseRanking = rankingExpression; } - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for second phase - */ - public void setSecondPhaseRankingString(String exp) { - this.secondPhaseRankingString = exp; + public void setFirstPhaseRankingString(String expression) { + try { + this.firstPhaseRanking = parseRankingExpression("firstphase", expression); + this.firstPhaseRankingString = expression; + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal first phase ranking function", e); + } } - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for first phase - */ - public void setFirstPhaseRankingString(String exp) { - this.firstPhaseRankingString = exp; + public void setSecondPhaseRankingString(String expression) { + try { + this.secondPhaseRanking = parseRankingExpression("secondphase", expression); + this.secondPhaseRankingString = expression; + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal second phase ranking function", e); + } } /** Returns a read-only view of the summary features to use in this profile. This is never null */ diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 03fa92f5cb9..07a36832094 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -5,10 +5,12 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.io.IOException; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,9 +29,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT new DerivedConfiguration(search, registry, new QueryProfileRegistry(), new ImportedModels()); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { - e.printStackTrace(); - assertTrue(e.getCause().getMessage().contains("Could not read ranking expression file")); - assertTrue(e.getCause().getMessage().contains("wrongending.expr.expression")); + String message = Exceptions.toMessageString(e); + assertTrue(message.contains("Could not read ranking expression file")); + assertTrue(message.contains("wrongending.expr.expression")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 3fe3a7c3de1..5e649c2e551 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -5,9 +5,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Ignore; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -29,9 +31,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success - // TODO: Where's the "com.yahoo.searchdefinition.parser.ParseException:" nonsense coming from? - assertTrue("Got unexpected error message: " + e.getCause().getMessage(), - e.getCause().getMessage().startsWith("com.yahoo.searchdefinition.parser.ParseException: Could not parse ranking expression '" + expression + "'")); + assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:")); } } -- cgit v1.2.3 From 66058ea16d4ce72ec3a57c6c1b2e14a232d3899f Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 13:41:51 +0200 Subject: Refactor: Don't reparse --- .../com/yahoo/searchdefinition/RankProfile.java | 27 ---------------------- .../processing/RankingExpressionTypeValidator.java | 1 - .../RankingExpressionConstantsTestCase.java | 2 -- .../processing/RankingExpressionsTestCase.java | 1 - 4 files changed, 31 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b687fac6c12..25120c4dc76 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -619,32 +619,6 @@ public class RankProfile implements Serializable, Cloneable { return retval; } - /** - * Will take the parser-set textual ranking expressions and turn into ranking expression objects, - * if not already done - */ - // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions - // until this is called. Simplify by parsing them right away. - public void parseExpressions() { - try { - parseRankingExpressions(); - } catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - /** - * Passes ranking expressions on to parser - * - * @throws ParseException if either of the ranking expressions could not be parsed - */ - private void parseRankingExpressions() throws ParseException { - if (getFirstPhaseRankingString() != null && firstPhaseRanking == null) - setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString())); - if (getSecondPhaseRankingString() != null && secondPhaseRanking == null) - setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); - } - private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { if (expression.trim().length() == 0) throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); @@ -713,7 +687,6 @@ public class RankProfile implements Serializable, Cloneable { } private void compileThis(QueryProfileRegistry queryProfiles, ImportedModels importedModels) { - parseExpressions(); checkNameCollisions(getFunctions(), getConstants()); ExpressionTransforms expressionTransforms = new ExpressionTransforms(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index 81455991cc9..b3fe8ec36f6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -48,7 +48,6 @@ public class RankingExpressionTypeValidator extends Processor { /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */ private void validate(RankProfile profile) { - profile.parseExpressions(); TypeContext context = profile.typeContext(queryProfiles); profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 45d2357d742..150469cc928 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -141,7 +141,6 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @@ -170,7 +169,6 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 1b434b71c5c..bf13807434e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -26,7 +26,6 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { rankProfileRegistry, new QueryProfileRegistry()).getSearch(); RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros"); - functionsRankProfile.parseExpressions(); Map functions = functionsRankProfile.getFunctions(); assertEquals(2, functions.get("titlematch$").function().arguments().size()); assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); -- cgit v1.2.3 From 7c6537f8622663401693bcd80c162535b1fb1735 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 13:50:29 +0200 Subject: Don't store expression strings --- .../com/yahoo/searchdefinition/RankProfile.java | 30 +++++++++------------- config-model/src/main/javacc/SDParser.jj | 4 +-- .../processing/RankingExpressionsTestCase.java | 2 +- 3 files changed, 15 insertions(+), 21 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 25120c4dc76..b7890889d19 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -97,10 +97,6 @@ public class RankProfile implements Serializable, Cloneable { private Boolean ignoreDefaultRankFeatures = null; - private String secondPhaseRankingString = null; - - private String firstPhaseRankingString = null; - private Map functions = new LinkedHashMap<>(); private Set filterFields = new HashSet<>(); @@ -348,6 +344,15 @@ public class RankProfile implements Serializable, Cloneable { this.firstPhaseRanking=rankingExpression; } + public void setFirstPhaseRanking(String expression) { + try { + this.firstPhaseRanking = parseRankingExpression("firstphase", expression); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal first phase ranking function", e); + } + } + /** * Returns the ranking expression to use by this. This expression must not be edited. * Returns null if no expression is set. @@ -362,20 +367,9 @@ public class RankProfile implements Serializable, Cloneable { this.secondPhaseRanking = rankingExpression; } - public void setFirstPhaseRankingString(String expression) { - try { - this.firstPhaseRanking = parseRankingExpression("firstphase", expression); - this.firstPhaseRankingString = expression; - } - catch (ParseException e) { - throw new IllegalArgumentException("Illegal first phase ranking function", e); - } - } - - public void setSecondPhaseRankingString(String expression) { + public void setSecondPhaseRanking(String expression) { try { this.secondPhaseRanking = parseRankingExpression("secondphase", expression); - this.secondPhaseRankingString = expression; } catch (ParseException e) { throw new IllegalArgumentException("Illegal second phase ranking function", e); @@ -534,7 +528,7 @@ public class RankProfile implements Serializable, Cloneable { * @return string form of second phase ranking expression */ public String getSecondPhaseRankingString() { - if (secondPhaseRankingString != null) return secondPhaseRankingString; + if (secondPhaseRanking != null) return secondPhaseRanking.getRoot().toString(); if (getInherited() != null) return getInherited().getSecondPhaseRankingString(); return null; } @@ -545,7 +539,7 @@ public class RankProfile implements Serializable, Cloneable { * @return string form of first phase ranking expression */ public String getFirstPhaseRankingString() { - if (firstPhaseRankingString != null) return firstPhaseRankingString; + if (firstPhaseRanking != null) return firstPhaseRanking.getRoot().toString(); if (getInherited() != null) return getInherited().getFirstPhaseRankingString(); return null; } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 4c1d484a932..391010fdf33 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -2037,7 +2037,7 @@ Object firstPhaseItem(RankProfile profile) : double dropLimit; } { - ( expression = expression() { profile.setFirstPhaseRankingString(expression); } + ( expression = expression() { profile.setFirstPhaseRanking(expression); } | ( rerankCount = integer()) { profile.setKeepRankCount(rerankCount); } | ( dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } ) @@ -2066,7 +2066,7 @@ Object secondPhaseItem(RankProfile profile) : int rerankCount; } { - ( expression = expression() { profile.setSecondPhaseRankingString(expression); } + ( expression = expression() { profile.setSecondPhaseRanking(expression); } | ( rerankCount = integer()) { profile.setRerankCount(rerankCount); } ) { return null; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index bf13807434e..1c4e67c3d11 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -31,7 +31,7 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); - assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", functionsRankProfile.getFirstPhaseRankingString().trim()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", functionsRankProfile.getFirstPhaseRankingString().trim()); assertEquals("78 + closeness(distance)", functions.get("artistmatch").function().getBody().getRoot().toString()); assertEquals(0, functions.get("artistmatch").function().arguments().size()); -- cgit v1.2.3 From 9ab734145093fe978ec09af3f30d87845b857323 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 13:52:53 +0200 Subject: Refactor: No separate string accessors --- .../com/yahoo/searchdefinition/RankProfile.java | 22 ---------------------- .../processing/RankingExpressionsTestCase.java | 6 ++++-- 2 files changed, 4 insertions(+), 24 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b7890889d19..20b0631ce5e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -522,28 +522,6 @@ public class RankProfile implements Serializable, Cloneable { return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures(); } - /** - * Returns the string form of the second phase ranking expression. - * - * @return string form of second phase ranking expression - */ - public String getSecondPhaseRankingString() { - if (secondPhaseRanking != null) return secondPhaseRanking.getRoot().toString(); - if (getInherited() != null) return getInherited().getSecondPhaseRankingString(); - return null; - } - - /** - * Returns the string form of the first phase ranking expression. - * - * @return string form of first phase ranking expression - */ - public String getFirstPhaseRankingString() { - if (firstPhaseRanking != null) return firstPhaseRanking.getRoot().toString(); - if (getInherited() != null) return getInherited().getFirstPhaseRankingString(); - return null; - } - /** Adds a function and returns it */ public RankingExpressionFunction addFunction(String name, List arguments, String expression, boolean inline) { try { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 1c4e67c3d11..fd048737b43 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -31,8 +31,10 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); - assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", functionsRankProfile.getFirstPhaseRankingString().trim()); - assertEquals("78 + closeness(distance)", functions.get("artistmatch").function().getBody().getRoot().toString()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", + functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); + assertEquals("78 + closeness(distance)", + functions.get("artistmatch").function().getBody().getRoot().toString()); assertEquals(0, functions.get("artistmatch").function().arguments().size()); List> rankProperties = new RawRankProfile(functionsRankProfile, -- cgit v1.2.3 From d382242496bc164c2eddd9e5d0d00c107a90eb9c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 17 Sep 2018 13:57:08 +0200 Subject: Nonfunctional changes only --- .../java/com/yahoo/searchdefinition/RankProfile.java | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 20b0631ce5e..a21a613d2c8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -341,7 +341,7 @@ public class RankProfile implements Serializable, Cloneable { } public void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking=rankingExpression; + this.firstPhaseRanking = rankingExpression; } public void setFirstPhaseRanking(String expression) { @@ -518,8 +518,8 @@ public class RankProfile implements Serializable, Cloneable { } public boolean getIgnoreDefaultRankFeatures() { - if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures; - return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures(); + if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; + return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures(); } /** Adds a function and returns it */ @@ -563,8 +563,8 @@ public class RankProfile implements Serializable, Cloneable { } public double getRankScoreDropLimit() { - if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit; - if (getInherited()!=null) return getInherited().getRankScoreDropLimit(); + if (rankScoreDropLimit >- Double.MAX_VALUE) return rankScoreDropLimit; + if (getInherited() != null) return getInherited().getRankScoreDropLimit(); return rankScoreDropLimit; } @@ -793,11 +793,11 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ private boolean isIndexLevel; - private Type(String name) { + Type(String name) { this(name,false); } - private Type(String name,boolean isIndexLevel) { + Type(String name,boolean isIndexLevel) { this.name = name; this.isIndexLevel=isIndexLevel; } @@ -805,7 +805,7 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ public boolean isIndexLevel() { return isIndexLevel; } - /** @return The name of this type */ + /** Returns the name of this type */ public String getName() { return name; } @@ -838,10 +838,12 @@ public class RankProfile implements Serializable, Cloneable { } } + @Override public int hashCode() { return fieldName.hashCode() + 17 * type.hashCode(); } + @Override public boolean equals(Object object) { if (!(object instanceof RankSetting)) { return false; @@ -852,6 +854,7 @@ public class RankProfile implements Serializable, Cloneable { type.equals(other.type); } + @Override public String toString() { return type + " setting " + fieldName + ": " + value; } @@ -1004,6 +1007,7 @@ public class RankProfile implements Serializable, Cloneable { public Map getTypes() { return Collections.unmodifiableMap(types); } + } } -- cgit v1.2.3 From 1c37f6a5646d991de68759f2312164799a89ccaa Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 18 Sep 2018 10:26:46 -0700 Subject: Don't build rank profiles when generating document Java classes --- .../searchdefinition/DocumentsOnlyRankProfile.java | 35 ++++++++++++++++++++++ .../MinimalProcessingSearchBuilder.java | 16 ---------- .../com/yahoo/searchdefinition/RankProfile.java | 6 ++-- .../com/yahoo/searchdefinition/SearchBuilder.java | 32 ++++++++++++++------ ...ttributeTransformToSummaryOfImportedFields.java | 2 +- .../processing/AddExtraFieldsToDocument.java | 2 +- .../processing/AttributeProperties.java | 2 +- .../processing/AttributesImplicitWord.java | 2 +- .../yahoo/searchdefinition/processing/Bolding.java | 2 +- .../processing/BuiltInFieldSets.java | 3 +- .../processing/CreatePositionZCurve.java | 2 +- .../processing/DeprecateAttributePrefetch.java | 2 +- .../DisallowComplexMapAndWsetKeyTypes.java | 2 +- .../processing/DiversitySettingsValidator.java | 3 +- .../searchdefinition/processing/ExactMatch.java | 2 +- .../processing/FastAccessValidator.java | 4 +-- .../processing/FieldSetValidity.java | 2 +- .../processing/FilterFieldNames.java | 4 ++- .../processing/ImplicitSummaries.java | 2 +- .../processing/ImplicitSummaryFields.java | 2 +- .../processing/ImportedFieldsResolver.java | 2 +- .../processing/IndexFieldNames.java | 2 +- .../processing/IndexSettingsNonFieldNames.java | 2 +- .../processing/IndexingInputs.java | 2 +- .../processing/IndexingOutputs.java | 2 +- .../processing/IndexingValidation.java | 2 +- .../processing/IndexingValues.java | 2 +- .../processing/IntegerIndex2Attribute.java | 2 +- .../searchdefinition/processing/LiteralBoost.java | 2 +- .../searchdefinition/processing/MakeAliases.java | 2 +- .../processing/MakeDefaultSummaryTheSuperSet.java | 2 +- .../processing/MatchConsistency.java | 2 +- .../processing/MatchPhaseSettingsValidator.java | 3 +- .../processing/MinimalProcessing.java | 10 ------- .../processing/MultifieldIndexHarmonizer.java | 2 +- .../processing/MutableAttributes.java | 6 ++-- .../searchdefinition/processing/NGramMatch.java | 2 +- .../processing/OptimizeIlscript.java | 2 +- .../processing/PredicateProcessor.java | 2 +- .../searchdefinition/processing/Processing.java | 25 +++++----------- .../searchdefinition/processing/Processor.java | 4 ++- .../processing/RankingExpressionTypeValidator.java | 3 +- .../processing/ReferenceFieldsProcessor.java | 2 +- .../processing/ReservedDocumentNames.java | 2 +- .../processing/ReservedFunctionNames.java | 5 ++-- .../processing/SearchMustHaveDocument.java | 2 +- .../searchdefinition/processing/SetLanguage.java | 3 +- .../processing/SetRankTypeEmptyOnFilters.java | 2 +- .../processing/SortingSettings.java | 2 +- .../StringSettingsOnNonStringFields.java | 2 +- .../processing/SummaryConsistency.java | 2 +- .../processing/SummaryDynamicStructsArrays.java | 2 +- .../SummaryFieldsMustHaveValidSource.java | 2 +- .../processing/SummaryNamesFieldCollisions.java | 2 +- .../yahoo/searchdefinition/processing/TagType.java | 2 +- .../processing/TensorFieldProcessor.java | 2 +- .../searchdefinition/processing/TextMatch.java | 2 +- .../yahoo/searchdefinition/processing/UriHack.java | 2 +- .../processing/UrlFieldValidator.java | 2 +- .../processing/ValidateFieldTypes.java | 2 +- ...ValidateFieldWithIndexSettingsCreatesIndex.java | 2 +- .../searchdefinition/processing/WordMatch.java | 2 +- .../RankProfileTypeSettingsProcessor.java | 6 ++-- config-model/src/main/javacc/SDParser.jj | 23 +++++++++++--- .../searchdefinition/SearchImporterTestCase.java | 11 ++++--- .../yahoo/searchdefinition/derived/IdTestCase.java | 2 +- .../derived/LiteralBoostTestCase.java | 2 +- .../derived/SummaryMapTestCase.java | 6 ++-- .../derived/TypeConversionTestCase.java | 2 +- ...buteTransformToSummaryOfImportedFieldsTest.java | 2 +- .../processing/AttributePropertiesTestCase.java | 4 +-- .../processing/BoldingTestCase.java | 2 +- .../processing/ImportedFieldsResolverTestCase.java | 2 +- .../processing/IndexingScriptRewriterTestCase.java | 2 +- .../processing/IntegerIndex2AttributeTestCase.java | 3 +- .../SummaryFieldsMustHaveValidSourceTestCase.java | 18 ++++++----- .../processing/ValidateFieldTypesTest.java | 2 +- .../main/java/com/yahoo/vespa/DocumentGenMojo.java | 13 ++++---- 78 files changed, 197 insertions(+), 157 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java new file mode 100644 index 00000000000..9335c0b4005 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java @@ -0,0 +1,35 @@ +package com.yahoo.searchdefinition; + +import java.util.List; + +/** + * A rank profile which ignores all calls made to it which may fail in a document only setting. + * This is used by the search definition parser when it is requested to parse documents only, + * to avoid having to check for this in every method which adds to the rank profile. + * (And why do we ever want to parse documents only? Because it is used when generating Java classes + * from documents, where the full application package may not be available.) + * + * @author bratseth + */ +public class DocumentsOnlyRankProfile extends RankProfile { + + public DocumentsOnlyRankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) { + super(name, search, rankProfileRegistry); + } + + @Override + public void setFirstPhaseRanking(String expression) { + // Ignore + } + + @Override + public void setSecondPhaseRanking(String expression) { + // Ignore + } + + @Override + public void addFunction(String name, List arguments, String expression, boolean inline) { + // Ignore + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java deleted file mode 100644 index 1e8f6cb91b9..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.processing.MinimalProcessing; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -public class MinimalProcessingSearchBuilder extends SearchBuilder { - public MinimalProcessingSearchBuilder() { - super(); - } - - @Override - protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { - new MinimalProcessing().process(search, deployLogger, getRankProfileRegistry(), queryProfiles, validate); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index a21a613d2c8..16e494c2db1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -522,10 +522,10 @@ public class RankProfile implements Serializable, Cloneable { return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures(); } - /** Adds a function and returns it */ - public RankingExpressionFunction addFunction(String name, List arguments, String expression, boolean inline) { + /** Adds a function */ + public void addFunction(String name, List arguments, String expression, boolean inline) { try { - return addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); } catch (ParseException e) { throw new IllegalArgumentException("Could not parse function '" + name + "'", e); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 592efcc2d85..3c2ebc058ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -15,7 +15,6 @@ import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.parser.SDParser; import com.yahoo.searchdefinition.parser.SimpleCharStream; import com.yahoo.searchdefinition.parser.TokenMgrException; -import com.yahoo.searchdefinition.processing.MinimalProcessing; import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -40,17 +39,25 @@ public class SearchBuilder { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); private List searchList = new LinkedList<>(); - private ApplicationPackage app = null; + private ApplicationPackage app; private boolean isBuilt = false; private DocumentModel model = new DocumentModel(); private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; + /** True to build the document aspect only, skipping instantiation of rank profiles */ + private final boolean documentsOnly; + /** For testing only */ public SearchBuilder() { this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry()); } + /** Used for generating documents for typed access to document fields in Java */ + public SearchBuilder(boolean documentsOnly) { + this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); + } + /** For testing only */ public SearchBuilder(ApplicationPackage app) { this(app, new RankProfileRegistry(), new QueryProfileRegistry()); @@ -69,9 +76,16 @@ public class SearchBuilder { public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + this(app, rankProfileRegistry, queryProfileRegistry, false); + } + public SearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { this.app = app; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -151,7 +165,7 @@ public class SearchBuilder { Search search; SimpleCharStream stream = new SimpleCharStream(str); try { - search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir); + search = new SDParser(stream, deployLogger, app, rankProfileRegistry, documentsOnly).search(docTypeMgr, searchDefDir); } catch (TokenMgrException e) { throw new ParseException("Unknown symbol: " + e.getMessage()); } catch (ParseException pe) { @@ -165,9 +179,9 @@ public class SearchBuilder { * {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed search objects used in unit tests. * - * @param rawSearch The object to import. - * @return The name of the imported object. - * @throws IllegalArgumentException Thrown if the given search object has already been processed. + * @param rawSearch the object to import. + * @return the name of the imported object. + * @throws IllegalArgumentException if the given search object has already been processed. */ public String importRawSearch(Search rawSearch) { if (rawSearch.getName() == null) { @@ -251,15 +265,15 @@ public class SearchBuilder { * #build()} method so that subclasses can choose not to build anything. */ protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { - new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate); + new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate, documentsOnly); } /** * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object * built. This method will never return null. * - * @return The build object. - * @throws IllegalStateException Thrown if there is not exactly one search. + * @return the built object + * @throws IllegalStateException if there is not exactly one search. */ public Search getSearch() { if ( ! isBuilt) throw new IllegalStateException("Searches not built."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java index cf123d0f7c1..fefb54a7fe3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -27,7 +27,7 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.allImportedFields() .flatMap(this::getSummaryFieldsForImportedField) .forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index e0d32ea8ccd..803a6c5ab40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,7 +26,7 @@ public class AddExtraFieldsToDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = search.getDocument(); if (document != null) { for (Field field : search.extraFieldList()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java index 9ec596792fa..94589d94255 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java @@ -20,7 +20,7 @@ public class AttributeProperties extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { String fieldName = field.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index a95f4264dc6..23257e5eafd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -23,7 +23,7 @@ public class AttributesImplicitWord extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (fieldImplicitlyWordMatch(field)) { field.getMatching().setType(Matching.Type.WORD); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java index a3c4c97cf31..b9be30e8485 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java @@ -22,7 +22,7 @@ public class Bolding extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { for (SummaryField summary : field.getSummaryFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java index 37d60c1d32e..a0c4c8adb2d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document + * * @author Vegard Havdal */ public class BuiltInFieldSets extends Processor { @@ -23,7 +24,7 @@ public class BuiltInFieldSets extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { addDocumentFieldSet(); addSearchFieldSet(); // "Hook" the field sets on search onto the document types, since we will include them diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index d7b688be203..ad862ef767f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -38,7 +38,7 @@ public class CreatePositionZCurve extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { DataType fieldType = field.getDataType(); if ( ! isSupportedPositionType(fieldType)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java index 7cc9b4e9b52..b34db6febd5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java @@ -15,7 +15,7 @@ public class DeprecateAttributePrefetch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java index 861ebad7085..076161a8584 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -23,7 +23,7 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 6b78da2146b..029892cba1c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -18,8 +18,9 @@ public class DiversitySettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 59bc3dc66f4..a7c0ebd4a07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -26,7 +26,7 @@ public class ExactMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { Matching.Type matching = field.getMatching().getType(); if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java index 9cfac625da5..cd33e4434e7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java @@ -21,7 +21,7 @@ public class FastAccessValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String invalidAttributes = search.allFields() @@ -29,7 +29,7 @@ public class FastAccessValidator extends Processor { .filter(FastAccessValidator::isIncompatibleAttribute) .map(Attribute::getName) .collect(Collectors.joining(", ")); - if (!invalidAttributes.isEmpty()) { + if ( ! invalidAttributes.isEmpty()) { throw new IllegalArgumentException( String.format( "For search '%s': The following attributes have a type that is incompatible with fast-access: %s. " + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java index 15c11589245..4ef9a9733d5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java @@ -24,7 +24,7 @@ public class FieldSetValidity extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index 0c75314ffa2..adb8ab62aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -26,7 +26,9 @@ public class FilterFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + for (SDField f : search.allConcreteFields()) { if (f.getRanking().isFilter()) { filterField(f.getName()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index b51524b7e62..1f795458875 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -28,7 +28,7 @@ public class ImplicitSummaries extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary = search.getSummary("default"); if (defaultSummary == null) { defaultSummary = new DocumentSummary("default"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index 7464f574255..0d99c698aca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -21,7 +21,7 @@ public class ImplicitSummaryFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary docsum : search.getSummaries().values()) { addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index 9f03cdf4b5e..a3efd086c6a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -33,7 +33,7 @@ public class ImportedFieldsResolver extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); search.setImportedFields(new ImportedFields(importedFields)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java index 018183b91d8..210a8e7009c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java @@ -23,7 +23,7 @@ public class IndexFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java index 0740029de90..41355a76f47 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java @@ -24,7 +24,7 @@ public class IndexSettingsNonFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java index 419268468c2..aeab2bb6638 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java @@ -27,7 +27,7 @@ public class IndexingInputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java index 6f04184c512..11d69bf6c75 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java @@ -29,7 +29,7 @@ public class IndexingOutputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index b73151768fd..27520647e3b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -25,7 +25,7 @@ public class IndexingValidation extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; VerificationContext context = new VerificationContext(new MyAdapter(search)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index f3907ae68cb..72777b7dfb4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -22,7 +22,7 @@ public class IndexingValues extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (Field field : search.getDocument().fieldSet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java index c119dc2660b..baaf145dbce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java @@ -30,7 +30,7 @@ public class IntegerIndex2Attribute extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { if (field.getIndex(field.getName()) != null diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java index 507a0e87cff..fe94ac9849f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java @@ -31,7 +31,7 @@ public class LiteralBoost extends Processor { /** Adds extra search fields and indices to express literal boosts */ @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { checkRankModifierRankType(search); addLiteralBoostsToFields(search); reduceFieldLiteralBoosts(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java index f853b99bbb2..0daf7265daa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java @@ -25,7 +25,7 @@ public class MakeAliases extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { List usedAliases = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { for (Map.Entry e : field.getAliasToName().entrySet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java index 2c43a65da99..6f67c22d9d2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -34,7 +34,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary=search.getSummary("default"); for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) { if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java index fe584fa41c0..ff8a5c2eb0b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java @@ -30,7 +30,7 @@ public class MatchConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map types = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index 479384e09ef..b1728b9bd89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -20,8 +20,9 @@ public class MatchPhaseSettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java deleted file mode 100644 index 0fee844ecbc..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.yahoo.searchdefinition.processing; - -import java.util.Collection; - -public class MinimalProcessing extends Processing { - @Override - protected Collection createProcessorFactories() { - return minimalSetOfProcessors(); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java index 45e018f8fc3..a52a8ab74e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java @@ -31,7 +31,7 @@ public class MultifieldIndexHarmonizer extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { populateIndexToFields(search); resolveAllConflicts(search); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java index 9bcb3929b3d..4d8f0032a78 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java @@ -15,12 +15,12 @@ public class MutableAttributes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { - if (!field.isExtraField() && field.getAttributes().containsKey(field.getName())) { + if ( ! field.isExtraField() && field.getAttributes().containsKey(field.getName())) { if (field.getAttributes().get(field.getName()).isMutable()) { throw new IllegalArgumentException("Field '" + field.getName() + "' in '" + search.getDocument().getName() + - "' can not be marked mutable as it is inside the document clause."); + "' can not be marked mutable as it is inside the document clause."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java index cdfba54ee5a..3f3fc11380b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java @@ -26,7 +26,7 @@ public class NGramMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType().equals(Matching.Type.GRAM)) implementGramMatch(search, field, validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java index c89c709ffbf..8f2a29abcb6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -22,7 +22,7 @@ public class OptimizeIlscript extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 3583c4a0162..79f19efe422 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -33,7 +33,7 @@ public class PredicateProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { if (validate && field.doesIndexing()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index d92979f3ae4..8c8c32389e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -20,7 +20,7 @@ import java.util.List; */ public class Processing { - protected Collection minimalSetOfProcessors() { + private Collection processors() { return Arrays.asList( SearchMustHaveDocument::new, UrlFieldValidator::new, @@ -74,24 +74,13 @@ public class Processing { RankProfileTypeSettingsProcessor::new, ReferenceFieldsProcessor::new, FastAccessValidator::new, - ReservedFunctionNames::new); - } - - private Collection extendedSetOfProcessors() { - return Arrays.asList( + ReservedFunctionNames::new, RankingExpressionTypeValidator::new, - - // These should be last. + // These should be last: IndexingValidation::new, IndexingValues::new); } - protected Collection createProcessorFactories() { - List processorFactories = new ArrayList<>(minimalSetOfProcessors()); - processorFactories.addAll(extendedSetOfProcessors()); - return processorFactories; - } - /** * Runs all search processors on the given {@link Search} object. These will modify the search object, possibly * exchanging it with another, as well as its document types. @@ -102,17 +91,17 @@ public class Processing { * @param queryProfiles The query profiles contained in the application this search is part of. */ public void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, boolean validate) - { - Collection factories = createProcessorFactories(); + QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { + Collection factories = processors(); search.process(); factories.stream() .map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles)) - .forEach(processor -> processor.process(validate)); + .forEach(processor -> processor.process(validate, documentsOnly)); } @FunctionalInterface public interface ProcessorFactory { Processor create(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index b938e40d9a2..6bfd0ef29ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -53,8 +53,10 @@ public abstract class Processor { * @param validate true to throw exceptions on validation errors, false to make the best possible effort * at completing processing without throwing an exception. * If we are not validating, emitting warnings have no effect and can (but must not) be skipped. + * @param documentsOnly true to skip processing (including validation, regardless of the validate setting) + * of aspects not relating to document definitions (e.g rank profiles) */ - public abstract void process(boolean validate); + public abstract void process(boolean validate, boolean documentsOnly); /** * Convenience method for adding a no-strings-attached implementation field for a regular field diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index b3fe8ec36f6..102d1910360 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -33,8 +33,9 @@ public class RankingExpressionTypeValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java index 76afcd5d520..0418538922b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java @@ -28,7 +28,7 @@ public class ReferenceFieldsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { clearSummaryAttributeAspectForConcreteFields(); clearSummaryAttributeAspectForExplicitSummaryFields(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java index f2aa31bb9c3..805cbaced0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java @@ -28,7 +28,7 @@ public class ReservedDocumentNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String docName = search.getDocument().getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java index 8aa8ff56756..d7099215f17 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -27,13 +27,14 @@ public class ReservedFunctionNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rp : rankProfileRegistry.all()) { for (String functionName : rp.getFunctions().keySet()) { if (reservedNames.contains(functionName)) { - deployLogger.log(Level.WARNING, "Funcion '" + functionName + "' " + + deployLogger.log(Level.WARNING, "Function '" + functionName + "' " + "in rank profile '" + rp.getName() + "' " + "has a reserved name. This might mean that the function shadows " + "the built-in function with the same name." diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java index 403de1253b4..2d8eaff7762 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java @@ -19,7 +19,7 @@ public class SearchMustHaveDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; if (search.getDocument() == null) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java index a0c884d25f9..8a4795c4dd2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java @@ -24,7 +24,8 @@ public class SetLanguage extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; List textFieldsWithoutLanguage = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java index a19ea8d7068..715828f808a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java @@ -20,7 +20,7 @@ public class SetRankTypeEmptyOnFilters extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getRanking().isFilter()) { field.setRankType(RankType.EMPTY); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java index 6426c724a07..defcf761649 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java @@ -21,7 +21,7 @@ public class SortingSettings extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java index d56b0272f06..e133f88f45c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java @@ -16,7 +16,7 @@ public class StringSettingsOnNonStringFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index a952d3732b3..d2c4968ca26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -26,7 +26,7 @@ public class SummaryConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary summary : search.getSummaries().values()) { if (summary.getName().equals("default")) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java index 647a433f201..b8d170c07f6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java @@ -26,7 +26,7 @@ public class SummaryDynamicStructsArrays extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java index 7bcbd9a267c..9b51c7c473e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -22,7 +22,7 @@ public class SummaryFieldsMustHaveValidSource extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (DocumentSummary summary : search.getSummaries().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java index 23569cf39ae..678d5324e38 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java @@ -26,7 +26,7 @@ public class SummaryNamesFieldCollisions extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map> fieldToClassAndSource = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java index 177fc7f2326..79b7a6067b9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java @@ -25,7 +25,7 @@ public class TagType extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) implementTagType(field); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 08571168336..8e54d7c00d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -22,7 +22,7 @@ public class TensorFieldProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index 645ed5121ea..74f30d6a730 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -34,7 +34,7 @@ public class TextMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType() != Matching.Type.TEXT) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index ac376982cfa..d81fdf70d20 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -28,7 +28,7 @@ public class UriHack extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing()) { DataType fieldType = field.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java index ed813b42fff..c6b83349691 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java @@ -18,7 +18,7 @@ public class UrlFieldValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java index 54ad9f13f6f..21b7f1d2675 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -28,7 +28,7 @@ public class ValidateFieldTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String searchName = search.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java index a0b204a25f2..408d60e1cff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -21,7 +21,7 @@ public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Matching defaultMatching = new Matching(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java index 892bcdad12c..13fe3f24d69 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -25,7 +25,7 @@ public class WordMatch extends Processor { super(search, deployLogger, rankProfileRegistry, queryProfiles); } - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if ( ! field.getMatching().getType().equals(Matching.Type.WORD)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index cc1638347f6..ec4cbdfe58b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.Optional; /** - * Class that processes a search instance and sets type settings on all rank profiles. + * This processes a search instance and sets type settings on all rank profiles. * * Currently, type settings are limited to the type of tensor attribute fields and tensor query features. * @@ -35,7 +35,9 @@ public class RankProfileTypeSettingsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + processAttributeFields(); processImportedFields(); processQueryProfileTypes(); diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 391010fdf33..813d1d47533 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -34,6 +34,7 @@ import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferen import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.DocumentsOnlyRankProfile; import com.yahoo.searchdefinition.DefaultRankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; @@ -75,6 +76,7 @@ public class SDParser { private ApplicationPackage app; private DeployLogger deployLogger; private RankProfileRegistry rankProfileRegistry; + private boolean documentsOnly; /** For testing only */ public SDParser(String input, DeployLogger deployLogger) { @@ -83,17 +85,24 @@ public class SDParser { /** For testing only */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger) { - this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry()); + this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry(), false); } + /** + * Creates a parser + * + * @param documentsOnly true to only parse the document aspect of a search definition (e.g skip rank profiles) + */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger, ApplicationPackage applicationPackage, - RankProfileRegistry rankProfileRegistry) { + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) { this(stream); this.deployLogger = deployLogger; this.app = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -1806,6 +1815,7 @@ void rankingConstant(Search search) : } lbrace() (rankingConstantItem(constant) ()*)+ ) { + if (documentsOnly) return; search.rankingConstants().add(constant); } } @@ -1861,7 +1871,10 @@ void rankProfile(Search search) : { ( name = identifier() { - if ("default".equals(name)) { + if (documentsOnly) { + profile = new DocumentsOnlyRankProfile(name, search, rankProfileRegistry); + } + else if ("default".equals(name)) { profile = rankProfileRegistry.get(search, "default"); } else { profile = new RankProfile(name, search, rankProfileRegistry); @@ -1870,6 +1883,7 @@ void rankProfile(Search search) : [inheritsRankProfile(profile)] lbrace() (rankProfileItem(profile) ()*)* ) { + if (documentsOnly) return; rankProfileRegistry.add(profile); } } @@ -1912,7 +1926,8 @@ void inheritsRankProfile(RankProfile profile) : String str; } { - str = identifier() { profile.setInherited(str); } + str = identifier() + { profile.setInherited(str); } } /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 8cdfdd51637..82c03c02f61 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -34,20 +34,19 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { assertEquals("simple",search.getName()); assertTrue(search.hasDocument()); - SDDocumentType document=search.getDocument(); - assertEquals("simple",document.getName()); - assertEquals(12,document.getFieldCount()); + SDDocumentType document = search.getDocument(); + assertEquals("simple", document.getName()); + assertEquals(12, document.getFieldCount()); SDField field; Attribute attribute; - new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true); + new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); // First field field=(SDField) document.getField("title"); assertEquals(DataType.STRING,field.getDataType()); - assertEquals("{ summary | index; }", - field.getIndexingScript().toString()); + assertEquals("{ summary | index; }", field.getIndexingScript().toString()); assertTrue(!search.getIndex("default").isPrefix()); assertTrue(search.getIndex("title").isPrefix()); Iterator titleAliases=search.getIndex("title").aliasIterator(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 3ef8591d203..f87a26c6f79 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -31,7 +31,7 @@ public class IdTestCase extends AbstractExportingTestCase { uri.parseIndexingScript("{ summary | index }"); document.addField(uri); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); assertNull(document.getField("uri")); assertNull(document.getField("Uri")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index fad80164135..2bae285301c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -41,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); // Check attribute fields diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index 9d19d4b154c..ba19a8312f6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -69,14 +69,14 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { assertTrue(!transforms.hasNext()); } @Test - public void testPositionDeriving() throws IOException, ParseException { + public void testPositionDeriving() { Search search = new Search("store", null); SDDocumentType document = new SDDocumentType("store"); search.addDocument(document); String fieldName = "location"; SDField field = document.addField(fieldName, PositionDataType.INSTANCE); field.parseIndexingScript("{ attribute | summary }"); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); SummaryMap summaryMap = new SummaryMap(search, new Summaries(search, new BaseDeployLogger())); Iterator transforms = summaryMap.resultTransformIterator(); @@ -141,7 +141,7 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { } @Test - public void testFailOnSummaryFieldSourceCollision() throws IOException, ParseException { + public void testFailOnSummaryFieldSourceCollision() { try { Search search = SearchBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 17cc9aaae4c..8941b07101d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -33,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase { a.parseIndexingScript("{ index }"); document.addField(a); - new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index a0556a156b5..48adc0eefc5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -36,7 +36,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( search,null,null,null); - processor.process(true); + processor.process(true, false); SummaryField summaryField = search.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); SummaryTransform actualTransform = summaryField.getTransform(); assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java index 8f1cc9c3de8..3a0fedfd550 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java @@ -24,7 +24,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { public void testInvalidAttributeProperties() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties1.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail("attribute property should not be set"); } catch (RuntimeException e) { // empty @@ -34,7 +34,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { @Test public void testValidAttributeProperties() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties2.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index 4dee939e3da..1ab8b054cb7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -24,7 +24,7 @@ public class BoldingTestCase extends SearchDefinitionTestCase { public void testBoldingNonString() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/processing/boldnonstring.sd"); - new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail(); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("'bolding: on' for non-text field")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 7230f176048..de08bf66548 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -188,7 +188,7 @@ public class ImportedFieldsResolverTestCase { private static ImportedFields resolve(Search search) { assertNotNull(search.temporaryImportedFields().get()); assertFalse(search.importedFields().isPresent()); - new ImportedFieldsResolver(search, null, null, null).process(true); + new ImportedFieldsResolver(search, null, null, null).process(true, false); assertFalse(search.temporaryImportedFields().isPresent()); assertNotNull(search.importedFields().get()); return search.importedFields().get(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index 2af3477fa62..e078d91f248 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -155,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { sdoc.addField(unprocessedField); Search search = new Search("test", null); search.addDocument(sdoc); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); return unprocessedField.getIndexingScript(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index 1e0270f293d..29bba224f46 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -19,11 +19,12 @@ import static org.junit.Assert.assertTrue; * @author baldersheim */ public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { + @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/integerindex2attribute.sd"); search.process(); - new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); SDField f; f = search.getConcreteField("s1"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index 85bee61c1b4..d0c1bf8b0ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -12,6 +12,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { @@ -20,41 +21,44 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTe Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); } } + @Test public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidimplicitsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); } } + @Test public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidselfreferringsummary.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); } } + @Test public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/documentidinsummary.sd"); search.process(); BaseDeployLogger deployLogger = new BaseDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true); + new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); assertEquals("documentid", search.getSummary("withid").getSummaryField("w").getSingleSource()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index ab3bb113727..d0b6524a7e1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -40,7 +40,7 @@ public class ValidateFieldTypesTest { exceptionRule.expectMessage( "For search '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); - validator.process(true); + validator.process(true, false); } private static Search createSearchWithDocument(String documentName) { diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index b7c955bd83b..d27352b8ea7 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -7,12 +7,9 @@ import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.MinimalProcessingSearchBuilder; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.UnprocessingSearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.tensor.TensorType; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -80,10 +77,10 @@ public class DocumentGenMojo extends AbstractMojo { void execute(File sdDir, File outputDir, String packageName) throws MojoFailureException { if ("".equals(packageName)) throw new IllegalArgumentException("You may not use empty package for generated types."); - searches = new HashMap(); - docTypes = new HashMap(); - structTypes = new HashMap(); - annotationTypes = new HashMap(); + searches = new HashMap<>(); + docTypes = new HashMap<>(); + structTypes = new HashMap<>(); + annotationTypes = new HashMap<>(); outputDir.mkdirs(); SearchBuilder builder = buildSearches(sdDir); @@ -111,7 +108,7 @@ public class DocumentGenMojo extends AbstractMojo { public boolean accept(File dir, String name) { return name.endsWith(".sd"); }}); - SearchBuilder builder = new MinimalProcessingSearchBuilder(); + SearchBuilder builder = new SearchBuilder(true); for (File f : sdFiles) { try { long modTime = f.lastModified(); -- cgit v1.2.3