diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-09-18 13:38:38 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-09-18 13:38:38 -0700 |
commit | b38fc011d4a5cdcdbdb5d71ca77252502957fa92 (patch) | |
tree | e5a20afb500270aac9e4476ab96d60329af99574 | |
parent | 04a2bea1982819f1e250dfc40b7fbff3fcfe303b (diff) | |
parent | 1c37f6a5646d991de68759f2312164799a89ccaa (diff) |
Merge pull request #6992 from vespa-engine/bratseth/rank-type-information
Bratseth/rank type information
120 files changed, 798 insertions, 863 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java new file mode 100644 index 00000000000..9335c0b4005 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java @@ -0,0 +1,35 @@ +package com.yahoo.searchdefinition; + +import java.util.List; + +/** + * A rank profile which ignores all calls made to it which may fail in a document only setting. + * This is used by the search definition parser when it is requested to parse documents only, + * to avoid having to check for this in every method which adds to the rank profile. + * (And why do we ever want to parse documents only? Because it is used when generating Java classes + * from documents, where the full application package may not be available.) + * + * @author bratseth + */ +public class DocumentsOnlyRankProfile extends RankProfile { + + public DocumentsOnlyRankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) { + super(name, search, rankProfileRegistry); + } + + @Override + public void setFirstPhaseRanking(String expression) { + // Ignore + } + + @Override + public void setSecondPhaseRanking(String expression) { + // Ignore + } + + @Override + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { + // Ignore + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index afd33da369f..0d9ea00bf73 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Stack; import java.util.stream.Collectors; /** @@ -71,7 +70,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement currentResolutionCallStack.stream().map(Reference::toString).collect(Collectors.joining(" -> ")) + " -> " + reference); - // A reference to a macro argument? + // A reference to a function argument? Optional<String> binding = boundIdentifier(reference); if (binding.isPresent()) { try { @@ -117,7 +116,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement } /** - * Returns the default type for this simple feature, or nullif it does not have a default + * Returns the default type for this simple feature, or null if it does not have a default */ public TensorType defaultTypeOf(Reference reference) { if ( ! FeatureNames.isSimpleFeature(reference)) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java deleted file mode 100644 index 1e8f6cb91b9..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MinimalProcessingSearchBuilder.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.processing.MinimalProcessing; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -public class MinimalProcessingSearchBuilder extends SearchBuilder { - public MinimalProcessingSearchBuilder() { - super(); - } - - @Override - protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { - new MinimalProcessing().process(search, deployLogger, getRankProfileRegistry(), queryProfiles, validate); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b7e1f9d4538..16e494c2db1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -95,13 +95,9 @@ public class RankProfile implements Serializable, Cloneable { /** The properties of this - a multimap */ private Map<String, List<RankProperty>> rankProperties = new LinkedHashMap<>(); - private Boolean ignoreDefaultRankFeatures=null; + private Boolean ignoreDefaultRankFeatures = null; - private String secondPhaseRankingString=null; - - private String firstPhaseRankingString=null; - - private Map<String, Macro> macros= new LinkedHashMap<>(); + private Map<String, RankingExpressionFunction> functions = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -339,13 +335,22 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getFirstPhaseRanking() { - if (firstPhaseRanking!=null) return firstPhaseRanking; - if (getInherited()!=null) return getInherited().getFirstPhaseRanking(); + if (firstPhaseRanking != null) return firstPhaseRanking; + if (getInherited() != null) return getInherited().getFirstPhaseRanking(); return null; } public void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking=rankingExpression; + this.firstPhaseRanking = rankingExpression; + } + + public void setFirstPhaseRanking(String expression) { + try { + this.firstPhaseRanking = parseRankingExpression("firstphase", expression); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal first phase ranking function", e); + } } /** @@ -353,31 +358,22 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getSecondPhaseRanking() { - if (secondPhaseRanking!=null) return secondPhaseRanking; - if (getInherited()!=null) return getInherited().getSecondPhaseRanking(); + if (secondPhaseRanking != null) return secondPhaseRanking; + if (getInherited() != null) return getInherited().getSecondPhaseRanking(); return null; } public void setSecondPhaseRanking(RankingExpression rankingExpression) { - this.secondPhaseRanking=rankingExpression; + this.secondPhaseRanking = rankingExpression; } - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for second phase - */ - public void setSecondPhaseRankingString(String exp) { - this.secondPhaseRankingString = exp; - } - - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for first phase - */ - public void setFirstPhaseRankingString(String exp) { - this.firstPhaseRankingString = exp; + public void setSecondPhaseRanking(String expression) { + try { + this.secondPhaseRanking = parseRankingExpression("secondphase", expression); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal second phase ranking function", e); + } } /** Returns a read-only view of the summary features to use in this profile. This is never null */ @@ -412,8 +408,8 @@ public class RankProfile implements Serializable, Cloneable { } public void addRankFeature(ReferenceNode feature) { - if (rankFeatures==null) - rankFeatures=new LinkedHashSet<>(); + if (rankFeatures == null) + rankFeatures = new LinkedHashSet<>(); rankFeatures.add(feature); } @@ -522,55 +518,43 @@ public class RankProfile implements Serializable, Cloneable { } public boolean getIgnoreDefaultRankFeatures() { - if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures; - return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures(); + if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; + return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures(); } - /** - * Returns the string form of the second phase ranking expression. - * - * @return string form of second phase ranking expression - */ - public String getSecondPhaseRankingString() { - if (secondPhaseRankingString != null) return secondPhaseRankingString; - if (getInherited() != null) return getInherited().getSecondPhaseRankingString(); - return null; - } - - /** - * Returns the string form of the first phase ranking expression. - * - * @return string form of first phase ranking expression - */ - public String getFirstPhaseRankingString() { - if (firstPhaseRankingString != null) return firstPhaseRankingString; - if (getInherited() != null) return getInherited().getFirstPhaseRankingString(); - return null; + /** Adds a function */ + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { + try { + addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + } + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse function '" + name + "'", e); + } } - /** Creates a new (empty) macro and returns it */ - public Macro addMacro(String name, boolean inline) { - Macro macro = new Macro(name, inline); - macros.put(name, macro); - return macro; + /** Adds a function and returns it */ + public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { + RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); + functions.put(function.getName(), rankingExpressionFunction); + return rankingExpressionFunction; } - /** Returns an unmodifiable view of the macros in this */ - public Map<String, Macro> getMacros() { - if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap(); - if (macros.size() == 0) return getInherited().getMacros(); - if (getInherited() == null) return Collections.unmodifiableMap(macros); + /** Returns an unmodifiable view of the functions in this */ + public Map<String, RankingExpressionFunction> getFunctions() { + if (functions.size() == 0 && getInherited() == null) return Collections.emptyMap(); + if (functions.size() == 0) return getInherited().getFunctions(); + if (getInherited() == null) return Collections.unmodifiableMap(functions); // Neither is null - Map<String, Macro> allMacros = new LinkedHashMap<>(getInherited().getMacros()); - allMacros.putAll(macros); - return Collections.unmodifiableMap(allMacros); + Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions()); + allFunctions.putAll(functions); + return Collections.unmodifiableMap(allFunctions); } public int getKeepRankCount() { - if (keepRankCount>=0) return keepRankCount; - if (getInherited()!=null) return getInherited().getKeepRankCount(); + if (keepRankCount >= 0) return keepRankCount; + if (getInherited() != null) return getInherited().getKeepRankCount(); return -1; } @@ -579,8 +563,8 @@ public class RankProfile implements Serializable, Cloneable { } public double getRankScoreDropLimit() { - if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit; - if (getInherited()!=null) return getInherited().getRankScoreDropLimit(); + if (rankScoreDropLimit >- Double.MAX_VALUE) return rankScoreDropLimit; + if (getInherited() != null) return getInherited().getRankScoreDropLimit(); return rankScoreDropLimit; } @@ -607,58 +591,15 @@ public class RankProfile implements Serializable, Cloneable { return retval; } - /** - * Will take the parser-set textual ranking expressions and turn into ranking expression objects, - * if not already done - */ - // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions - // until this is called. Simplify by parsing them right away. - public void parseExpressions() { - try { - parseRankingExpressions(); - parseMacros(); - } catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - /** - * Passes the contents of macros on to parser. Then put all the implied rank properties - * from those macros into the profile's props map. - */ - private void parseMacros() throws ParseException { - for (Map.Entry<String, Macro> e : getMacros().entrySet()) { - String macroName = e.getKey(); - Macro macro = e.getValue(); - if (macro.getRankingExpression() == null) { - RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); - macro.setRankingExpression(expr); - macro.setTextualExpression(expr.getRoot().toString()); - } - } - } - - /** - * Passes ranking expressions on to parser - * - * @throws ParseException if either of the ranking expressions could not be parsed - */ - private void parseRankingExpressions() throws ParseException { - if (getFirstPhaseRankingString() != null && firstPhaseRanking == null) - setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString())); - if (getSecondPhaseRankingString() != null && secondPhaseRanking == null) - setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); - } - - private RankingExpression parseRankingExpression(String expressionName, String exp) throws ParseException { - if (exp.trim().length() == 0) + private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { + if (expression.trim().length() == 0) throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); - try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, exp.trim())) { + try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) { return new RankingExpression(expressionName, rankingExpressionReader); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { - ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() + + ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + "' in " + getName()+ ", " + expressionName + "."); throw (ParseException)exception.initCause(e); } @@ -686,14 +627,13 @@ public class RankProfile implements Serializable, Cloneable { @Override public RankProfile clone() { try { - // Note: This treats RankingExpression in Macros as immutables even though they are not RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; clone.rankProperties = new LinkedHashMap<>(this.rankProperties); - clone.macros = new LinkedHashMap<>(this.macros); + clone.functions = new LinkedHashMap<>(this.functions); clone.filterFields = new HashSet<>(this.filterFields); clone.constants = new HashMap<>(this.constants); return clone; @@ -719,60 +659,59 @@ public class RankProfile implements Serializable, Cloneable { } private void compileThis(QueryProfileRegistry queryProfiles, ImportedModels importedModels) { - parseExpressions(); - checkNameCollisions(getMacros(), getConstants()); + checkNameCollisions(getFunctions(), getConstants()); ExpressionTransforms expressionTransforms = new ExpressionTransforms(); - // Macro compiling first pass: compile inline macros without resolving other macros - Map<String, Macro> inlineMacros = compileMacros(getInlineMacros(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); + // Function compiling first pass: compile inline functions without resolving other functions + Map<String, RankingExpressionFunction> inlineFunctions = + compileFunctions(getInlineFunctions(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); - // Macro compiling second pass: compile all macros and insert previously compiled inline macros - macros = compileMacros(getMacros(), queryProfiles, importedModels, inlineMacros, expressionTransforms); + // Function compiling second pass: compile all functions and insert previously compiled inline functions + functions = compileFunctions(getFunctions(), queryProfiles, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); } - private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) { - for (Map.Entry<String, Macro> macroEntry : macros.entrySet()) { - if (constants.get(macroEntry.getKey()) != null) - throw new IllegalArgumentException("Cannot have both a constant and macro named '" + - macroEntry.getKey() + "'"); + private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { + for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) { + if (constants.get(functionEntry.getKey()) != null) + throw new IllegalArgumentException("Cannot have both a constant and function named '" + + functionEntry.getKey() + "'"); } } - private Map<String, Macro> getInlineMacros() { - return getMacros().entrySet().stream().filter(x -> x.getValue().getInline()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + private Map<String, RankingExpressionFunction> getInlineFunctions() { + return getFunctions().entrySet().stream().filter(x -> x.getValue().inline()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private Map<String, Macro> compileMacros(Map<String, Macro> macros, - QueryProfileRegistry queryProfiles, - ImportedModels importedModels, - Map<String, Macro> inlineMacros, - ExpressionTransforms expressionTransforms) { - Map<String, Macro> compiledMacros = new LinkedHashMap<>(); - for (Map.Entry<String, Macro> entry : macros.entrySet()) { - Macro macro = entry.getValue().clone(); - RankingExpression exp = compile(macro.getRankingExpression(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - macro.setRankingExpression(exp); - compiledMacros.put(entry.getKey(), macro); + private Map<String, RankingExpressionFunction> compileFunctions(Map<String, RankingExpressionFunction> functions, + QueryProfileRegistry queryProfiles, + ImportedModels importedModels, + Map<String, RankingExpressionFunction> inlineFunctions, + ExpressionTransforms expressionTransforms) { + Map<String, RankingExpressionFunction> compiledFunctions = new LinkedHashMap<>(); + for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { + RankingExpressionFunction rankingExpressionFunction = entry.getValue(); + RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withBody(compiled)); } - return compiledMacros; + return compiledFunctions; } private RankingExpression compile(RankingExpression expression, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map<String, Value> constants, - Map<String, Macro> inlineMacros, + Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { if (expression == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, importedModels, constants, - inlineMacros); + inlineFunctions); expression = expressionTransforms.transform(expression, context); for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); @@ -785,9 +724,9 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public TypeContext<Reference> typeContext(QueryProfileRegistry queryProfiles) { - MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() - .map(Macro::asExpressionFunction) - .collect(Collectors.toList())); + MapEvaluationTypeContext context = new MapEvaluationTypeContext(getFunctions().values().stream() + .map(RankingExpressionFunction::function) + .collect(Collectors.toList())); // Add small and large constants, respectively getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); @@ -854,11 +793,11 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ private boolean isIndexLevel; - private Type(String name) { + Type(String name) { this(name,false); } - private Type(String name,boolean isIndexLevel) { + Type(String name,boolean isIndexLevel) { this.name = name; this.isIndexLevel=isIndexLevel; } @@ -866,7 +805,7 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ public boolean isIndexLevel() { return isIndexLevel; } - /** @return The name of this type */ + /** Returns the name of this type */ public String getName() { return name; } @@ -899,10 +838,12 @@ public class RankProfile implements Serializable, Cloneable { } } + @Override public int hashCode() { return fieldName.hashCode() + 17 * type.hashCode(); } + @Override public boolean equals(Object object) { if (!(object instanceof RankSetting)) { return false; @@ -913,6 +854,7 @@ public class RankProfile implements Serializable, Cloneable { type.equals(other.type); } + @Override public String toString() { return type + " setting " + fieldName + ": " + value; } @@ -936,7 +878,7 @@ public class RankProfile implements Serializable, Cloneable { @Override public int hashCode() { - return name.hashCode() + 17*value.hashCode(); + return name.hashCode() + 17 * value.hashCode(); } @Override @@ -953,73 +895,32 @@ public class RankProfile implements Serializable, Cloneable { } - /** - * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro - */ - public static class Macro implements Serializable, Cloneable { + /** A function in a rank profile */ + public static class RankingExpressionFunction { - private final String name; - private String textualExpression = null; - private RankingExpression expression = null; - private List<String> formalParams = new ArrayList<>(); + private final ExpressionFunction function; - /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ + /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ private final boolean inline; - public Macro(String name, boolean inline) { - this.name = name; + public RankingExpressionFunction(ExpressionFunction function, boolean inline) { + this.function = function; this.inline = inline; } - public void addParam(String name) { - formalParams.add(name); - } - - public List<String> getFormalParams() { - return formalParams; - } - - public String getTextualExpression() { - return textualExpression; - } - - public void setTextualExpression(String textualExpression) { - this.textualExpression = textualExpression; - } - - public void setRankingExpression(RankingExpression expr) { - this.expression=expr; - } - - public RankingExpression getRankingExpression() { - return expression; - } - - public String getName() { - return name; - } + public ExpressionFunction function() { return function; } - public boolean getInline() { - return inline && formalParams.size() == 0; // only inline no-arg macros; + public boolean inline() { + return inline && function.arguments().isEmpty(); // only inline no-arg functions; } - public ExpressionFunction asExpressionFunction() { - return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression()); - } - - @Override - public Macro clone() { - try { - return (Macro)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Won't happen", e); - } + public RankingExpressionFunction withBody(RankingExpression expression) { + return new RankingExpressionFunction(function.withBody(expression), inline); } @Override public String toString() { - return "macro " + getName() + ": " + expression; + return "function " + function; } } @@ -1106,6 +1007,7 @@ public class RankProfile implements Serializable, Cloneable { public Map<String, String> getTypes() { return Collections.unmodifiableMap(types); } + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 592efcc2d85..3c2ebc058ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -15,7 +15,6 @@ import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.parser.SDParser; import com.yahoo.searchdefinition.parser.SimpleCharStream; import com.yahoo.searchdefinition.parser.TokenMgrException; -import com.yahoo.searchdefinition.processing.MinimalProcessing; import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -40,17 +39,25 @@ public class SearchBuilder { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); private List<Search> searchList = new LinkedList<>(); - private ApplicationPackage app = null; + private ApplicationPackage app; private boolean isBuilt = false; private DocumentModel model = new DocumentModel(); private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; + /** True to build the document aspect only, skipping instantiation of rank profiles */ + private final boolean documentsOnly; + /** For testing only */ public SearchBuilder() { this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry()); } + /** Used for generating documents for typed access to document fields in Java */ + public SearchBuilder(boolean documentsOnly) { + this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); + } + /** For testing only */ public SearchBuilder(ApplicationPackage app) { this(app, new RankProfileRegistry(), new QueryProfileRegistry()); @@ -69,9 +76,16 @@ public class SearchBuilder { public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + this(app, rankProfileRegistry, queryProfileRegistry, false); + } + public SearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { this.app = app; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -151,7 +165,7 @@ public class SearchBuilder { Search search; SimpleCharStream stream = new SimpleCharStream(str); try { - search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir); + search = new SDParser(stream, deployLogger, app, rankProfileRegistry, documentsOnly).search(docTypeMgr, searchDefDir); } catch (TokenMgrException e) { throw new ParseException("Unknown symbol: " + e.getMessage()); } catch (ParseException pe) { @@ -165,9 +179,9 @@ public class SearchBuilder { * {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed search objects used in unit tests. * - * @param rawSearch The object to import. - * @return The name of the imported object. - * @throws IllegalArgumentException Thrown if the given search object has already been processed. + * @param rawSearch the object to import. + * @return the name of the imported object. + * @throws IllegalArgumentException if the given search object has already been processed. */ public String importRawSearch(Search rawSearch) { if (rawSearch.getName() == null) { @@ -251,15 +265,15 @@ public class SearchBuilder { * #build()} method so that subclasses can choose not to build anything. */ protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { - new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate); + new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate, documentsOnly); } /** * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object * built. This method will never return null. * - * @return The build object. - * @throws IllegalStateException Thrown if there is not exactly one search. + * @return the built object + * @throws IllegalStateException if there is not exactly one search. */ public Search getSearch() { if ( ! isBuilt) throw new IllegalStateException("Searches not built."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java index c1b05c0fcdf..49b7ad621af 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java @@ -1,8 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.collections.Pair; + +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -54,22 +58,18 @@ public class FieldRankSettings { table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY)); } - public Map<String,String> deriveRankProperties(int part) { - Map<String,String> ret = new LinkedHashMap<>(); - int i = part; - for (Iterator<NativeTable> itr = tables.values().iterator(); itr.hasNext(); ++i) { - NativeTable table = itr.next(); - if (isFieldMatchTable(table)) { - ret.put("nativeFieldMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } - if (isAttributeMatchTable(table)) { - ret.put("nativeAttributeMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } - if (isProximityTable(table)) { - ret.put("nativeProximity." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } + public List<Pair<String, String>> deriveRankProperties() { + List<Pair<String, String>> properties = new ArrayList<>(); + for (Iterator<NativeTable> i = tables.values().iterator(); i.hasNext();) { + NativeTable table = i.next(); + if (isFieldMatchTable(table)) + properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isAttributeMatchTable(table)) + properties.add(new Pair<>("nativeAttributeMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isProximityTable(table)) + properties.add(new Pair<>("nativeProximity." + table.getType().getName() + "." + fieldName, table.getName())); } - return ret; + return properties; } public String toString() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 43cc2fad285..c041d5c6a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -50,17 +50,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, AttributeFields attributeFields) { this.name = rankProfile.getName(); - compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive())); - } - - private List<Pair<String, String>> removePartFromKeys(Map<String, String> map) { - ImmutableList.Builder<Pair<String, String>> replaced = new ImmutableList.Builder<>(); - for (Map.Entry<String, String> e : map.entrySet()) { - String key = e.getKey().replaceFirst(".part\\d+$", ""); - String val = e.getValue(); - replaced.add(new Pair<>(key, val)); - } - return replaced.build(); + compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive()); } private Compressor.Compression compress(List<Pair<String, String>> properties) { @@ -185,57 +175,57 @@ public class RawRankProfile implements RankProfilesConfig.Producer { rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros()); + derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); } - private void derivePropertiesAndSummaryFeaturesFromMacros(Map<String, RankProfile.Macro> macros) { - if (macros.isEmpty()) return; - Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>(); - for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) { - expressionMacros.put(macro.getKey(), macro.getValue().asExpressionFunction()); + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { + if (functions.isEmpty()) return; + Map<String, ExpressionFunction> expressionFunctions = new LinkedHashMap<>(); + for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : functions.entrySet()) { + expressionFunctions.put(function.getKey(), function.getValue().function()); } - Map<String, String> macroProperties = new LinkedHashMap<>(); - macroProperties.putAll(deriveMacroProperties(expressionMacros)); + Map<String, String> functionProperties = new LinkedHashMap<>(); + functionProperties.putAll(deriveFunctionProperties(expressionFunctions)); if (firstPhaseRanking != null) { - macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } if (secondPhaseRanking != null) { - macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } - for (Map.Entry<String, String> e : macroProperties.entrySet()) { + + for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties); - replaceMacroSummaryFeatures(context); + SerializationContext context = new SerializationContext(expressionFunctions.values(), null, functionProperties); + replaceFunctionSummaryFeatures(context); } - private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) { - SerializationContext context = new SerializationContext(eMacros); - for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) { + private Map<String, String> deriveFunctionProperties(Map<String, ExpressionFunction> functions) { + SerializationContext context = new SerializationContext(functions); + for (Map.Entry<String, ExpressionFunction> e : functions.entrySet()) { String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression); - } return context.serializedFunctions(); } - private void replaceMacroSummaryFeatures(SerializationContext context) { + private void replaceFunctionSummaryFeatures(SerializationContext context) { if (summaryFeatures == null) return; - Map<String, ReferenceNode> macroSummaryFeatures = new LinkedHashMap<>(); + Map<String, ReferenceNode> functionSummaryFeatures = new LinkedHashMap<>(); for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) { ReferenceNode referenceNode = i.next(); - // Is the feature a macro? + // Is the feature a function? if (context.getFunction(referenceNode.getName()) != null) { context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), referenceNode.toString(new StringBuilder(), context, null, null).toString()); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); - macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); + functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block } } - // Then, replace the summary features that were macros - for (Map.Entry<String, ReferenceNode> e : macroSummaryFeatures.entrySet()) { + // Then, replace the summary features that were functions + for (Map.Entry<String, ReferenceNode> e : functionSummaryFeatures.entrySet()) { summaryFeatures.add(e.getValue()); } } @@ -300,17 +290,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return settings; } - /** - * Derives the properties this produces. Equal keys are suffixed with .part0 etc, remove when exporting to file - * - * @return map of the derived properties - */ - public Map<String, String> derive() { - Map<String, String> properties = new LinkedHashMap<>(); - int i = 0; + /** Derives the properties this produces */ + public List<Pair<String, String>> derive() { + List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) { - // Could have been set by macro expansion. Set expressions, then skip this property. + // Could have been set by function expansion. Set expressions, then skip this property. try { firstPhaseRanking = new RankingExpression(property.getValue()); } catch (ParseException e) { @@ -325,100 +310,92 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } } else { - properties.put(property.getName() + ".part" + i, property.getValue()); - i++; + properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.putAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); - properties.putAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); for (FieldRankSettings settings : fieldRankSettings.values()) { - properties.putAll(settings.deriveRankProperties(i)); + properties.addAll(settings.deriveRankProperties()); } - i = 0; for (RankProfile.RankProperty property : boostAndWeightRankProperties) { - properties.put(property.getName() + ".part" + i, property.getValue()); - i++; + properties.add(new Pair<>(property.getName(), property.getValue())); } - i = 0; for (ReferenceNode feature : summaryFeatures) { - properties.put(summaryFeatureFefPropertyPrefix + ".part" + i, feature.toString()); - i++; + properties.add(new Pair<>(summaryFeatureFefPropertyPrefix, feature.toString())); } - i = 0; for (ReferenceNode feature : rankFeatures) { - properties.put(rankFeatureFefPropertyPrefix + ".part" + i, feature.toString()); - i++; + properties.add(new Pair<>(rankFeatureFefPropertyPrefix, feature.toString())); } if (numThreadsPerSearch > 0) { - properties.put("vespa.matching.numthreadspersearch", numThreadsPerSearch + ""); + properties.add(new Pair<>("vespa.matching.numthreadspersearch", numThreadsPerSearch + "")); } if (minHitsPerThread > 0) { - properties.put("vespa.matching.minhitsperthread", minHitsPerThread + ""); + properties.add(new Pair<>("vespa.matching.minhitsperthread", minHitsPerThread + "")); } if (numSearchPartitions >= 0) { - properties.put("vespa.matching.numsearchpartitions", numSearchPartitions + ""); + properties.add(new Pair<>("vespa.matching.numsearchpartitions", numSearchPartitions + "")); } if (termwiseLimit < 1.0) { - properties.put("vespa.matching.termwise_limit", termwiseLimit + ""); + properties.add(new Pair<>("vespa.matching.termwise_limit", termwiseLimit + "")); } if (matchPhaseSettings != null) { - properties.put("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute()); - properties.put("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + ""); - properties.put("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + ""); - properties.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + ""); - properties.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + ""); - properties.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + ""); + properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); if (diversitySettings != null) { - properties.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute()); - properties.put("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups())); - properties.put("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor())); - properties.put("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy())); + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); } } if (rerankCount > -1) { - properties.put("vespa.hitcollector.heapsize", rerankCount + ""); + properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); } if (keepRankCount > -1) { - properties.put("vespa.hitcollector.arraysize", keepRankCount + ""); + properties.add(new Pair<>("vespa.hitcollector.arraysize", keepRankCount + "")); } if (rankScoreDropLimit > -Double.MAX_VALUE) { - properties.put("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + ""); + properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); } if (ignoreDefaultRankFeatures) { - properties.put("vespa.dump.ignoredefaultfeatures", String.valueOf(true)); + properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } Iterator filterFieldsIterator = filterFields.iterator(); while (filterFieldsIterator.hasNext()) { String fieldName = (String) filterFieldsIterator.next(); - properties.put("vespa.isfilterfield." + fieldName + ".part42", String.valueOf(true)); + properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); } for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { - properties.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue()); + properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) { - properties.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue()); + properties.add(new Pair<>("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue())); } if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties"); return properties; } - private Map<String, String> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { - Map<String, String> ret = new LinkedHashMap<>(); - if (expression == null) { - return ret; - } + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { + List<Pair<String, String>> properties = new ArrayList<>(); + if (expression == null) return properties; + String name = expression.getName(); - if ("".equals(name)) { + if ("".equals(name)) name = phase; - } + if (expression.getRoot() instanceof ReferenceNode) { - ret.put("vespa.rank." + phase, expression.getRoot().toString()); + properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { - ret.put("vespa.rank." + phase, "rankingExpression(" + name + ")"); - ret.put("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString()); + properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); + properties.add(new Pair<>("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString())); } - return ret; + return properties; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index a639165d297..cbabfffb7a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -28,8 +28,8 @@ public class ExpressionTransforms { new XgboostFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), - new MacroInliner(), - new MacroShadower(), + new FunctionInliner(), + new FunctionShadower(), new TensorTransformer(), new Simplifier()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java index 6aef39db4da..c15ef20a455 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java @@ -8,11 +8,11 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; /** - * Inlines macros in ranking expressions + * Inlines functions in ranking expressions * * @author bratseth */ -public class MacroInliner extends ExpressionTransformer<RankProfileTransformContext> { +public class FunctionInliner extends ExpressionTransformer<RankProfileTransformContext> { @Override public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { @@ -24,9 +24,9 @@ public class MacroInliner extends ExpressionTransformer<RankProfileTransformCont } private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) { - RankProfile.Macro macro = context.inlineMacros().get(feature.getName()); - if (macro == null) return feature; - return transform(macro.getRankingExpression().getRoot(), context); // inline recursively and return + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineFunctions().get(feature.getName()); + if (rankingExpressionFunction == null) return feature; + return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java index 758d2b2a87d..74b6471d291 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java @@ -10,19 +10,19 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; /** - * Transforms function nodes to reference nodes if a macro shadows a built-in function. - * This has the effect of allowing macros to redefine built-in functions. - * Another effect is that we can more or less add built-in functions over time - * without fear of breaking existing users' macros with the same name. + * Transforms function nodes to reference nodes if a rank profile function shadows a built-in function. + * This has the effect of allowing rank profile functions to redefine built-in functions. + * Another effect is that we can add built-in functions over time + * without fear of breaking existing users' functions with the same name. * - * However, there is a (largish) caveat. If a user has a macro with a certain number + * However, there is a (largish) caveat. If a user has a function with a certain number * of arguments, and we add in a built-in function with a different arity, * this will cause parse errors as the Java parser gives precedence to * built-in functions. * * @author lesters */ -public class MacroShadower extends ExpressionTransformer<RankProfileTransformContext> { +public class FunctionShadower extends ExpressionTransformer<RankProfileTransformContext> { @Override public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { @@ -43,16 +43,14 @@ public class MacroShadower extends ExpressionTransformer<RankProfileTransformCon private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) { String name = function.getFunction().toString(); - RankProfile.Macro macro = context.rankProfile().getMacros().get(name); - if (macro == null) { + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().getFunctions().get(name); + if (rankingExpressionFunction == null) { return transformChildren(function, context); } int functionArity = function.getFunction().arity(); - int macroArity = macro.getFormalParams() != null ? macro.getFormalParams().size() : 0; - if (functionArity != macroArity) { + if (functionArity != rankingExpressionFunction.function().arguments().size()) return transformChildren(function, context); - } ReferenceNode node = new ReferenceNode(name, function.children(), null); return transformChildren(node, context); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 40c3b997daa..2fe2dacf2ce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -20,25 +20,25 @@ public class RankProfileTransformContext extends TransformContext { private final RankProfile rankProfile; private final QueryProfileRegistry queryProfiles; private final ImportedModels importedModels; - private final Map<String, RankProfile.Macro> inlineMacros; + private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions; private final Map<String, String> rankProperties = new HashMap<>(); public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map<String, Value> constants, - Map<String, RankProfile.Macro> inlineMacros) { + Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) { super(constants); this.rankProfile = rankProfile; this.queryProfiles = queryProfiles; this.importedModels = importedModels; - this.inlineMacros = inlineMacros; + this.inlineFunctions = inlineFunctions; } public RankProfile rankProfile() { return rankProfile; } public QueryProfileRegistry queryProfiles() { return queryProfiles; } public ImportedModels importedModels() { return importedModels; } - public Map<String, RankProfile.Macro> inlineMacros() { return inlineMacros; } + public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; } public Map<String, String> rankProperties() { return rankProperties; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java index cf123d0f7c1..fefb54a7fe3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -27,7 +27,7 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.allImportedFields() .flatMap(this::getSummaryFieldsForImportedField) .forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index e0d32ea8ccd..803a6c5ab40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,7 +26,7 @@ public class AddExtraFieldsToDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = search.getDocument(); if (document != null) { for (Field field : search.extraFieldList()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java index 9ec596792fa..94589d94255 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java @@ -20,7 +20,7 @@ public class AttributeProperties extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { String fieldName = field.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index a95f4264dc6..23257e5eafd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -23,7 +23,7 @@ public class AttributesImplicitWord extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (fieldImplicitlyWordMatch(field)) { field.getMatching().setType(Matching.Type.WORD); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java index a3c4c97cf31..b9be30e8485 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java @@ -22,7 +22,7 @@ public class Bolding extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { for (SummaryField summary : field.getSummaryFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java index 37d60c1d32e..a0c4c8adb2d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document + * * @author Vegard Havdal */ public class BuiltInFieldSets extends Processor { @@ -23,7 +24,7 @@ public class BuiltInFieldSets extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { addDocumentFieldSet(); addSearchFieldSet(); // "Hook" the field sets on search onto the document types, since we will include them diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index d7b688be203..ad862ef767f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -38,7 +38,7 @@ public class CreatePositionZCurve extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { DataType fieldType = field.getDataType(); if ( ! isSupportedPositionType(fieldType)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java index 7cc9b4e9b52..b34db6febd5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java @@ -15,7 +15,7 @@ public class DeprecateAttributePrefetch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java index 861ebad7085..076161a8584 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -23,7 +23,7 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 6b78da2146b..029892cba1c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -18,8 +18,9 @@ public class DiversitySettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 59bc3dc66f4..a7c0ebd4a07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -26,7 +26,7 @@ public class ExactMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { Matching.Type matching = field.getMatching().getType(); if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java index 9cfac625da5..cd33e4434e7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java @@ -21,7 +21,7 @@ public class FastAccessValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String invalidAttributes = search.allFields() @@ -29,7 +29,7 @@ public class FastAccessValidator extends Processor { .filter(FastAccessValidator::isIncompatibleAttribute) .map(Attribute::getName) .collect(Collectors.joining(", ")); - if (!invalidAttributes.isEmpty()) { + if ( ! invalidAttributes.isEmpty()) { throw new IllegalArgumentException( String.format( "For search '%s': The following attributes have a type that is incompatible with fast-access: %s. " + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java index 15c11589245..4ef9a9733d5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java @@ -24,7 +24,7 @@ public class FieldSetValidity extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index 0c75314ffa2..adb8ab62aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -26,7 +26,9 @@ public class FilterFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + for (SDField f : search.allConcreteFields()) { if (f.getRanking().isFilter()) { filterField(f.getName()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index b51524b7e62..1f795458875 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -28,7 +28,7 @@ public class ImplicitSummaries extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary = search.getSummary("default"); if (defaultSummary == null) { defaultSummary = new DocumentSummary("default"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index 7464f574255..0d99c698aca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -21,7 +21,7 @@ public class ImplicitSummaryFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary docsum : search.getSummaries().values()) { addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index 9f03cdf4b5e..a3efd086c6a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -33,7 +33,7 @@ public class ImportedFieldsResolver extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); search.setImportedFields(new ImportedFields(importedFields)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java index 018183b91d8..210a8e7009c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java @@ -23,7 +23,7 @@ public class IndexFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java index 0740029de90..41355a76f47 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java @@ -24,7 +24,7 @@ public class IndexSettingsNonFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java index 419268468c2..aeab2bb6638 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java @@ -27,7 +27,7 @@ public class IndexingInputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java index 6f04184c512..11d69bf6c75 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java @@ -29,7 +29,7 @@ public class IndexingOutputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index b73151768fd..27520647e3b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -25,7 +25,7 @@ public class IndexingValidation extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; VerificationContext context = new VerificationContext(new MyAdapter(search)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index f3907ae68cb..72777b7dfb4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -22,7 +22,7 @@ public class IndexingValues extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (Field field : search.getDocument().fieldSet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java index c119dc2660b..baaf145dbce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java @@ -30,7 +30,7 @@ public class IntegerIndex2Attribute extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { if (field.getIndex(field.getName()) != null diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java index 507a0e87cff..fe94ac9849f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java @@ -31,7 +31,7 @@ public class LiteralBoost extends Processor { /** Adds extra search fields and indices to express literal boosts */ @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { checkRankModifierRankType(search); addLiteralBoostsToFields(search); reduceFieldLiteralBoosts(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java index f853b99bbb2..0daf7265daa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java @@ -25,7 +25,7 @@ public class MakeAliases extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { List<String> usedAliases = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java index 2c43a65da99..6f67c22d9d2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -34,7 +34,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary=search.getSummary("default"); for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) { if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java index fe584fa41c0..ff8a5c2eb0b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java @@ -30,7 +30,7 @@ public class MatchConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map<String, Matching.Type> types = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index 479384e09ef..b1728b9bd89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -20,8 +20,9 @@ public class MatchPhaseSettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java deleted file mode 100644 index 0fee844ecbc..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MinimalProcessing.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.yahoo.searchdefinition.processing; - -import java.util.Collection; - -public class MinimalProcessing extends Processing { - @Override - protected Collection<ProcessorFactory> createProcessorFactories() { - return minimalSetOfProcessors(); - } -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java index 45e018f8fc3..a52a8ab74e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java @@ -31,7 +31,7 @@ public class MultifieldIndexHarmonizer extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { populateIndexToFields(search); resolveAllConflicts(search); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java index 9bcb3929b3d..4d8f0032a78 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java @@ -15,12 +15,12 @@ public class MutableAttributes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { - if (!field.isExtraField() && field.getAttributes().containsKey(field.getName())) { + if ( ! field.isExtraField() && field.getAttributes().containsKey(field.getName())) { if (field.getAttributes().get(field.getName()).isMutable()) { throw new IllegalArgumentException("Field '" + field.getName() + "' in '" + search.getDocument().getName() + - "' can not be marked mutable as it is inside the document clause."); + "' can not be marked mutable as it is inside the document clause."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java index cdfba54ee5a..3f3fc11380b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java @@ -26,7 +26,7 @@ public class NGramMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType().equals(Matching.Type.GRAM)) implementGramMatch(search, field, validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java index c89c709ffbf..8f2a29abcb6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -22,7 +22,7 @@ public class OptimizeIlscript extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 3583c4a0162..79f19efe422 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -33,7 +33,7 @@ public class PredicateProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { if (validate && field.doesIndexing()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 9b4b74d07d8..8c8c32389e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -20,7 +20,7 @@ import java.util.List; */ public class Processing { - protected Collection<ProcessorFactory> minimalSetOfProcessors() { + private Collection<ProcessorFactory> processors() { return Arrays.asList( SearchMustHaveDocument::new, UrlFieldValidator::new, @@ -74,24 +74,13 @@ public class Processing { RankProfileTypeSettingsProcessor::new, ReferenceFieldsProcessor::new, FastAccessValidator::new, - ReservedMacroNames::new); - } - - private Collection<ProcessorFactory> extendedSetOfProcessors() { - return Arrays.asList( + ReservedFunctionNames::new, RankingExpressionTypeValidator::new, - - // These should be last. + // These should be last: IndexingValidation::new, IndexingValues::new); } - protected Collection<ProcessorFactory> createProcessorFactories() { - List<ProcessorFactory> processorFactories = new ArrayList<>(minimalSetOfProcessors()); - processorFactories.addAll(extendedSetOfProcessors()); - return processorFactories; - } - /** * Runs all search processors on the given {@link Search} object. These will modify the search object, <b>possibly * exchanging it with another</b>, as well as its document types. @@ -102,17 +91,17 @@ public class Processing { * @param queryProfiles The query profiles contained in the application this search is part of. */ public void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, boolean validate) - { - Collection<ProcessorFactory> factories = createProcessorFactories(); + QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { + Collection<ProcessorFactory> factories = processors(); search.process(); factories.stream() .map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles)) - .forEach(processor -> processor.process(validate)); + .forEach(processor -> processor.process(validate, documentsOnly)); } @FunctionalInterface public interface ProcessorFactory { Processor create(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index b938e40d9a2..6bfd0ef29ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -53,8 +53,10 @@ public abstract class Processor { * @param validate true to throw exceptions on validation errors, false to make the best possible effort * at completing processing without throwing an exception. * If we are not validating, emitting warnings have no effect and can (but must not) be skipped. + * @param documentsOnly true to skip processing (including validation, regardless of the validate setting) + * of aspects not relating to document definitions (e.g rank profiles) */ - public abstract void process(boolean validate); + public abstract void process(boolean validate, boolean documentsOnly); /** * Convenience method for adding a no-strings-attached implementation field for a regular field diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index 81455991cc9..102d1910360 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -33,8 +33,9 @@ public class RankingExpressionTypeValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { @@ -48,7 +49,6 @@ public class RankingExpressionTypeValidator extends Processor { /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */ private void validate(RankProfile profile) { - profile.parseExpressions(); TypeContext context = profile.typeContext(queryProfiles); profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java index 76afcd5d520..0418538922b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java @@ -28,7 +28,7 @@ public class ReferenceFieldsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { clearSummaryAttributeAspectForConcreteFields(); clearSummaryAttributeAspectForExplicitSummaryFields(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java index f2aa31bb9c3..805cbaced0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java @@ -28,7 +28,7 @@ public class ReservedDocumentNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String docName = search.getDocument().getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java index adcebed9254..d7099215f17 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -13,29 +13,30 @@ import java.util.Set; import java.util.logging.Level; /** - * Issues a warning if some macro has a reserved name. This is not necessarily - * an error, as a macro can shadow a built-in function. + * Issues a warning if some function has a reserved name. This is not necessarily + * an error, as a rank profile function can shadow a built-in function. * * @author lesters */ -public class ReservedMacroNames extends Processor { +public class ReservedFunctionNames extends Processor { private static Set<String> reservedNames = getReservedNames(); - public ReservedMacroNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public ReservedFunctionNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rp : rankProfileRegistry.all()) { - for (String macroName : rp.getMacros().keySet()) { - if (reservedNames.contains(macroName)) { - deployLogger.log(Level.WARNING, "Macro \"" + macroName + "\" " + - "in rank profile \"" + rp.getName() + "\" " + - "has a reserved name. This might mean that the macro shadows " + + for (String functionName : rp.getFunctions().keySet()) { + if (reservedNames.contains(functionName)) { + deployLogger.log(Level.WARNING, "Function '" + functionName + "' " + + "in rank profile '" + rp.getName() + "' " + + "has a reserved name. This might mean that the function shadows " + "the built-in function with the same name." ); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java index 403de1253b4..2d8eaff7762 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java @@ -19,7 +19,7 @@ public class SearchMustHaveDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; if (search.getDocument() == null) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java index a0c884d25f9..8a4795c4dd2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java @@ -24,7 +24,8 @@ public class SetLanguage extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; List<String> textFieldsWithoutLanguage = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java index a19ea8d7068..715828f808a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java @@ -20,7 +20,7 @@ public class SetRankTypeEmptyOnFilters extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getRanking().isFilter()) { field.setRankType(RankType.EMPTY); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java index 6426c724a07..defcf761649 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java @@ -21,7 +21,7 @@ public class SortingSettings extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java index d56b0272f06..e133f88f45c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java @@ -16,7 +16,7 @@ public class StringSettingsOnNonStringFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index a952d3732b3..d2c4968ca26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -26,7 +26,7 @@ public class SummaryConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary summary : search.getSummaries().values()) { if (summary.getName().equals("default")) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java index 647a433f201..b8d170c07f6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java @@ -26,7 +26,7 @@ public class SummaryDynamicStructsArrays extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java index 7bcbd9a267c..9b51c7c473e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -22,7 +22,7 @@ public class SummaryFieldsMustHaveValidSource extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (DocumentSummary summary : search.getSummaries().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java index 23569cf39ae..678d5324e38 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java @@ -26,7 +26,7 @@ public class SummaryNamesFieldCollisions extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java index 177fc7f2326..79b7a6067b9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java @@ -25,7 +25,7 @@ public class TagType extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) implementTagType(field); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 08571168336..8e54d7c00d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -22,7 +22,7 @@ public class TensorFieldProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index 645ed5121ea..74f30d6a730 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -34,7 +34,7 @@ public class TextMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType() != Matching.Type.TEXT) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index ac376982cfa..d81fdf70d20 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -28,7 +28,7 @@ public class UriHack extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing()) { DataType fieldType = field.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java index ed813b42fff..c6b83349691 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java @@ -18,7 +18,7 @@ public class UrlFieldValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java index 54ad9f13f6f..21b7f1d2675 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -28,7 +28,7 @@ public class ValidateFieldTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String searchName = search.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java index a0b204a25f2..408d60e1cff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -21,7 +21,7 @@ public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Matching defaultMatching = new Matching(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java index 892bcdad12c..13fe3f24d69 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -25,7 +25,7 @@ public class WordMatch extends Processor { super(search, deployLogger, rankProfileRegistry, queryProfiles); } - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if ( ! field.getMatching().getType().equals(Matching.Type.WORD)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index cc1638347f6..ec4cbdfe58b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.Optional; /** - * Class that processes a search instance and sets type settings on all rank profiles. + * This processes a search instance and sets type settings on all rank profiles. * * Currently, type settings are limited to the type of tensor attribute fields and tensor query features. * @@ -35,7 +35,9 @@ public class RankProfileTypeSettingsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + processAttributeFields(); processImportedFields(); processQueryProfileTypes(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 282e5a29962..4b70b1b5ae2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -32,6 +32,7 @@ import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel; @@ -236,7 +237,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), model.name(), profile, queryProfiles, model); for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue()); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } @@ -248,7 +249,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue()); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index e2236feb336..adf5c81283e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -13,6 +13,7 @@ import com.yahoo.searchdefinition.FeatureNames; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; @@ -139,7 +140,7 @@ public class ConvertedModel { public ExpressionNode expression(FeatureArguments arguments, RankProfileTransformContext context) { RankingExpression expression = selectExpression(arguments); if (sourceModel.isPresent()) // we can verify - verifyRequiredMacros(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); + verifyRequiredFunctions(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); return expression.getRoot(); } @@ -183,41 +184,41 @@ public class ConvertedModel { QueryProfileRegistry queryProfiles, ModelStore store) { // Add constants - Set<String> constantsReplacedByMacros = new HashSet<>(); + Set<String> constantsReplacedByFunctions = new HashSet<>(); model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, queryProfiles, - constantsReplacedByMacros, k, v)); + constantsReplacedByFunctions, k, v)); - // Add macros - addGeneratedMacros(model, profile); + // Add functions + addGeneratedFunctions(model, profile); // Add expressions Map<String, RankingExpression> expressions = new HashMap<>(); for (Pair<String, RankingExpression> output : model.outputExpressions()) { addExpression(output.getSecond(), output.getFirst(), - constantsReplacedByMacros, + constantsReplacedByFunctions, model, store, profile, queryProfiles, expressions); } - // Transform and save macro - must come after reading expressions due to optimization transforms - // and must use the macro expression added to the profile, which may differ from the one saved in the model, + // Transform and save function - must come after reading expressions due to optimization transforms + // and must use the function expression added to the profile, which may differ from the one saved in the model, // after rewrite - model.macros().forEach((k, v) -> transformGeneratedMacro(store, constantsReplacedByMacros, k, - profile.getMacros().get(k).getRankingExpression())); + model.functions().forEach((k, v) -> transformGeneratedFunction(store, constantsReplacedByFunctions, k, + profile.getFunctions().get(k).function().getBody())); return expressions; } private static void addExpression(RankingExpression expression, String expressionName, - Set<String> constantsReplacedByMacros, + Set<String> constantsReplacedByFunctions, ImportedModel model, ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, Map<String, RankingExpression> expressions) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); reduceBatchDimensions(expression, model, profile, queryProfiles); store.writeExpression(expressionName, expression); expressions.put(expressionName, expression); @@ -232,8 +233,8 @@ public class ConvertedModel { profile.rankingConstants().add(constant); } - for (Pair<String, RankingExpression> macro : store.readMacros()) { - addGeneratedMacroToProfile(profile, macro.getFirst(), macro.getSecond()); + for (Pair<String, RankingExpression> function : store.readFunctions()) { + addGeneratedFunctionToProfile(profile, function.getFirst(), function.getSecond()); } return store.readExpressions(); @@ -247,16 +248,16 @@ public class ConvertedModel { private static void transformLargeConstant(ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, - Set<String> constantsReplacedByMacros, + Set<String> constantsReplacedByFunctions, String constantName, Tensor constantValue) { - RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName); - if (macroOverridingConstant != null) { - TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles)); - if ( ! macroType.equals(constantValue.type())) - throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + - typeMismatchExplanation(constantValue.type(), macroType)); - constantsReplacedByMacros.add(constantName); // will replace constant(constantName) by constantName later + RankProfile.RankingExpressionFunction rankingExpressionFunctionOverridingConstant = profile.getFunctions().get(constantName); + if (rankingExpressionFunctionOverridingConstant != null) { + TensorType functionType = rankingExpressionFunctionOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); + if ( ! functionType.equals(constantValue.type())) + throw new IllegalArgumentException("Function '" + constantName + "' replaces the constant with this name. " + + typeMismatchExplanation(constantValue.type(), functionType)); + constantsReplacedByFunctions.add(constantName); // will replace constant(constantName) by constantName later } else { Path constantPath = store.writeLargeConstant(constantName, constantValue); @@ -267,79 +268,75 @@ public class ConvertedModel { } } - private static void transformGeneratedMacro(ModelStore store, - Set<String> constantsReplacedByMacros, - String macroName, - RankingExpression expression) { + private static void transformGeneratedFunction(ModelStore store, + Set<String> constantsReplacedByFunctions, + String functionName, + RankingExpression expression) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); - store.writeMacro(macroName, expression); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); + store.writeFunction(functionName, expression); } - private static void addGeneratedMacroToProfile(RankProfile profile, String macroName, RankingExpression expression) { - if (profile.getMacros().containsKey(macroName)) { - if ( ! profile.getMacros().get(macroName).getRankingExpression().equals(expression)) - throw new IllegalArgumentException("Generated macro '" + macroName + "' already exists in " + profile + + private static void addGeneratedFunctionToProfile(RankProfile profile, String functionName, RankingExpression expression) { + if (profile.getFunctions().containsKey(functionName)) { + if ( ! profile.getFunctions().get(functionName).function().getBody().equals(expression)) + throw new IllegalArgumentException("Generated function '" + functionName + "' already exists in " + profile + " - with a different definition" + - ": Has\n" + profile.getMacros().get(macroName).getRankingExpression() + + ": Has\n" + profile.getFunctions().get(functionName).function().getBody() + "\nwant to add " + expression + "\n"); return; } - RankProfile.Macro macro = profile.addMacro(macroName, false); // TODO: Inline if only used once - macro.setRankingExpression(expression); - macro.setTextualExpression(expression.getRoot().toString()); + profile.addFunction(new ExpressionFunction(functionName, expression), false); // TODO: Inline if only used once } /** - * Verify that the macros referred in the given expression exists in the given rank profile, - * and return tensors of the types specified in requiredMacros. + * Verify that the functions referred in the given expression exists in the given rank profile, + * and return tensors of the types specified in requiredFunctions. */ - private static void verifyRequiredMacros(RankingExpression expression, ImportedModel model, - RankProfile profile, QueryProfileRegistry queryProfiles) { - Set<String> macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - TensorType requiredType = model.requiredMacros().get(macroName); - if (requiredType == null) continue; // Not a required macro - - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) - throw new IllegalArgumentException("Model refers input '" + macroName + - "' of type " + requiredType + " but this macro is not present in " + + private static void verifyRequiredFunctions(RankingExpression expression, ImportedModel model, + RankProfile profile, QueryProfileRegistry queryProfiles) { + Set<String> functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + TensorType requiredType = model.requiredFunctions().get(functionName); + if (requiredType == null) continue; // Not a required function + + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); + if (rankingExpressionFunction == null) + throw new IllegalArgumentException("Model refers input '" + functionName + + "' of type " + requiredType + " but this function is not present in " + profile); // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second // phase and summary features), as it may only resolve correctly given those bindings - // Or, probably better, annotate the macros with type constraints here and verify during general + // Or, probably better, annotate the functions with type constraints here and verify during general // type verification - TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); + TensorType actualType = rankingExpressionFunction.function().getBody().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) - throw new IllegalArgumentException("Model refers input '" + macroName + + throw new IllegalArgumentException("Model refers input '" + functionName + "' of type " + requiredType + - " which must be produced by a macro in the rank profile, but " + - "this macro references a feature which is not declared"); + " which must be produced by a function in the rank profile, but " + + "this function references a feature which is not declared"); if ( ! actualType.isAssignableTo(requiredType)) - throw new IllegalArgumentException("Model refers input '" + macroName + "'. " + + throw new IllegalArgumentException("Model refers input '" + functionName + "'. " + typeMismatchExplanation(requiredType, actualType)); } } private static String typeMismatchExplanation(TensorType requiredType, TensorType actualType) { - return "The required type of this is " + requiredType + ", but this macro returns " + actualType + + return "The required type of this is " + requiredType + ", but this function returns " + actualType + (actualType.rank() == 0 ? ". This is often due to missing declaration of query tensor features " + "in query profile types - see the documentation." : ""); } - /** - * Add the generated macros to the rank profile - */ - private static void addGeneratedMacros(ImportedModel model, RankProfile profile) { - model.macros().forEach((k, v) -> addGeneratedMacroToProfile(profile, k, v.copy())); + /** Add the generated functions to the rank profile */ + private static void addGeneratedFunctions(ImportedModel model, RankProfile profile) { + model.functions().forEach((k, v) -> addGeneratedFunctionToProfile(profile, k, v.copy())); } /** * Check if batch dimensions of inputs can be reduced out. If the input - * macro specifies that a single exemplar should be evaluated, we can + * function specifies that a single exemplar should be evaluated, we can * reduce the batch dimension out. */ private static void reduceBatchDimensions(RankingExpression expression, ImportedModel model, @@ -347,19 +344,19 @@ public class ConvertedModel { TypeContext<Reference> typeContext = profile.typeContext(queryProfiles); TensorType typeBeforeReducing = expression.getRoot().type(typeContext); - // Check generated macros for inputs to reduce - Set<String> macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - if ( ! model.macros().containsKey(macroName)) continue; + // Check generated functions for inputs to reduce + Set<String> functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + if ( ! model.functions().containsKey(functionName)) continue; - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) { - throw new IllegalArgumentException("Model refers to generated macro '" + macroName + - "but this macro is not present in " + profile); + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); + if (rankingExpressionFunction == null) { + throw new IllegalArgumentException("Model refers to generated function '" + functionName + + "but this function is not present in " + profile); } - RankingExpression macroExpression = macro.getRankingExpression(); - macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext)); + RankingExpression functionExpression = rankingExpressionFunction.function().getBody(); + functionExpression.setRoot(reduceBatchDimensionsAtInput(functionExpression.getRoot(), model, typeContext)); } // Check expression for inputs to reduce @@ -378,7 +375,7 @@ public class ConvertedModel { List<ExpressionNode> children = ((TensorFunctionNode)node).children(); if (children.size() == 1 && children.get(0) instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) children.get(0); - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(tensorFunction, typeContext); } } @@ -386,7 +383,7 @@ public class ConvertedModel { } if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) node; - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext); } } @@ -447,47 +444,47 @@ public class ConvertedModel { } /** - * If a constant c is overridden by a macro, we need to replace instances of "constant(c)" by "c" in expressions. + * If a constant c is overridden by a function, we need to replace instances of "constant(c)" by "c" in expressions. * This method does that for the given expression and returns the result. */ - private static RankingExpression replaceConstantsByMacros(RankingExpression expression, - Set<String> constantsReplacedByMacros) { - if (constantsReplacedByMacros.isEmpty()) return expression; + private static RankingExpression replaceConstantsByFunctions(RankingExpression expression, + Set<String> constantsReplacedByFunctions) { + if (constantsReplacedByFunctions.isEmpty()) return expression; return new RankingExpression(expression.getName(), - replaceConstantsByMacros(expression.getRoot(), constantsReplacedByMacros)); + replaceConstantsByFunctions(expression.getRoot(), constantsReplacedByFunctions)); } - private static ExpressionNode replaceConstantsByMacros(ExpressionNode node, Set<String> constantsReplacedByMacros) { + private static ExpressionNode replaceConstantsByFunctions(ExpressionNode node, Set<String> constantsReplacedByFunctions) { if (node instanceof ReferenceNode) { Reference reference = ((ReferenceNode)node).reference(); if (FeatureNames.isSimpleFeature(reference) && reference.name().equals("constant")) { String argument = reference.simpleArgument().get(); - if (constantsReplacedByMacros.contains(argument)) + if (constantsReplacedByFunctions.contains(argument)) return new ReferenceNode(argument); } } if (node instanceof CompositeNode) { // not else: this matches some of the same nodes as the outer if above CompositeNode composite = (CompositeNode)node; return composite.setChildren(composite.children().stream() - .map(child -> replaceConstantsByMacros(child, constantsReplacedByMacros)) + .map(child -> replaceConstantsByFunctions(child, constantsReplacedByFunctions)) .collect(Collectors.toList())); } return node; } - private static void addMacroNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) { + private static void addFunctionNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) { if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode)node; - if (referenceNode.getOutput() == null) { // macro references cannot specify outputs + if (referenceNode.getOutput() == null) { // function references cannot specify outputs names.add(referenceNode.getName()); - if (model.macros().containsKey(referenceNode.getName())) { - addMacroNamesIn(model.macros().get(referenceNode.getName()).getRoot(), names, model); + if (model.functions().containsKey(referenceNode.getName())) { + addFunctionNamesIn(model.functions().get(referenceNode.getName()).getRoot(), names, model); } } } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode)node).children()) - addMacroNamesIn(child, names, model); + addFunctionNamesIn(child, names, model); } } @@ -551,19 +548,19 @@ public class ConvertedModel { return expressions; } - /** Adds this macro expression to the application package so it can be read later. */ - void writeMacro(String name, RankingExpression expression) { - application.getFile(modelFiles.macrosPath()).appendFile(name + "\t" + - expression.getRoot().toString() + "\n"); + /** Adds this function expression to the application package so it can be read later. */ + void writeFunction(String name, RankingExpression expression) { + application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" + + expression.getRoot().toString() + "\n"); } - /** Reads the previously stored macro expressions for these arguments */ - List<Pair<String, RankingExpression>> readMacros() { + /** Reads the previously stored function expressions for these arguments */ + List<Pair<String, RankingExpression>> readFunctions() { try { - ApplicationFile file = application.getFile(modelFiles.macrosPath()); + ApplicationFile file = application.getFile(modelFiles.functionsPath()); if ( ! file.exists()) return Collections.emptyList(); - List<Pair<String, RankingExpression>> macros = new ArrayList<>(); + List<Pair<String, RankingExpression>> functions = new ArrayList<>(); BufferedReader reader = new BufferedReader(file.createReader()); String line; while (null != (line = reader.readLine())) { @@ -571,13 +568,13 @@ public class ConvertedModel { String name = parts[0]; try { RankingExpression expression = new RankingExpression(parts[0], parts[1]); - macros.add(new Pair<>(name, expression)); + functions.add(new Pair<>(name, expression)); } catch (ParseException e) { throw new IllegalStateException("Could not parse " + name, e); } } - return macros; + return functions; } catch (IOException e) { throw new UncheckedIOException(e); @@ -725,9 +722,9 @@ public class ConvertedModel { return storedModelReplicatedPath().append("constants"); } - /** Path to the macros file */ - public Path macrosPath() { - return storedModelReplicatedPath().append("macros.txt"); + /** Path to the functions file */ + public Path functionsPath() { + return storedModelReplicatedPath().append("functions.txt"); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 63d3926afad..813d1d47533 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -34,6 +34,7 @@ import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferen import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.DocumentsOnlyRankProfile; import com.yahoo.searchdefinition.DefaultRankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; @@ -58,6 +59,8 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.query.ranking.Diversity; import java.util.Map; +import java.util.List; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.logging.Level; import org.apache.commons.lang.StringUtils; @@ -73,6 +76,7 @@ public class SDParser { private ApplicationPackage app; private DeployLogger deployLogger; private RankProfileRegistry rankProfileRegistry; + private boolean documentsOnly; /** For testing only */ public SDParser(String input, DeployLogger deployLogger) { @@ -81,17 +85,24 @@ public class SDParser { /** For testing only */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger) { - this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry()); + this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry(), false); } + /** + * Creates a parser + * + * @param documentsOnly true to only parse the document aspect of a search definition (e.g skip rank profiles) + */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger, ApplicationPackage applicationPackage, - RankProfileRegistry rankProfileRegistry) { + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) { this(stream); this.deployLogger = deployLogger; this.app = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -1804,6 +1815,7 @@ void rankingConstant(Search search) : } lbrace() (rankingConstantItem(constant) (<NL>)*)+ <RBRACE> ) { + if (documentsOnly) return; search.rankingConstants().add(constant); } } @@ -1859,7 +1871,10 @@ void rankProfile(Search search) : { ( <RANKPROFILE> name = identifier() { - if ("default".equals(name)) { + if (documentsOnly) { + profile = new DocumentsOnlyRankProfile(name, search, rankProfileRegistry); + } + else if ("default".equals(name)) { profile = rankProfileRegistry.get(search, "default"); } else { profile = new RankProfile(name, search, rankProfileRegistry); @@ -1868,6 +1883,7 @@ void rankProfile(Search search) : [inheritsRankProfile(profile)] lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> ) { + if (documentsOnly) return; rankProfileRegistry.add(profile); } } @@ -1885,7 +1901,7 @@ Object rankProfileItem(RankProfile profile) : { } | fieldRankFilter(profile) | firstPhase(profile) | matchPhase(profile) - | macro(profile) + | function(profile) | ignoreRankFeatures(profile) | numThreadsPerSearch(profile) | minHitsPerThread(profile) @@ -1910,7 +1926,8 @@ void inheritsRankProfile(RankProfile profile) : String str; } { - <INHERITS> str = identifier() { profile.setInherited(str); } + <INHERITS> str = identifier() + { profile.setInherited(str); } } /** @@ -1918,19 +1935,20 @@ void inheritsRankProfile(RankProfile profile) : * * @param profile The profile to modify. */ -void macro(RankProfile profile) : +void function(RankProfile profile) : { - String macro, param, expr; + String name, expression, parameter; + List parameters = new ArrayList(); boolean inline = false; } { - ( <MACRO> inline = inline() macro = identifier() [ "$" { macro = macro + token.image; } ] - "(" { profile.addMacro(macro, inline); } - [ param = identifier() { profile.getMacros().get(macro).addParam(param); } - ( <COMMA> param = identifier() { profile.getMacros().get(macro).addParam(param); } )* ] + ( ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ] + "(" + [ parameter = identifier() { parameters.add(parameter); } + ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ] ")" - lbrace() expr = expression() (<NL>)* <RBRACE> ) - { profile.getMacros().get(macro).setTextualExpression(expr); } + lbrace() expression = expression() (<NL>)* <RBRACE> ) + { profile.addFunction(name, parameters, expression, inline); } } boolean inline() : @@ -2034,7 +2052,7 @@ Object firstPhaseItem(RankProfile profile) : double dropLimit; } { - ( expression = expression() { profile.setFirstPhaseRankingString(expression); } + ( expression = expression() { profile.setFirstPhaseRanking(expression); } | (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); } | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } ) @@ -2063,7 +2081,7 @@ Object secondPhaseItem(RankProfile profile) : int rerankCount; } { - ( expression = expression() { profile.setSecondPhaseRankingString(expression); } + ( expression = expression() { profile.setSecondPhaseRanking(expression); } | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } ) { return null; } diff --git a/config-model/src/test/derived/gemini2/gemini.sd b/config-model/src/test/derived/gemini2/gemini.sd index 18be346a758..01e20c1b30a 100644 --- a/config-model/src/test/derived/gemini2/gemini.sd +++ b/config-model/src/test/derived/gemini2/gemini.sd @@ -6,19 +6,19 @@ search gemini { rank-profile test { - macro wrapper2(x) { + function wrapper2(x) { expression: x } - macro wrapper1(x) { + function wrapper1(x) { expression: wrapper2(x) } - macro toplevel() { + function toplevel() { expression: wrapper1(attribute(right)) } - macro interfering() { + function interfering() { expression: wrapper1(attribute(wrong)) } diff --git a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd index 7f09095c5e7..6e399c03a2c 100644 --- a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd +++ b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd @@ -22,11 +22,11 @@ search rankexpression { } rank-profile macros { - macro titlematch$(var1, var2) { + function titlematch$(var1, var2) { expression: file: titlematch } - macro artistmatch() { + function artistmatch() { expression: 78+closeness(distance) } diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index 96b0fa98098..0435ea439df 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -121,7 +121,7 @@ search simple { second-phase { rerank-count: 99 } - macro openTicket() { + function openTicket() { expression: if(attribute(status) == "accepted",1, if(attribute(status) == "new",1,if(attribute(status) == "reopened",1,0))) } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 03fa92f5cb9..07a36832094 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -5,10 +5,12 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.io.IOException; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,9 +29,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT new DerivedConfiguration(search, registry, new QueryProfileRegistry(), new ImportedModels()); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { - e.printStackTrace(); - assertTrue(e.getCause().getMessage().contains("Could not read ranking expression file")); - assertTrue(e.getCause().getMessage().contains("wrongending.expr.expression")); + String message = Exceptions.toMessageString(e); + assertTrue(message.contains("Could not read ranking expression file")); + assertTrue(message.contains("wrongending.expr.expression")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index 28559f351ac..02ec597c3ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -4,6 +4,8 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; @@ -17,6 +19,7 @@ import static org.junit.Assert.assertNull; * @author Ulf Lilleengen */ public class RankProfileRegistryTest { + private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles"; @Test @@ -43,11 +46,11 @@ public class RankProfileRegistryTest { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { - assertNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); - RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry); - rankProfileWithAddedMacro.addMacro("foo", true); - rankProfileRegistry.add(rankProfileWithAddedMacro); - assertNotNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); + assertNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); + RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, search, rankProfileRegistry); + rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); + rankProfileRegistry.add(rankProfileWithAddedFunction); + assertNotNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index a524a26cbef..150469cc928 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -59,7 +59,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -76,7 +76,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedModels()); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); - assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); + assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString()); List<Pair<String, String>> rankProperties = new RawRankProfile(child2, queryProfileRegistry, new ImportedModels(), @@ -101,7 +101,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " c: 7 \n" + " }\n" + - " macro c() {\n" + + " function c() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -114,7 +114,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase fail("Should have caused an exception"); } catch (IllegalArgumentException e) { - assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and macro named 'c'", + assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'", Exceptions.toMessageString(e)); } } @@ -132,7 +132,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" + " }\n" + " }\n" + @@ -141,8 +141,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -161,7 +160,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " myValue: -9.21034037\n" + " }\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, myValue)\n" + " }\n" + " }\n" + @@ -170,14 +169,13 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test - public void testConstantDivisorInMacro() throws ParseException { + public void testConstantDivisorInFunction() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -186,7 +184,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: k1 + (k2 + k3) / 100000000.0\n\n" + " }\n" + " }\n" + @@ -196,7 +194,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 100000000.0", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } @Test @@ -212,7 +210,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" + " }\n" + " }\n" + @@ -222,7 +220,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index b13ffabda77..e507a6c48e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase { @Test - public void testMacroInliningPreserveArithemticOrdering() throws ParseException { + public void testFunctionInliningPreserveArithemticOrdering() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -44,18 +44,18 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: p1 * add\n" + " }\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 3 + attribute(a) + attribute(b) * mul3\n" + " }\n" + - " macro inline mul3() {\n" + + " function inline mul3() {\n" + " expression: attribute(a) * 3 + singleif\n" + " }\n" + - " macro inline singleif() {\n" + + " function inline singleif() {\n" + " expression: if (p1 < attribute(a), 1, 2) == 0\n" + " }\n" + " }\n" + " rank-profile child inherits parent {\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 9 + attribute(a)\n" + " }\n" + " }\n" + @@ -95,7 +95,7 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " second-phase {\n" + " expression: p2 * foo\n" + " }\n" + - " macro inline foo() {\n" + + " function inline foo() {\n" + " expression: 3 + p1 + p2\n" + " }\n" + " }\n" + @@ -106,16 +106,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro bar() {\n" + + " function bar() {\n" + " expression: p2*p1\n" + " }\n" + - " macro inline baz() {\n" + + " function inline baz() {\n" + " expression: p2+p1+boz\n" + " }\n" + - " macro inline boz() {\n" + + " function inline boz() {\n" + " expression: 3.0\n" + " }\n" + - " macro inline arg(a1) {\n" + + " function inline arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -162,16 +162,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: A + C + D\n" + " }\n" + - " macro inline D() {\n" + + " function inline D() {\n" + " expression: B + 1\n" + " }\n" + - " macro C() {\n" + + " function C() {\n" + " expression: A + B\n" + " }\n" + - " macro inline B() {\n" + + " function inline B() {\n" + " expression: attribute(b)\n" + " }\n" + - " macro inline A() {\n" + + " function inline A() {\n" + " expression: attribute(a)\n" + " }\n" + " }\n" + @@ -187,8 +187,8 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase } /** - * Expression evaluation has no stack so macro arguments are bound at config time creating a separate version of - * each macro for each binding, using hashes to name the bound variants of the macro. + * Expression evaluation has no stack so function arguments are bound at config time creating a separate version of + * each function for each binding, using hashes to name the bound variants of the function. * This method censors those hashes for string comparison. */ private String censorBindingHash(String s) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index df9a40d29e2..17bebcba70e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -29,7 +29,7 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: foo\n" + " }\n" + " }\n" + @@ -61,10 +61,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(5)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: foo + a1*2\n" + " }\n" + " }\n" + @@ -96,10 +96,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(foo)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -131,10 +131,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -159,10 +159,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3) + bar(a1)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -183,10 +183,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(bar(2))\n" + " }\n" + - " macro foo(x) {\n" + + " function foo(x) {\n" + " expression: x * x\n" + " }\n" + - " macro bar(x) {\n" + + " function bar(x) {\n" + " expression: x + x\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 1ece2355a92..e15d4075b19 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase { @Test - public void testBasicMacroShadowing() throws ParseException { + public void testBasicFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -31,7 +31,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -57,7 +57,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMultiLevelMacroShadowing() throws ParseException { + public void testMultiLevelFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -69,13 +69,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro tan(x) {\n" + + " function tan(x) {\n" + " expression: x * x\n" + " }\n" + - " macro cos(x) {\n" + + " function cos(x) {\n" + " expression: tan(x)\n" + " }\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: cos(x)\n" + " }\n" + " first-phase {\n" + @@ -113,7 +113,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMacroShadowingArguments() throws ParseException { + public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -125,7 +125,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -168,13 +168,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro relu(x) {\n" + // relu is a built in function, redefined here + " function relu(x) {\n" + // relu is a built in function, redefined here " expression: max(1.0, x)\n" + " }\n" + - " macro hidden_layer() {\n" + + " function hidden_layer() {\n" + " expression: relu(sum(query(q) * constant(W_hidden), input) + constant(b_input))\n" + " }\n" + - " macro final_layer() {\n" + + " function final_layer() {\n" + " expression: sigmoid(sum(hidden_layer * constant(W_final), hidden) + constant(b_final))\n" + " }\n" + " second-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 3fe3a7c3de1..5e649c2e551 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -5,9 +5,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Ignore; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -29,9 +31,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success - // TODO: Where's the "com.yahoo.searchdefinition.parser.ParseException:" nonsense coming from? - assertTrue("Got unexpected error message: " + e.getCause().getMessage(), - e.getCause().getMessage().startsWith("com.yahoo.searchdefinition.parser.ParseException: Could not parse ranking expression '" + expression + "'")); + assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 8cdfdd51637..82c03c02f61 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -34,20 +34,19 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { assertEquals("simple",search.getName()); assertTrue(search.hasDocument()); - SDDocumentType document=search.getDocument(); - assertEquals("simple",document.getName()); - assertEquals(12,document.getFieldCount()); + SDDocumentType document = search.getDocument(); + assertEquals("simple", document.getName()); + assertEquals(12, document.getFieldCount()); SDField field; Attribute attribute; - new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true); + new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); // First field field=(SDField) document.getField("title"); assertEquals(DataType.STRING,field.getDataType()); - assertEquals("{ summary | index; }", - field.getIndexingScript().toString()); + assertEquals("{ summary | index; }", field.getIndexingScript().toString()); assertTrue(!search.getIndex("default").isPrefix()); assertTrue(search.getIndex("title").isPrefix()); Iterator<String> titleAliases=search.getIndex("title").aliasIterator(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 3ef8591d203..f87a26c6f79 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -31,7 +31,7 @@ public class IdTestCase extends AbstractExportingTestCase { uri.parseIndexingScript("{ summary | index }"); document.addField(uri); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); assertNull(document.getField("uri")); assertNull(document.getField("Uri")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index fad80164135..2bae285301c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -41,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); // Check attribute fields diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index 9d19d4b154c..ba19a8312f6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -69,14 +69,14 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { assertTrue(!transforms.hasNext()); } @Test - public void testPositionDeriving() throws IOException, ParseException { + public void testPositionDeriving() { Search search = new Search("store", null); SDDocumentType document = new SDDocumentType("store"); search.addDocument(document); String fieldName = "location"; SDField field = document.addField(fieldName, PositionDataType.INSTANCE); field.parseIndexingScript("{ attribute | summary }"); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); SummaryMap summaryMap = new SummaryMap(search, new Summaries(search, new BaseDeployLogger())); Iterator transforms = summaryMap.resultTransformIterator(); @@ -141,7 +141,7 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { } @Test - public void testFailOnSummaryFieldSourceCollision() throws IOException, ParseException { + public void testFailOnSummaryFieldSourceCollision() { try { Search search = SearchBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 17cc9aaae4c..8941b07101d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -33,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase { a.parseIndexingScript("{ index }"); document.addField(a); - new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index a0556a156b5..48adc0eefc5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -36,7 +36,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( search,null,null,null); - processor.process(true); + processor.process(true, false); SummaryField summaryField = search.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); SummaryTransform actualTransform = summaryField.getTransform(); assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java index 8f1cc9c3de8..3a0fedfd550 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java @@ -24,7 +24,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { public void testInvalidAttributeProperties() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties1.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail("attribute property should not be set"); } catch (RuntimeException e) { // empty @@ -34,7 +34,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { @Test public void testValidAttributeProperties() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties2.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index 4dee939e3da..1ab8b054cb7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -24,7 +24,7 @@ public class BoldingTestCase extends SearchDefinitionTestCase { public void testBoldingNonString() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/processing/boldnonstring.sd"); - new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail(); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("'bolding: on' for non-text field")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 7230f176048..de08bf66548 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -188,7 +188,7 @@ public class ImportedFieldsResolverTestCase { private static ImportedFields resolve(Search search) { assertNotNull(search.temporaryImportedFields().get()); assertFalse(search.importedFields().isPresent()); - new ImportedFieldsResolver(search, null, null, null).process(true); + new ImportedFieldsResolver(search, null, null, null).process(true, false); assertFalse(search.temporaryImportedFields().isPresent()); assertNotNull(search.importedFields().get()); return search.importedFields().get(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index 2af3477fa62..e078d91f248 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -155,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { sdoc.addField(unprocessedField); Search search = new Search("test", null); search.addDocument(sdoc); - new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); return unprocessedField.getIndexingScript(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index 1e0270f293d..29bba224f46 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -19,11 +19,12 @@ import static org.junit.Assert.assertTrue; * @author baldersheim */ public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { + @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/integerindex2attribute.sd"); search.process(); - new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); SDField f; f = search.getConcreteField("s1"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index f67c85e2881..cff9abb08ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -72,9 +72,9 @@ class RankProfileSearchFixture { assertEquals(expValue, rankPropertyList.get(0).getValue()); } - public void assertMacro(String expexctedExpression, String macroName, String rankProfile) { + public void assertFunction(String expexctedExpression, String functionName, String rankProfile) { assertEquals(expexctedExpression, - compiledRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString()); + compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString()); } public RankProfile compileRankProfile(String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java index d8eb4368b57..0d8cbbf2e6a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java @@ -109,7 +109,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testMacroInvocationTypes() throws Exception { + public void testFunctionInvocationTypes() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString(joinLines( @@ -123,7 +123,7 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro macro1(attribute_to_use) {", + " function macro1(attribute_to_use) {", " expression: attribute(attribute_to_use)", " }", " summary-features {", @@ -143,7 +143,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testTensorMacroInvocationTypes_Nested() throws Exception { + public void testTensorFunctionInvocationTypes_Nested() throws Exception { SearchBuilder builder = new SearchBuilder(); builder.importString(joinLines( "search test {", @@ -156,16 +156,16 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro return_a() {", + " function return_a() {", " expression: return_first(attribute(a), attribute(b))", " }", - " macro return_b() {", + " function return_b() {", " expression: return_second(attribute(a), attribute(b))", " }", - " macro return_first(e1, e2) {", + " function return_first(e1, e2) {", " expression: e1", " }", - " macro return_second(e1, e2) {", + " function return_second(e1, e2) {", " expression: return_first(e2, e1)", " }", " summary-features {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index b046d60f948..8944409e1e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -128,7 +128,7 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testOnnxReferenceMissingMacro() throws ParseException { + public void testOnnxReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -145,14 +145,14 @@ public class RankingExpressionWithOnnxTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testOnnxReferenceWithWrongMacroType() { + public void testOnnxReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "onnx('mnist_softmax.onnx')"); @@ -163,7 +163,7 @@ public class RankingExpressionWithOnnxTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -213,13 +213,13 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstant() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstant() throws IOException { String rankProfile = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_Variable() {\n" + + " function " + name + "_Variable() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -234,7 +234,7 @@ public class RankingExpressionWithOnnxTestCase { search.compileRankProfile("my_profile", applicationDir.append("models")); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get( name + "_Variable")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -247,7 +247,7 @@ public class RankingExpressionWithOnnxTestCase { RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get( name + "_Variable")); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -275,19 +275,19 @@ public class RankingExpressionWithOnnxTestCase { } } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 14632a568ea..cba931e81f0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -160,7 +160,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceMissingMacro() throws ParseException { + public void testTensorFlowReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -177,14 +177,14 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testTensorFlowReferenceWithWrongMacroType() { + public void testTensorFlowReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "tensorflow('mnist_softmax/saved')"); @@ -195,7 +195,7 @@ public class RankingExpressionWithTensorFlowTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -261,13 +261,13 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstantAndInheritance() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException { String rankProfiles = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_layer_Variable_read() {\n" + + " function " + name + "_layer_Variable_read() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -285,7 +285,7 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -300,7 +300,7 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); } finally { @@ -317,11 +317,11 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testMacroGeneration() { + public void testFunctionGeneration() { final String name = "mnist_saved"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", "tensorflow('mnist/saved')", @@ -330,8 +330,8 @@ public class RankingExpressionWithTensorFlowTestCase { "input", new StoringApplicationPackage(applicationDir)); search.assertFirstPhaseExpression(expression, "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); } @Test @@ -339,7 +339,7 @@ public class RankingExpressionWithTensorFlowTestCase { final String name = "mnist_saved"; final String rankProfiles = " rank-profile my_profile {\n" + - " macro input() {\n" + + " function input() {\n" + " expression: tensor(d0[1],d1[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -349,9 +349,9 @@ public class RankingExpressionWithTensorFlowTestCase { " rank-profile my_profile_child inherits my_profile {\n" + " }"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWithUncompiled(rankProfiles, new StoringApplicationPackage(applicationDir)); search.compileRankProfile("my_profile", applicationDir.append("models")); @@ -359,10 +359,10 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(expression, "my_profile"); search.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); // At this point the expression is stored - copy application to another location which do not have a models dir Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); @@ -377,10 +377,10 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -429,19 +429,19 @@ public class RankingExpressionWithTensorFlowTestCase { new StoringApplicationPackage(applicationDir)); } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 0866d3192cf..48f2bf43e81 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -95,10 +95,10 @@ public class RankingExpressionWithTensorTestCase { } @Test - public void requireThatConstantTensorsCanBeUsedInMacro() throws ParseException { + public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + - " macro my_macro() {\n" + + " function my_macro() {\n" + " expression: sum(my_tensor)\n" + " }\n" + " first-phase {\n" + @@ -112,7 +112,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); - f.assertMacro("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); + f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 86127a260c5..fd048737b43 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -20,24 +20,24 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test - public void testMacros() throws IOException, ParseException { + public void testFunctions() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", rankProfileRegistry, new QueryProfileRegistry()).getSearch(); - final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); - macrosRankProfile.parseExpressions(); - final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros(); - assertEquals(2, macros.get("titlematch$").getFormalParams().size()); - assertEquals("var1", macros.get("titlematch$").getFormalParams().get(0)); - assertEquals("var2", macros.get("titlematch$").getFormalParams().get(1)); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getTextualExpression().trim()); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString()); - assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); - assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); - assertEquals(0, macros.get("artistmatch").getFormalParams().size()); + RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros"); + Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions(); + assertEquals(2, functions.get("titlematch$").function().arguments().size()); + assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); + assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); + assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", + functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); + assertEquals("78 + closeness(distance)", + functions.get("artistmatch").function().getBody().getRoot().toString()); + assertEquals(0, functions.get("artistmatch").function().arguments().size()); - List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, + List<Pair<String, String>> rankProperties = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(), new ImportedModels(), new AttributeFields(search)).configProperties(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java index 8a07e99101c..b39c48b67bf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -16,10 +16,10 @@ import static org.junit.Assert.assertTrue; /** * @author lesters */ -public class ReservedMacroNamesTestCase { +public class ReservedRankingExpressionFunctionNamesTestCase { @Test - public void requireThatMacrosWithReservedNamesIssueAWarning() throws ParseException { + public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { TestDeployLogger deployLogger = new TestDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); @@ -32,10 +32,10 @@ public class ReservedMacroNamesTestCase { " }\n" + " \n" + " rank-profile test_rank_profile {\n" + - " macro not_a_reserved_name(x) {\n" + + " function not_a_reserved_name(x) {\n" + " expression: x + x\n" + " }\n" + - " macro sigmoid(x) {\n" + + " function sigmoid(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -43,7 +43,7 @@ public class ReservedMacroNamesTestCase { " }\n" + " }\n" + " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index 85bee61c1b4..d0c1bf8b0ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -12,6 +12,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { @@ -20,41 +21,44 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTe Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); } } + @Test public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidimplicitsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); } } + @Test public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidselfreferringsummary.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); } } + @Test public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/documentidinsummary.sd"); search.process(); BaseDeployLogger deployLogger = new BaseDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true); + new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); assertEquals("documentid", search.getSummary("withid").getSummaryField("w").getSingleSource()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 76c50821cb9..8e721dbe503 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -113,7 +113,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { } @Test - public void requireThatMaxAndMinWithTensorsReturnedFromMacrosAreReplaced() throws ParseException { + public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException { assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", "max(returns_tensor,x)"); assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", @@ -171,7 +171,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:0 }\n" + " }\n" + " }\n" + - " macro base_tensor() {\n" + + " function base_tensor() {\n" + " expression: constant(base_constant_tensor)\n" + " }\n" + " }\n" + @@ -181,19 +181,19 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:1 }\n" + " }\n" + " }\n" + - " macro returns_tensor_with_arg(arg1) {\n" + + " function returns_tensor_with_arg(arg1) {\n" + " expression: 2.0 * arg1\n" + " }\n" + - " macro wraps_returns_tensor() {\n" + + " function wraps_returns_tensor() {\n" + " expression: returns_tensor\n" + " }\n" + - " macro returns_tensor() {\n" + + " function returns_tensor() {\n" + " expression: attribute(tensor_field_2)\n" + " }\n" + - " macro tensor_inheriting() {\n" + + " function tensor_inheriting() {\n" + " expression: base_tensor\n" + " }\n" + - " macro testexpression() {\n" + + " function testexpression() {\n" + " expression: " + expression + "\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index ab3bb113727..d0b6524a7e1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -40,7 +40,7 @@ public class ValidateFieldTypesTest { exceptionRule.expectMessage( "For search '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); - validator.process(true); + validator.process(true, false); } private static Search createSearchWithDocument(String documentName) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java index 334228333c5..ea2ce324a27 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java @@ -6,8 +6,7 @@ import java.util.Objects; /** * Represents a tenant in the provision API. * - * @author lulf - * @since 5.12 + * @author Ulf Lilleengen */ public class TenantName implements Comparable<TenantName> { diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index d2fca309a19..7bea2d0825a 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -81,11 +81,11 @@ public class RankProfilesConfigImporter { referencedFunctions.put(reference.get(), new ExpressionFunction(reference.get().serialForm(), arguments, expression)); } - else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to macros + else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to functions firstPhase = new ExpressionFunction("firstphase", new ArrayList<>(), new RankingExpression("first-phase", property.value())); } - else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to macros + else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to functions secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(), new RankingExpression("second-phase", property.value())); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index c6d8f70fde8..da34ab8822d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,13 +3,16 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.text.Utf8; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.*; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * A function defined by a ranking expression @@ -24,6 +27,16 @@ public class ExpressionFunction { private final RankingExpression body; /** + * Constructs a new function with no arguments + * + * @param name the name of this function + * @param body the ranking expression that defines this function + */ + public ExpressionFunction(String name, RankingExpression body) { + this(name, Collections.emptyList(), body); + } + + /** * Constructs a new function * * @param name the name of this function diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index ed82ba20fbe..722520fea08 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -250,12 +250,12 @@ public class RankingExpression implements Serializable { /** * Creates the necessary rank properties required to implement this expression. * - * @param macros the expression macros to expand. - * @return a list of named rank properties required to implement this expression. + * @param functions the expression functions to expand + * @return a list of named rank properties required to implement this expression */ - public Map<String, String> getRankProperties(List<ExpressionFunction> macros) { + public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { Deque<String> path = new LinkedList<>(); - SerializationContext context = new SerializationContext(macros); + SerializationContext context = new SerializationContext(functions); String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString(); Map<String, String> serializedExpressions = context.serializedFunctions(); serializedExpressions.put(propertyName(name), serializedRoot); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java index ac5eefcc5b2..282a4c5e0a9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java @@ -30,8 +30,8 @@ public class ImportedModel { private final Map<String, Tensor> smallConstants = new HashMap<>(); private final Map<String, Tensor> largeConstants = new HashMap<>(); private final Map<String, RankingExpression> expressions = new HashMap<>(); - private final Map<String, RankingExpression> macros = new HashMap<>(); - private final Map<String, TensorType> requiredMacros = new HashMap<>(); + private final Map<String, RankingExpression> functions = new HashMap<>(); + private final Map<String, TensorType> requiredFunctions = new HashMap<>(); /** * Creates a new imported model. @@ -77,13 +77,13 @@ public class ImportedModel { public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); } /** - * Returns an immutable map of macros that are part of this model. - * Note that the macros themselves are *not* copies and *not* immutable - they must be copied before modification. + * Returns an immutable map of the functions that are part of this model. + * Note that the functions themselves are *not* copies and *not* immutable - they must be copied before modification. */ - public Map<String, RankingExpression> macros() { return Collections.unmodifiableMap(macros); } + public Map<String, RankingExpression> functions() { return Collections.unmodifiableMap(functions); } - /** Returns an immutable map of the macros that must be provided by the environment running this model */ - public Map<String, TensorType> requiredMacros() { return Collections.unmodifiableMap(requiredMacros); } + /** Returns an immutable map of the functions that must be provided by the environment running this model */ + public Map<String, TensorType> requiredFunctions() { return Collections.unmodifiableMap(requiredFunctions); } /** Returns an immutable map of the signatures of this */ public Map<String, Signature> signatures() { return Collections.unmodifiableMap(signatures); } @@ -100,8 +100,8 @@ public class ImportedModel { void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); } void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); } void expression(String name, RankingExpression expression) { expressions.put(name, expression); } - void macro(String name, RankingExpression expression) { macros.put(name, expression); } - void requiredMacro(String name, TensorType type) { requiredMacros.put(name, type); } + void function(String name, RankingExpression expression) { functions.put(name, expression); } + void requiredFunction(String name, TensorType type) { requiredFunctions.put(name, type); } /** * Returns all the output expressions of this indexed by name. The names consist of one or two parts diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java index 2ae107a5770..d25502fd149 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java @@ -24,7 +24,7 @@ import java.util.logging.Logger; * ranking expressions. The general mechanism for import is for the * specific ML platform import implementations to create an * IntermediateGraph. This class offers common code to convert the - * IntermediateGraph to Vespa ranking expressions and macros. + * IntermediateGraph to Vespa ranking expressions and functions. * * @author lesters */ @@ -122,7 +122,7 @@ public abstract class ModelImporter { importExpressionInputs(operation, model); importRankingExpression(operation, model); importArgumentExpression(operation, model); - importMacroExpression(operation, model); + importFunctionExpression(operation, model); return operation.function(); } @@ -188,15 +188,15 @@ public abstract class ModelImporter { // All inputs must have dimensions with standard naming convention: d0, d1, ... OrderedTensorType standardNamingConvention = OrderedTensorType.standardType(operation.type().get()); model.argument(operation.vespaName(), standardNamingConvention.type()); - model.requiredMacro(operation.vespaName(), standardNamingConvention.type()); + model.requiredFunction(operation.vespaName(), standardNamingConvention.type()); } } - private static void importMacroExpression(IntermediateOperation operation, ImportedModel model) { - if (operation.macro().isPresent()) { - TensorFunction function = operation.macro().get(); + private static void importFunctionExpression(IntermediateOperation operation, ImportedModel model) { + if (operation.rankingExpressionFunction().isPresent()) { + TensorFunction function = operation.rankingExpressionFunction().get(); try { - model.macro(operation.macroName(), new RankingExpression(operation.macroName(), function.toString())); + model.function(operation.rankingExpressionFunctionName(), new RankingExpression(operation.rankingExpressionFunctionName(), function.toString())); } catch (ParseException e) { throw new RuntimeException("Tensorflow function " + function + diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java index 43de29cedd5..34f5f1365a1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java @@ -29,7 +29,7 @@ import java.util.function.Function; */ public abstract class IntermediateOperation { - private final static String MACRO_PREFIX = "imported_ml_macro_"; + private final static String FUNCTION_PREFIX = "imported_ml_function_"; protected final String name; protected final String modelName; @@ -38,7 +38,7 @@ public abstract class IntermediateOperation { protected OrderedTensorType type; protected TensorFunction function; - protected TensorFunction macro = null; + protected TensorFunction rankingExpressionFunction = null; private final List<String> importWarnings = new ArrayList<>(); private Value constantValue = null; @@ -71,8 +71,8 @@ public abstract class IntermediateOperation { ExpressionNode constant = new ReferenceNode(Reference.simple("constant", vespaName())); function = new TensorFunctionNode.TensorFunctionExpressionNode(constant); } else if (outputs.size() > 1) { - macro = lazyGetFunction(); - function = new VariableTensor(macroName(), type.type()); + rankingExpressionFunction = lazyGetFunction(); + function = new VariableTensor(rankingExpressionFunctionName(), type.type()); } else { function = lazyGetFunction(); } @@ -86,11 +86,13 @@ public abstract class IntermediateOperation { /** Return unmodifiable list of inputs */ public List<IntermediateOperation> inputs() { return inputs; } - /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a macro. */ + /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a function. */ public List<IntermediateOperation> outputs() { return Collections.unmodifiableList(outputs); } - /** Returns a Vespa ranking expression that should be added as a macro */ - public Optional<TensorFunction> macro() { return Optional.ofNullable(macro); } + /** Returns a function that should be added as a ranking expression function */ + public Optional<TensorFunction> rankingExpressionFunction() { + return Optional.ofNullable(rankingExpressionFunction); + } /** Add dimension name constraints for this operation */ public void addDimensionNameConstraints(DimensionRenamer renamer) { } @@ -131,8 +133,10 @@ public abstract class IntermediateOperation { public String vespaName() { return vespaName(name); } public String vespaName(String name) { return name != null ? namePartOf(name).replace('/', '_') : null; } - /** Retrieve the valid Vespa name of this node if it is a macro */ - public String macroName() { return vespaName() != null ? MACRO_PREFIX + modelName + "_" + vespaName() : null; } + /** Retrieve the valid Vespa name of this node if it is a ranking expression function */ + public String rankingExpressionFunctionName() { + return vespaName() != null ? FUNCTION_PREFIX + modelName + "_" + vespaName() : null; + } /** Retrieve the list of warnings produced during its lifetime */ public List<String> warnings() { return Collections.unmodifiableList(importWarnings); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java index 9299ae9be12..b335fd7e1c5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java @@ -26,13 +26,13 @@ public class PlaceholderWithDefault extends IntermediateOperation { if (!allInputFunctionsPresent(1)) { return null; } - // This should be a call to the macro we add below, but for now + // This should be a call to the function we add below, but for now // we treat this as as identity function and just pass the constant. return inputs.get(0).function().orElse(null); } @Override - public Optional<TensorFunction> macro() { + public Optional<TensorFunction> rankingExpressionFunction() { // For now, it is much more efficient to assume we always will return // the default value, as we can prune away large parts of the expression // tree by having it calculated as a constant. If a case arises where @@ -42,7 +42,7 @@ public class PlaceholderWithDefault extends IntermediateOperation { @Override public boolean isConstant() { - return true; // not true if we add to macro + return true; // not true if we add to function } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index e2dc170c168..eb8d2229a6d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -15,7 +15,7 @@ import java.util.Deque; import java.util.List; /** - * A node referring either to a value in the context or to a named ranking expression (function aka macro). + * A node referring either to a value in the context or to a named ranking expression function. * * @author bratseth */ @@ -64,7 +64,7 @@ public final class ReferenceNode extends CompositeNode { @Override public StringBuilder toString(StringBuilder string, SerializationContext context, Deque<String> path, CompositeNode parent) { - // A reference to a macro argument? + // A reference to a function argument? if (reference.isIdentifier() && context.getBinding(getName()) != null) { // a bound identifier: replace by the value it is bound to return string.append(context.getBinding(getName())); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index 7c929ae24b3..571e1f4d608 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -91,26 +91,26 @@ public class RankingExpressionTestCase { @Test public void testSelfRecursionSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", null, new RankingExpression("foo"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(macros); + exp.getRankProperties(functions); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[]]", e.getMessage()); } } @Test - public void testMacroCycleSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); - macros.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); + public void testFunctionCycleSerialization() throws ParseException { + List<ExpressionFunction> funnctions = new ArrayList<>(); + funnctions.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); + funnctions.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(macros); + exp.getRankProperties(funnctions); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[], bar[]]", e.getMessage()); } @@ -118,11 +118,11 @@ public class RankingExpressionTestCase { @Test public void testSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); - macros.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); - macros.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); - macros.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); + functions.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); + functions.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); + functions.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977"))); assertSerialization(Arrays.asList( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(foo@af74e3fd9070bd18.a368ed0a5ba3a5d0) * rankingExpression(foo@dbab346efdad5362.e5c39e42ebd91c30)", @@ -130,19 +130,19 @@ public class RankingExpressionTestCase { "min(6,pow(7,2))", "min(1,pow(2,2))", "min(3,pow(4,2))", - "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", macros); + "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", functions); assertSerialization(Arrays.asList( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(bar@af74e3fd9070bd18.a368ed0a5ba3a5d0)", "min(1,pow(2,2))", - "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", macros); + "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", functions); assertSerialization(Arrays.asList( "rankingExpression(baz@e2dc17a89864aed0.12232eb692c6c502)", "min(1,pow(2,2))", "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) / rankingExpression(bar@e2dc17a89864aed0.12232eb692c6c502)", - "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", macros); + "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", functions); assertSerialization(Arrays.asList( "rankingExpression(cox)", - "10 + 08 * 1977"), "cox", macros + "10 + 08 * 1977"), "cox", functions ); } @@ -159,8 +159,8 @@ public class RankingExpressionTestCase { @Test public void testBug3464208() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69"))); String lhs = "log10(0.01+attribute(user_followers_count)) * log10(socialratio) * " + "log10(userage/(0.01+attribute(user_statuses_count)))"; @@ -172,8 +172,8 @@ public class RankingExpressionTestCase { String expRhs = "(rankingExpression(log10tweetage) * rankingExpression(log10tweetage) * " + "rankingExpression(log10tweetage)) + 5.0 * attribute(ythl)"; - assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, macros); - assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, macros); + assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, functions); + assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, functions); } @Test @@ -295,12 +295,12 @@ public class RankingExpressionTestCase { assertEquals(expected, new RankingExpression(expression).toString()); } - /** Test serialization with no macros */ + /** Test serialization with no functions */ private void assertSerialization(String expectedSerialization, String expressionString) { String serializedExpression; try { RankingExpression expression = new RankingExpression(expressionString); - // No macros -> expect one rank property + // No functions -> expect one rank property serializedExpression = expression.getRankProperties(Collections.emptyList()).values().iterator().next(); assertEquals(expectedSerialization, serializedExpression); } @@ -309,7 +309,7 @@ public class RankingExpressionTestCase { } try { - // No macros -> output should be parseable to a ranking expression + // No functions -> output should be parseable to a ranking expression // (but not the same one due to primitivization) RankingExpression reparsedExpression = new RankingExpression(serializedExpression); // Serializing the primitivized expression should yield the same expression again @@ -323,17 +323,17 @@ public class RankingExpressionTestCase { } private void assertSerialization(List<String> expectedSerialization, String expressionString, - List<ExpressionFunction> macros) { - assertSerialization(expectedSerialization, expressionString, macros, false); + List<ExpressionFunction> functions) { + assertSerialization(expectedSerialization, expressionString, functions, false); } private void assertSerialization(List<String> expectedSerialization, String expressionString, - List<ExpressionFunction> macros, boolean print) { + List<ExpressionFunction> functions, boolean print) { try { if (print) System.out.println("Parsing expression '" + expressionString + "'."); RankingExpression expression = new RankingExpression(expressionString); - Map<String, String> rankProperties = expression.getRankProperties(macros); + Map<String, String> rankProperties = expression.getRankProperties(functions); if (print) { for (String key : rankProperties.keySet()) System.out.println("Property '" + key + "': " + rankProperties.get(key)); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java index a63c7346335..a8f7542f3a4 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java @@ -18,11 +18,11 @@ public class DropoutImportTestCase { public void testDropoutImport() { TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/files/integration/tensorflow/dropout/saved"); - // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("X")); + // Check required functions + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("X")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("X")); + model.get().requiredFunctions().get("X")); ImportedModel.Signature signature = model.get().signature("serving_default"); @@ -32,7 +32,7 @@ public class DropoutImportTestCase { RankingExpression output = signature.outputExpression("y"); assertNotNull(output); assertEquals("outputs/Maximum", output.getName()); - assertEquals("join(join(imported_ml_macro_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_macro_test_outputs_BiasAdd, f(a,b)(max(a,b)))", + assertEquals("join(join(imported_ml_function_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_function_test_outputs_BiasAdd, f(a,b)(max(a,b)))", output.getRoot().toString()); model.assertEqualResult("X", output.getName()); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java index bcfc6ce0a04..e20ac16a691 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java @@ -36,11 +36,11 @@ public class OnnxMnistSoftmaxImportTestCase { constant1.type()); assertEquals(10, constant1.size()); - // Check required macros (inputs) - assertEquals(1, model.requiredMacros().size()); - assertTrue(model.requiredMacros().containsKey("Placeholder")); + // Check required functions (inputs) + assertEquals(1, model.requiredFunctions().size()); + assertTrue(model.requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.requiredMacros().get("Placeholder")); + model.requiredFunctions().get("Placeholder")); // Check outputs RankingExpression output = model.defaultSignature().outputExpression("add"); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java index dd6c8095e3c..ef28eb4678f 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java @@ -34,14 +34,14 @@ public class TensorFlowMnistSoftmaxImportTestCase { constant1.type()); assertEquals(10, constant1.size()); - // Check (provided) macros - assertEquals(0, model.get().macros().size()); + // Check (provided) functions + assertEquals(0, model.get().functions().size()); - // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("Placeholder")); + // Check required functions + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("Placeholder")); + model.get().requiredFunctions().get("Placeholder")); // Check signatures assertEquals(1, model.get().signatures().size()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java index 4de3aa5d635..5447e5240f7 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java @@ -48,7 +48,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateFunction(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", @@ -62,7 +62,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateFunction(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", tfResult, vespaResult); @@ -96,24 +96,24 @@ public class TestableTensorFlowModel { return b.build(); } - private void evaluateMacro(Context context, ImportedModel model, String macroName) { - if (!context.names().contains(macroName)) { - RankingExpression e = model.macros().get(macroName); - evaluateMacroDependencies(context, model, e.getRoot()); - context.put(macroName, new TensorValue(e.evaluate(context).asTensor())); + private void evaluateFunction(Context context, ImportedModel model, String functionName) { + if (!context.names().contains(functionName)) { + RankingExpression e = model.functions().get(functionName); + evaluateFunctionDependencies(context, model, e.getRoot()); + context.put(functionName, new TensorValue(e.evaluate(context).asTensor())); } } - private void evaluateMacroDependencies(Context context, ImportedModel model, ExpressionNode node) { + private void evaluateFunctionDependencies(Context context, ImportedModel model, ExpressionNode node) { if (node instanceof ReferenceNode) { String name = node.toString(); - if (model.macros().containsKey(name)) { - evaluateMacro(context, model, name); + if (model.functions().containsKey(name)) { + evaluateFunction(context, model, name); } } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode)node).children()) { - evaluateMacroDependencies(context, model, child); + evaluateFunctionDependencies(context, model, child); } } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java index 84e51835458..1f28f0b0129 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java @@ -27,7 +27,7 @@ public class ConstantDereferencerTestCase { TransformContext context = new TransformContext(constants); assertEquals("1.0 + 2.0 + 3.5", c.transform(new RankingExpression("a + b + c"), context).toString()); - assertEquals("myMacro(1.0,2.0)", c.transform(new RankingExpression("myMacro(a, b)"), context).toString()); + assertEquals("myFunction(1.0,2.0)", c.transform(new RankingExpression("myFunction(a, b)"), context).toString()); } } diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index b7c955bd83b..d27352b8ea7 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -7,12 +7,9 @@ import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.MinimalProcessingSearchBuilder; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.UnprocessingSearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.tensor.TensorType; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -80,10 +77,10 @@ public class DocumentGenMojo extends AbstractMojo { void execute(File sdDir, File outputDir, String packageName) throws MojoFailureException { if ("".equals(packageName)) throw new IllegalArgumentException("You may not use empty package for generated types."); - searches = new HashMap<String, Search>(); - docTypes = new HashMap<String, String>(); - structTypes = new HashMap<String, String>(); - annotationTypes = new HashMap<String, String>(); + searches = new HashMap<>(); + docTypes = new HashMap<>(); + structTypes = new HashMap<>(); + annotationTypes = new HashMap<>(); outputDir.mkdirs(); SearchBuilder builder = buildSearches(sdDir); @@ -111,7 +108,7 @@ public class DocumentGenMojo extends AbstractMojo { public boolean accept(File dir, String name) { return name.endsWith(".sd"); }}); - SearchBuilder builder = new MinimalProcessingSearchBuilder(); + SearchBuilder builder = new SearchBuilder(true); for (File f : sdFiles) { try { long modTime = f.lastModified(); |