diff options
161 files changed, 1527 insertions, 1147 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java new file mode 100644 index 00000000000..9335c0b4005 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java @@ -0,0 +1,35 @@ +package com.yahoo.searchdefinition; + +import java.util.List; + +/** + * A rank profile which ignores all calls made to it which may fail in a document only setting. + * This is used by the search definition parser when it is requested to parse documents only, + * to avoid having to check for this in every method which adds to the rank profile. + * (And why do we ever want to parse documents only? Because it is used when generating Java classes + * from documents, where the full application package may not be available.) + * + * @author bratseth + */ +public class DocumentsOnlyRankProfile extends RankProfile { + + public DocumentsOnlyRankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) { + super(name, search, rankProfileRegistry); + } + + @Override + public void setFirstPhaseRanking(String expression) { + // Ignore + } + + @Override + public void setSecondPhaseRanking(String expression) { + // Ignore + } + + @Override + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { + // Ignore + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index afd33da369f..0d9ea00bf73 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Stack; import java.util.stream.Collectors; /** @@ -71,7 +70,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement currentResolutionCallStack.stream().map(Reference::toString).collect(Collectors.joining(" -> ")) + " -> " + reference); - // A reference to a macro argument? + // A reference to a function argument? Optional<String> binding = boundIdentifier(reference); if (binding.isPresent()) { try { @@ -117,7 +116,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement } /** - * Returns the default type for this simple feature, or nullif it does not have a default + * Returns the default type for this simple feature, or null if it does not have a default */ public TensorType defaultTypeOf(Reference reference) { if ( ! FeatureNames.isSimpleFeature(reference)) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b7e1f9d4538..16e494c2db1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -95,13 +95,9 @@ public class RankProfile implements Serializable, Cloneable { /** The properties of this - a multimap */ private Map<String, List<RankProperty>> rankProperties = new LinkedHashMap<>(); - private Boolean ignoreDefaultRankFeatures=null; + private Boolean ignoreDefaultRankFeatures = null; - private String secondPhaseRankingString=null; - - private String firstPhaseRankingString=null; - - private Map<String, Macro> macros= new LinkedHashMap<>(); + private Map<String, RankingExpressionFunction> functions = new LinkedHashMap<>(); private Set<String> filterFields = new HashSet<>(); @@ -339,13 +335,22 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getFirstPhaseRanking() { - if (firstPhaseRanking!=null) return firstPhaseRanking; - if (getInherited()!=null) return getInherited().getFirstPhaseRanking(); + if (firstPhaseRanking != null) return firstPhaseRanking; + if (getInherited() != null) return getInherited().getFirstPhaseRanking(); return null; } public void setFirstPhaseRanking(RankingExpression rankingExpression) { - this.firstPhaseRanking=rankingExpression; + this.firstPhaseRanking = rankingExpression; + } + + public void setFirstPhaseRanking(String expression) { + try { + this.firstPhaseRanking = parseRankingExpression("firstphase", expression); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal first phase ranking function", e); + } } /** @@ -353,31 +358,22 @@ public class RankProfile implements Serializable, Cloneable { * Returns null if no expression is set. */ public RankingExpression getSecondPhaseRanking() { - if (secondPhaseRanking!=null) return secondPhaseRanking; - if (getInherited()!=null) return getInherited().getSecondPhaseRanking(); + if (secondPhaseRanking != null) return secondPhaseRanking; + if (getInherited() != null) return getInherited().getSecondPhaseRanking(); return null; } public void setSecondPhaseRanking(RankingExpression rankingExpression) { - this.secondPhaseRanking=rankingExpression; + this.secondPhaseRanking = rankingExpression; } - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for second phase - */ - public void setSecondPhaseRankingString(String exp) { - this.secondPhaseRankingString = exp; - } - - /** - * Called by parser to store the expression string, for delayed evaluation - * - * @param exp ranking expression for first phase - */ - public void setFirstPhaseRankingString(String exp) { - this.firstPhaseRankingString = exp; + public void setSecondPhaseRanking(String expression) { + try { + this.secondPhaseRanking = parseRankingExpression("secondphase", expression); + } + catch (ParseException e) { + throw new IllegalArgumentException("Illegal second phase ranking function", e); + } } /** Returns a read-only view of the summary features to use in this profile. This is never null */ @@ -412,8 +408,8 @@ public class RankProfile implements Serializable, Cloneable { } public void addRankFeature(ReferenceNode feature) { - if (rankFeatures==null) - rankFeatures=new LinkedHashSet<>(); + if (rankFeatures == null) + rankFeatures = new LinkedHashSet<>(); rankFeatures.add(feature); } @@ -522,55 +518,43 @@ public class RankProfile implements Serializable, Cloneable { } public boolean getIgnoreDefaultRankFeatures() { - if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures; - return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures(); + if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; + return (getInherited() != null) && getInherited().getIgnoreDefaultRankFeatures(); } - /** - * Returns the string form of the second phase ranking expression. - * - * @return string form of second phase ranking expression - */ - public String getSecondPhaseRankingString() { - if (secondPhaseRankingString != null) return secondPhaseRankingString; - if (getInherited() != null) return getInherited().getSecondPhaseRankingString(); - return null; - } - - /** - * Returns the string form of the first phase ranking expression. - * - * @return string form of first phase ranking expression - */ - public String getFirstPhaseRankingString() { - if (firstPhaseRankingString != null) return firstPhaseRankingString; - if (getInherited() != null) return getInherited().getFirstPhaseRankingString(); - return null; + /** Adds a function */ + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { + try { + addFunction(new ExpressionFunction(name, arguments, parseRankingExpression(name, expression)), inline); + } + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse function '" + name + "'", e); + } } - /** Creates a new (empty) macro and returns it */ - public Macro addMacro(String name, boolean inline) { - Macro macro = new Macro(name, inline); - macros.put(name, macro); - return macro; + /** Adds a function and returns it */ + public RankingExpressionFunction addFunction(ExpressionFunction function, boolean inline) { + RankingExpressionFunction rankingExpressionFunction = new RankingExpressionFunction(function, inline); + functions.put(function.getName(), rankingExpressionFunction); + return rankingExpressionFunction; } - /** Returns an unmodifiable view of the macros in this */ - public Map<String, Macro> getMacros() { - if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap(); - if (macros.size() == 0) return getInherited().getMacros(); - if (getInherited() == null) return Collections.unmodifiableMap(macros); + /** Returns an unmodifiable view of the functions in this */ + public Map<String, RankingExpressionFunction> getFunctions() { + if (functions.size() == 0 && getInherited() == null) return Collections.emptyMap(); + if (functions.size() == 0) return getInherited().getFunctions(); + if (getInherited() == null) return Collections.unmodifiableMap(functions); // Neither is null - Map<String, Macro> allMacros = new LinkedHashMap<>(getInherited().getMacros()); - allMacros.putAll(macros); - return Collections.unmodifiableMap(allMacros); + Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions()); + allFunctions.putAll(functions); + return Collections.unmodifiableMap(allFunctions); } public int getKeepRankCount() { - if (keepRankCount>=0) return keepRankCount; - if (getInherited()!=null) return getInherited().getKeepRankCount(); + if (keepRankCount >= 0) return keepRankCount; + if (getInherited() != null) return getInherited().getKeepRankCount(); return -1; } @@ -579,8 +563,8 @@ public class RankProfile implements Serializable, Cloneable { } public double getRankScoreDropLimit() { - if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit; - if (getInherited()!=null) return getInherited().getRankScoreDropLimit(); + if (rankScoreDropLimit >- Double.MAX_VALUE) return rankScoreDropLimit; + if (getInherited() != null) return getInherited().getRankScoreDropLimit(); return rankScoreDropLimit; } @@ -607,58 +591,15 @@ public class RankProfile implements Serializable, Cloneable { return retval; } - /** - * Will take the parser-set textual ranking expressions and turn into ranking expression objects, - * if not already done - */ - // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions - // until this is called. Simplify by parsing them right away. - public void parseExpressions() { - try { - parseRankingExpressions(); - parseMacros(); - } catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - /** - * Passes the contents of macros on to parser. Then put all the implied rank properties - * from those macros into the profile's props map. - */ - private void parseMacros() throws ParseException { - for (Map.Entry<String, Macro> e : getMacros().entrySet()) { - String macroName = e.getKey(); - Macro macro = e.getValue(); - if (macro.getRankingExpression() == null) { - RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); - macro.setRankingExpression(expr); - macro.setTextualExpression(expr.getRoot().toString()); - } - } - } - - /** - * Passes ranking expressions on to parser - * - * @throws ParseException if either of the ranking expressions could not be parsed - */ - private void parseRankingExpressions() throws ParseException { - if (getFirstPhaseRankingString() != null && firstPhaseRanking == null) - setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString())); - if (getSecondPhaseRankingString() != null && secondPhaseRanking == null) - setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); - } - - private RankingExpression parseRankingExpression(String expressionName, String exp) throws ParseException { - if (exp.trim().length() == 0) + private RankingExpression parseRankingExpression(String expressionName, String expression) throws ParseException { + if (expression.trim().length() == 0) throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expressionName + "."); - try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, exp.trim())) { + try (Reader rankingExpressionReader = openRankingExpressionReader(expressionName, expression.trim())) { return new RankingExpression(expressionName, rankingExpressionReader); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { - ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() + + ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + "' in " + getName()+ ", " + expressionName + "."); throw (ParseException)exception.initCause(e); } @@ -686,14 +627,13 @@ public class RankProfile implements Serializable, Cloneable { @Override public RankProfile clone() { try { - // Note: This treats RankingExpression in Macros as immutables even though they are not RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; clone.rankProperties = new LinkedHashMap<>(this.rankProperties); - clone.macros = new LinkedHashMap<>(this.macros); + clone.functions = new LinkedHashMap<>(this.functions); clone.filterFields = new HashSet<>(this.filterFields); clone.constants = new HashMap<>(this.constants); return clone; @@ -719,60 +659,59 @@ public class RankProfile implements Serializable, Cloneable { } private void compileThis(QueryProfileRegistry queryProfiles, ImportedModels importedModels) { - parseExpressions(); - checkNameCollisions(getMacros(), getConstants()); + checkNameCollisions(getFunctions(), getConstants()); ExpressionTransforms expressionTransforms = new ExpressionTransforms(); - // Macro compiling first pass: compile inline macros without resolving other macros - Map<String, Macro> inlineMacros = compileMacros(getInlineMacros(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); + // Function compiling first pass: compile inline functions without resolving other functions + Map<String, RankingExpressionFunction> inlineFunctions = + compileFunctions(getInlineFunctions(), queryProfiles, importedModels, Collections.emptyMap(), expressionTransforms); - // Macro compiling second pass: compile all macros and insert previously compiled inline macros - macros = compileMacros(getMacros(), queryProfiles, importedModels, inlineMacros, expressionTransforms); + // Function compiling second pass: compile all functions and insert previously compiled inline functions + functions = compileFunctions(getFunctions(), queryProfiles, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); } - private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) { - for (Map.Entry<String, Macro> macroEntry : macros.entrySet()) { - if (constants.get(macroEntry.getKey()) != null) - throw new IllegalArgumentException("Cannot have both a constant and macro named '" + - macroEntry.getKey() + "'"); + private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { + for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) { + if (constants.get(functionEntry.getKey()) != null) + throw new IllegalArgumentException("Cannot have both a constant and function named '" + + functionEntry.getKey() + "'"); } } - private Map<String, Macro> getInlineMacros() { - return getMacros().entrySet().stream().filter(x -> x.getValue().getInline()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + private Map<String, RankingExpressionFunction> getInlineFunctions() { + return getFunctions().entrySet().stream().filter(x -> x.getValue().inline()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private Map<String, Macro> compileMacros(Map<String, Macro> macros, - QueryProfileRegistry queryProfiles, - ImportedModels importedModels, - Map<String, Macro> inlineMacros, - ExpressionTransforms expressionTransforms) { - Map<String, Macro> compiledMacros = new LinkedHashMap<>(); - for (Map.Entry<String, Macro> entry : macros.entrySet()) { - Macro macro = entry.getValue().clone(); - RankingExpression exp = compile(macro.getRankingExpression(), queryProfiles, importedModels, getConstants(), inlineMacros, expressionTransforms); - macro.setRankingExpression(exp); - compiledMacros.put(entry.getKey(), macro); + private Map<String, RankingExpressionFunction> compileFunctions(Map<String, RankingExpressionFunction> functions, + QueryProfileRegistry queryProfiles, + ImportedModels importedModels, + Map<String, RankingExpressionFunction> inlineFunctions, + ExpressionTransforms expressionTransforms) { + Map<String, RankingExpressionFunction> compiledFunctions = new LinkedHashMap<>(); + for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { + RankingExpressionFunction rankingExpressionFunction = entry.getValue(); + RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); + compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withBody(compiled)); } - return compiledMacros; + return compiledFunctions; } private RankingExpression compile(RankingExpression expression, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map<String, Value> constants, - Map<String, Macro> inlineMacros, + Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { if (expression == null) return null; RankProfileTransformContext context = new RankProfileTransformContext(this, queryProfiles, importedModels, constants, - inlineMacros); + inlineFunctions); expression = expressionTransforms.transform(expression, context); for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) { addRankProperty(rankProperty.getKey(), rankProperty.getValue()); @@ -785,9 +724,9 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public TypeContext<Reference> typeContext(QueryProfileRegistry queryProfiles) { - MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() - .map(Macro::asExpressionFunction) - .collect(Collectors.toList())); + MapEvaluationTypeContext context = new MapEvaluationTypeContext(getFunctions().values().stream() + .map(RankingExpressionFunction::function) + .collect(Collectors.toList())); // Add small and large constants, respectively getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); @@ -854,11 +793,11 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ private boolean isIndexLevel; - private Type(String name) { + Type(String name) { this(name,false); } - private Type(String name,boolean isIndexLevel) { + Type(String name,boolean isIndexLevel) { this.name = name; this.isIndexLevel=isIndexLevel; } @@ -866,7 +805,7 @@ public class RankProfile implements Serializable, Cloneable { /** True if this setting really pertains to an index, not a field within an index */ public boolean isIndexLevel() { return isIndexLevel; } - /** @return The name of this type */ + /** Returns the name of this type */ public String getName() { return name; } @@ -899,10 +838,12 @@ public class RankProfile implements Serializable, Cloneable { } } + @Override public int hashCode() { return fieldName.hashCode() + 17 * type.hashCode(); } + @Override public boolean equals(Object object) { if (!(object instanceof RankSetting)) { return false; @@ -913,6 +854,7 @@ public class RankProfile implements Serializable, Cloneable { type.equals(other.type); } + @Override public String toString() { return type + " setting " + fieldName + ": " + value; } @@ -936,7 +878,7 @@ public class RankProfile implements Serializable, Cloneable { @Override public int hashCode() { - return name.hashCode() + 17*value.hashCode(); + return name.hashCode() + 17 * value.hashCode(); } @Override @@ -953,73 +895,32 @@ public class RankProfile implements Serializable, Cloneable { } - /** - * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro - */ - public static class Macro implements Serializable, Cloneable { + /** A function in a rank profile */ + public static class RankingExpressionFunction { - private final String name; - private String textualExpression = null; - private RankingExpression expression = null; - private List<String> formalParams = new ArrayList<>(); + private final ExpressionFunction function; - /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ + /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ private final boolean inline; - public Macro(String name, boolean inline) { - this.name = name; + public RankingExpressionFunction(ExpressionFunction function, boolean inline) { + this.function = function; this.inline = inline; } - public void addParam(String name) { - formalParams.add(name); - } - - public List<String> getFormalParams() { - return formalParams; - } - - public String getTextualExpression() { - return textualExpression; - } - - public void setTextualExpression(String textualExpression) { - this.textualExpression = textualExpression; - } - - public void setRankingExpression(RankingExpression expr) { - this.expression=expr; - } - - public RankingExpression getRankingExpression() { - return expression; - } - - public String getName() { - return name; - } + public ExpressionFunction function() { return function; } - public boolean getInline() { - return inline && formalParams.size() == 0; // only inline no-arg macros; + public boolean inline() { + return inline && function.arguments().isEmpty(); // only inline no-arg functions; } - public ExpressionFunction asExpressionFunction() { - return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression()); - } - - @Override - public Macro clone() { - try { - return (Macro)super.clone(); - } - catch (CloneNotSupportedException e) { - throw new RuntimeException("Won't happen", e); - } + public RankingExpressionFunction withBody(RankingExpression expression) { + return new RankingExpressionFunction(function.withBody(expression), inline); } @Override public String toString() { - return "macro " + getName() + ": " + expression; + return "function " + function; } } @@ -1106,6 +1007,7 @@ public class RankProfile implements Serializable, Cloneable { public Map<String, String> getTypes() { return Collections.unmodifiableMap(types); } + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 91f86bb1c2a..3c2ebc058ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -39,17 +39,25 @@ public class SearchBuilder { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); private List<Search> searchList = new LinkedList<>(); - private ApplicationPackage app = null; + private ApplicationPackage app; private boolean isBuilt = false; private DocumentModel model = new DocumentModel(); private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; + /** True to build the document aspect only, skipping instantiation of rank profiles */ + private final boolean documentsOnly; + /** For testing only */ public SearchBuilder() { this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry()); } + /** Used for generating documents for typed access to document fields in Java */ + public SearchBuilder(boolean documentsOnly) { + this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); + } + /** For testing only */ public SearchBuilder(ApplicationPackage app) { this(app, new RankProfileRegistry(), new QueryProfileRegistry()); @@ -68,9 +76,16 @@ public class SearchBuilder { public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + this(app, rankProfileRegistry, queryProfileRegistry, false); + } + public SearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { this.app = app; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -150,7 +165,7 @@ public class SearchBuilder { Search search; SimpleCharStream stream = new SimpleCharStream(str); try { - search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir); + search = new SDParser(stream, deployLogger, app, rankProfileRegistry, documentsOnly).search(docTypeMgr, searchDefDir); } catch (TokenMgrException e) { throw new ParseException("Unknown symbol: " + e.getMessage()); } catch (ParseException pe) { @@ -164,9 +179,9 @@ public class SearchBuilder { * {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed search objects used in unit tests. * - * @param rawSearch The object to import. - * @return The name of the imported object. - * @throws IllegalArgumentException Thrown if the given search object has already been processed. + * @param rawSearch the object to import. + * @return the name of the imported object. + * @throws IllegalArgumentException if the given search object has already been processed. */ public String importRawSearch(Search rawSearch) { if (rawSearch.getName() == null) { @@ -250,15 +265,15 @@ public class SearchBuilder { * #build()} method so that subclasses can choose not to build anything. */ protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles, boolean validate) { - Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles, validate); + new Processing().process(search, deployLogger, rankProfileRegistry, queryProfiles, validate, documentsOnly); } /** * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object * built. This method will never return null. * - * @return The build object. - * @throws IllegalStateException Thrown if there is not exactly one search. + * @return the built object + * @throws IllegalStateException if there is not exactly one search. */ public Search getSearch() { if ( ! isBuilt) throw new IllegalStateException("Searches not built."); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java index c1b05c0fcdf..49b7ad621af 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java @@ -1,8 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.collections.Pair; + +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -54,22 +58,18 @@ public class FieldRankSettings { table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY)); } - public Map<String,String> deriveRankProperties(int part) { - Map<String,String> ret = new LinkedHashMap<>(); - int i = part; - for (Iterator<NativeTable> itr = tables.values().iterator(); itr.hasNext(); ++i) { - NativeTable table = itr.next(); - if (isFieldMatchTable(table)) { - ret.put("nativeFieldMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } - if (isAttributeMatchTable(table)) { - ret.put("nativeAttributeMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } - if (isProximityTable(table)) { - ret.put("nativeProximity." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); - } + public List<Pair<String, String>> deriveRankProperties() { + List<Pair<String, String>> properties = new ArrayList<>(); + for (Iterator<NativeTable> i = tables.values().iterator(); i.hasNext();) { + NativeTable table = i.next(); + if (isFieldMatchTable(table)) + properties.add(new Pair<>("nativeFieldMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isAttributeMatchTable(table)) + properties.add(new Pair<>("nativeAttributeMatch." + table.getType().getName() + "." + fieldName, table.getName())); + if (isProximityTable(table)) + properties.add(new Pair<>("nativeProximity." + table.getType().getName() + "." + fieldName, table.getName())); } - return ret; + return properties; } public String toString() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 43cc2fad285..c041d5c6a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -50,17 +50,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, AttributeFields attributeFields) { this.name = rankProfile.getName(); - compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive())); - } - - private List<Pair<String, String>> removePartFromKeys(Map<String, String> map) { - ImmutableList.Builder<Pair<String, String>> replaced = new ImmutableList.Builder<>(); - for (Map.Entry<String, String> e : map.entrySet()) { - String key = e.getKey().replaceFirst(".part\\d+$", ""); - String val = e.getValue(); - replaced.add(new Pair<>(key, val)); - } - return replaced.build(); + compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive()); } private Compressor.Compression compress(List<Pair<String, String>> properties) { @@ -185,57 +175,57 @@ public class RawRankProfile implements RankProfilesConfig.Producer { rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros()); + derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); } - private void derivePropertiesAndSummaryFeaturesFromMacros(Map<String, RankProfile.Macro> macros) { - if (macros.isEmpty()) return; - Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>(); - for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) { - expressionMacros.put(macro.getKey(), macro.getValue().asExpressionFunction()); + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { + if (functions.isEmpty()) return; + Map<String, ExpressionFunction> expressionFunctions = new LinkedHashMap<>(); + for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : functions.entrySet()) { + expressionFunctions.put(function.getKey(), function.getValue().function()); } - Map<String, String> macroProperties = new LinkedHashMap<>(); - macroProperties.putAll(deriveMacroProperties(expressionMacros)); + Map<String, String> functionProperties = new LinkedHashMap<>(); + functionProperties.putAll(deriveFunctionProperties(expressionFunctions)); if (firstPhaseRanking != null) { - macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } if (secondPhaseRanking != null) { - macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + functionProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } - for (Map.Entry<String, String> e : macroProperties.entrySet()) { + + for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties); - replaceMacroSummaryFeatures(context); + SerializationContext context = new SerializationContext(expressionFunctions.values(), null, functionProperties); + replaceFunctionSummaryFeatures(context); } - private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) { - SerializationContext context = new SerializationContext(eMacros); - for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) { + private Map<String, String> deriveFunctionProperties(Map<String, ExpressionFunction> functions) { + SerializationContext context = new SerializationContext(functions); + for (Map.Entry<String, ExpressionFunction> e : functions.entrySet()) { String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression); - } return context.serializedFunctions(); } - private void replaceMacroSummaryFeatures(SerializationContext context) { + private void replaceFunctionSummaryFeatures(SerializationContext context) { if (summaryFeatures == null) return; - Map<String, ReferenceNode> macroSummaryFeatures = new LinkedHashMap<>(); + Map<String, ReferenceNode> functionSummaryFeatures = new LinkedHashMap<>(); for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) { ReferenceNode referenceNode = i.next(); - // Is the feature a macro? + // Is the feature a function? if (context.getFunction(referenceNode.getName()) != null) { context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), referenceNode.toString(new StringBuilder(), context, null, null).toString()); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); - macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); + functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block } } - // Then, replace the summary features that were macros - for (Map.Entry<String, ReferenceNode> e : macroSummaryFeatures.entrySet()) { + // Then, replace the summary features that were functions + for (Map.Entry<String, ReferenceNode> e : functionSummaryFeatures.entrySet()) { summaryFeatures.add(e.getValue()); } } @@ -300,17 +290,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return settings; } - /** - * Derives the properties this produces. Equal keys are suffixed with .part0 etc, remove when exporting to file - * - * @return map of the derived properties - */ - public Map<String, String> derive() { - Map<String, String> properties = new LinkedHashMap<>(); - int i = 0; + /** Derives the properties this produces */ + public List<Pair<String, String>> derive() { + List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) { - // Could have been set by macro expansion. Set expressions, then skip this property. + // Could have been set by function expansion. Set expressions, then skip this property. try { firstPhaseRanking = new RankingExpression(property.getValue()); } catch (ParseException e) { @@ -325,100 +310,92 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } } else { - properties.put(property.getName() + ".part" + i, property.getValue()); - i++; + properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.putAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); - properties.putAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); for (FieldRankSettings settings : fieldRankSettings.values()) { - properties.putAll(settings.deriveRankProperties(i)); + properties.addAll(settings.deriveRankProperties()); } - i = 0; for (RankProfile.RankProperty property : boostAndWeightRankProperties) { - properties.put(property.getName() + ".part" + i, property.getValue()); - i++; + properties.add(new Pair<>(property.getName(), property.getValue())); } - i = 0; for (ReferenceNode feature : summaryFeatures) { - properties.put(summaryFeatureFefPropertyPrefix + ".part" + i, feature.toString()); - i++; + properties.add(new Pair<>(summaryFeatureFefPropertyPrefix, feature.toString())); } - i = 0; for (ReferenceNode feature : rankFeatures) { - properties.put(rankFeatureFefPropertyPrefix + ".part" + i, feature.toString()); - i++; + properties.add(new Pair<>(rankFeatureFefPropertyPrefix, feature.toString())); } if (numThreadsPerSearch > 0) { - properties.put("vespa.matching.numthreadspersearch", numThreadsPerSearch + ""); + properties.add(new Pair<>("vespa.matching.numthreadspersearch", numThreadsPerSearch + "")); } if (minHitsPerThread > 0) { - properties.put("vespa.matching.minhitsperthread", minHitsPerThread + ""); + properties.add(new Pair<>("vespa.matching.minhitsperthread", minHitsPerThread + "")); } if (numSearchPartitions >= 0) { - properties.put("vespa.matching.numsearchpartitions", numSearchPartitions + ""); + properties.add(new Pair<>("vespa.matching.numsearchpartitions", numSearchPartitions + "")); } if (termwiseLimit < 1.0) { - properties.put("vespa.matching.termwise_limit", termwiseLimit + ""); + properties.add(new Pair<>("vespa.matching.termwise_limit", termwiseLimit + "")); } if (matchPhaseSettings != null) { - properties.put("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute()); - properties.put("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + ""); - properties.put("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + ""); - properties.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + ""); - properties.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + ""); - properties.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + ""); + properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); + properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); if (diversitySettings != null) { - properties.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute()); - properties.put("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups())); - properties.put("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor())); - properties.put("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy())); + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); } } if (rerankCount > -1) { - properties.put("vespa.hitcollector.heapsize", rerankCount + ""); + properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); } if (keepRankCount > -1) { - properties.put("vespa.hitcollector.arraysize", keepRankCount + ""); + properties.add(new Pair<>("vespa.hitcollector.arraysize", keepRankCount + "")); } if (rankScoreDropLimit > -Double.MAX_VALUE) { - properties.put("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + ""); + properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); } if (ignoreDefaultRankFeatures) { - properties.put("vespa.dump.ignoredefaultfeatures", String.valueOf(true)); + properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } Iterator filterFieldsIterator = filterFields.iterator(); while (filterFieldsIterator.hasNext()) { String fieldName = (String) filterFieldsIterator.next(); - properties.put("vespa.isfilterfield." + fieldName + ".part42", String.valueOf(true)); + properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); } for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { - properties.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue()); + properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) { - properties.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue()); + properties.add(new Pair<>("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue())); } if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties"); return properties; } - private Map<String, String> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { - Map<String, String> ret = new LinkedHashMap<>(); - if (expression == null) { - return ret; - } + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { + List<Pair<String, String>> properties = new ArrayList<>(); + if (expression == null) return properties; + String name = expression.getName(); - if ("".equals(name)) { + if ("".equals(name)) name = phase; - } + if (expression.getRoot() instanceof ReferenceNode) { - ret.put("vespa.rank." + phase, expression.getRoot().toString()); + properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { - ret.put("vespa.rank." + phase, "rankingExpression(" + name + ")"); - ret.put("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString()); + properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); + properties.add(new Pair<>("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString())); } - return ret; + return properties; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index a639165d297..cbabfffb7a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -28,8 +28,8 @@ public class ExpressionTransforms { new XgboostFeatureConverter(), new ConstantDereferencer(), new ConstantTensorTransformer(), - new MacroInliner(), - new MacroShadower(), + new FunctionInliner(), + new FunctionShadower(), new TensorTransformer(), new Simplifier()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java index 6aef39db4da..c15ef20a455 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroInliner.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java @@ -8,11 +8,11 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; /** - * Inlines macros in ranking expressions + * Inlines functions in ranking expressions * * @author bratseth */ -public class MacroInliner extends ExpressionTransformer<RankProfileTransformContext> { +public class FunctionInliner extends ExpressionTransformer<RankProfileTransformContext> { @Override public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { @@ -24,9 +24,9 @@ public class MacroInliner extends ExpressionTransformer<RankProfileTransformCont } private ExpressionNode transformFeatureNode(ReferenceNode feature, RankProfileTransformContext context) { - RankProfile.Macro macro = context.inlineMacros().get(feature.getName()); - if (macro == null) return feature; - return transform(macro.getRankingExpression().getRoot(), context); // inline recursively and return + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.inlineFunctions().get(feature.getName()); + if (rankingExpressionFunction == null) return feature; + return transform(rankingExpressionFunction.function().getBody().getRoot(), context); // inline recursively and return } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java index 758d2b2a87d..74b6471d291 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/MacroShadower.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java @@ -10,19 +10,19 @@ import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; /** - * Transforms function nodes to reference nodes if a macro shadows a built-in function. - * This has the effect of allowing macros to redefine built-in functions. - * Another effect is that we can more or less add built-in functions over time - * without fear of breaking existing users' macros with the same name. + * Transforms function nodes to reference nodes if a rank profile function shadows a built-in function. + * This has the effect of allowing rank profile functions to redefine built-in functions. + * Another effect is that we can add built-in functions over time + * without fear of breaking existing users' functions with the same name. * - * However, there is a (largish) caveat. If a user has a macro with a certain number + * However, there is a (largish) caveat. If a user has a function with a certain number * of arguments, and we add in a built-in function with a different arity, * this will cause parse errors as the Java parser gives precedence to * built-in functions. * * @author lesters */ -public class MacroShadower extends ExpressionTransformer<RankProfileTransformContext> { +public class FunctionShadower extends ExpressionTransformer<RankProfileTransformContext> { @Override public RankingExpression transform(RankingExpression expression, RankProfileTransformContext context) { @@ -43,16 +43,14 @@ public class MacroShadower extends ExpressionTransformer<RankProfileTransformCon private ExpressionNode transformFunctionNode(FunctionNode function, RankProfileTransformContext context) { String name = function.getFunction().toString(); - RankProfile.Macro macro = context.rankProfile().getMacros().get(name); - if (macro == null) { + RankProfile.RankingExpressionFunction rankingExpressionFunction = context.rankProfile().getFunctions().get(name); + if (rankingExpressionFunction == null) { return transformChildren(function, context); } int functionArity = function.getFunction().arity(); - int macroArity = macro.getFormalParams() != null ? macro.getFormalParams().size() : 0; - if (functionArity != macroArity) { + if (functionArity != rankingExpressionFunction.function().arguments().size()) return transformChildren(function, context); - } ReferenceNode node = new ReferenceNode(name, function.children(), null); return transformChildren(node, context); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 40c3b997daa..2fe2dacf2ce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -20,25 +20,25 @@ public class RankProfileTransformContext extends TransformContext { private final RankProfile rankProfile; private final QueryProfileRegistry queryProfiles; private final ImportedModels importedModels; - private final Map<String, RankProfile.Macro> inlineMacros; + private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions; private final Map<String, String> rankProperties = new HashMap<>(); public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedModels importedModels, Map<String, Value> constants, - Map<String, RankProfile.Macro> inlineMacros) { + Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) { super(constants); this.rankProfile = rankProfile; this.queryProfiles = queryProfiles; this.importedModels = importedModels; - this.inlineMacros = inlineMacros; + this.inlineFunctions = inlineFunctions; } public RankProfile rankProfile() { return rankProfile; } public QueryProfileRegistry queryProfiles() { return queryProfiles; } public ImportedModels importedModels() { return importedModels; } - public Map<String, RankProfile.Macro> inlineMacros() { return inlineMacros; } + public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; } public Map<String, String> rankProperties() { return rankProperties; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java index cf123d0f7c1..fefb54a7fe3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -27,7 +27,7 @@ public class AddAttributeTransformToSummaryOfImportedFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.allImportedFields() .flatMap(this::getSummaryFieldsForImportedField) .forEach(AddAttributeTransformToSummaryOfImportedFields::setAttributeTransform); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java index e0d32ea8ccd..803a6c5ab40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -26,7 +26,7 @@ public class AddExtraFieldsToDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { SDDocumentType document = search.getDocument(); if (document != null) { for (Field field : search.extraFieldList()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java index 9ec596792fa..94589d94255 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java @@ -20,7 +20,7 @@ public class AttributeProperties extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { String fieldName = field.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index a95f4264dc6..23257e5eafd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -23,7 +23,7 @@ public class AttributesImplicitWord extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (fieldImplicitlyWordMatch(field)) { field.getMatching().setType(Matching.Type.WORD); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java index a3c4c97cf31..b9be30e8485 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java @@ -22,7 +22,7 @@ public class Bolding extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { for (SummaryField summary : field.getSummaryFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java index 37d60c1d32e..a0c4c8adb2d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document + * * @author Vegard Havdal */ public class BuiltInFieldSets extends Processor { @@ -23,7 +24,7 @@ public class BuiltInFieldSets extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { addDocumentFieldSet(); addSearchFieldSet(); // "Hook" the field sets on search onto the document types, since we will include them diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index d7b688be203..ad862ef767f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -38,7 +38,7 @@ public class CreatePositionZCurve extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { DataType fieldType = field.getDataType(); if ( ! isSupportedPositionType(fieldType)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java index 7cc9b4e9b52..b34db6febd5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java @@ -15,7 +15,7 @@ public class DeprecateAttributePrefetch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java index 861ebad7085..076161a8584 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -23,7 +23,7 @@ public class DisallowComplexMapAndWsetKeyTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 6b78da2146b..029892cba1c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -18,8 +18,9 @@ public class DiversitySettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index 59bc3dc66f4..a7c0ebd4a07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -26,7 +26,7 @@ public class ExactMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { Matching.Type matching = field.getMatching().getType(); if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java index 9cfac625da5..cd33e4434e7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java @@ -21,7 +21,7 @@ public class FastAccessValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String invalidAttributes = search.allFields() @@ -29,7 +29,7 @@ public class FastAccessValidator extends Processor { .filter(FastAccessValidator::isIncompatibleAttribute) .map(Attribute::getName) .collect(Collectors.joining(", ")); - if (!invalidAttributes.isEmpty()) { + if ( ! invalidAttributes.isEmpty()) { throw new IllegalArgumentException( String.format( "For search '%s': The following attributes have a type that is incompatible with fast-access: %s. " + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java index 15c11589245..4ef9a9733d5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java @@ -24,7 +24,7 @@ public class FieldSetValidity extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index 0c75314ffa2..adb8ab62aab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -26,7 +26,9 @@ public class FilterFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + for (SDField f : search.allConcreteFields()) { if (f.getRanking().isFilter()) { filterField(f.getName()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index b51524b7e62..1f795458875 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -28,7 +28,7 @@ public class ImplicitSummaries extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary = search.getSummary("default"); if (defaultSummary == null) { defaultSummary = new DocumentSummary("default"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java index 7464f574255..0d99c698aca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -21,7 +21,7 @@ public class ImplicitSummaryFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary docsum : search.getSummaries().values()) { addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES), validate); addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES), validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index 9f03cdf4b5e..a3efd086c6a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -33,7 +33,7 @@ public class ImportedFieldsResolver extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { search.temporaryImportedFields().get().fields().forEach((name, field) -> resolveImportedField(field, validate)); search.setImportedFields(new ImportedFields(importedFields)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java index 018183b91d8..210a8e7009c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java @@ -23,7 +23,7 @@ public class IndexFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java index 0740029de90..41355a76f47 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java @@ -24,7 +24,7 @@ public class IndexSettingsNonFieldNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java index 419268468c2..aeab2bb6638 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java @@ -27,7 +27,7 @@ public class IndexingInputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java index 6f04184c512..11d69bf6c75 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java @@ -29,7 +29,7 @@ public class IndexingOutputs extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index b73151768fd..27520647e3b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -25,7 +25,7 @@ public class IndexingValidation extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; VerificationContext context = new VerificationContext(new MyAdapter(search)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index f3907ae68cb..72777b7dfb4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -22,7 +22,7 @@ public class IndexingValues extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (Field field : search.getDocument().fieldSet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java index c119dc2660b..baaf145dbce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java @@ -30,7 +30,7 @@ public class IntegerIndex2Attribute extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { if (field.getIndex(field.getName()) != null diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java index 507a0e87cff..fe94ac9849f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java @@ -31,7 +31,7 @@ public class LiteralBoost extends Processor { /** Adds extra search fields and indices to express literal boosts */ @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { checkRankModifierRankType(search); addLiteralBoostsToFields(search); reduceFieldLiteralBoosts(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java index f853b99bbb2..0daf7265daa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java @@ -25,7 +25,7 @@ public class MakeAliases extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { List<String> usedAliases = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java index 2c43a65da99..6f67c22d9d2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -34,7 +34,7 @@ public class MakeDefaultSummaryTheSuperSet extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { DocumentSummary defaultSummary=search.getSummary("default"); for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) { if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java index fe584fa41c0..ff8a5c2eb0b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java @@ -30,7 +30,7 @@ public class MatchConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map<String, Matching.Type> types = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index 479384e09ef..b1728b9bd89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -20,8 +20,9 @@ public class MatchPhaseSettingsValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(search)) { RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java index 45e018f8fc3..a52a8ab74e6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java @@ -31,7 +31,7 @@ public class MultifieldIndexHarmonizer extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { populateIndexToFields(search); resolveAllConflicts(search); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java index 9bcb3929b3d..4d8f0032a78 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java @@ -15,12 +15,12 @@ public class MutableAttributes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { - if (!field.isExtraField() && field.getAttributes().containsKey(field.getName())) { + if ( ! field.isExtraField() && field.getAttributes().containsKey(field.getName())) { if (field.getAttributes().get(field.getName()).isMutable()) { throw new IllegalArgumentException("Field '" + field.getName() + "' in '" + search.getDocument().getName() + - "' can not be marked mutable as it is inside the document clause."); + "' can not be marked mutable as it is inside the document clause."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java index cdfba54ee5a..3f3fc11380b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java @@ -26,7 +26,7 @@ public class NGramMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType().equals(Matching.Type.GRAM)) implementGramMatch(search, field, validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java index c89c709ffbf..8f2a29abcb6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -22,7 +22,7 @@ public class OptimizeIlscript extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java index 3583c4a0162..79f19efe422 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -33,7 +33,7 @@ public class PredicateProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() == DataType.PREDICATE) { if (validate && field.doesIndexing()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 19025d37f8c..8c8c32389e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -7,7 +7,9 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; /** @@ -18,9 +20,7 @@ import java.util.List; */ public class Processing { - private static final List<ProcessorFactory> factories = createProcessorFactories(); - - private static List<ProcessorFactory> createProcessorFactories() { + private Collection<ProcessorFactory> processors() { return Arrays.asList( SearchMustHaveDocument::new, UrlFieldValidator::new, @@ -74,10 +74,9 @@ public class Processing { RankProfileTypeSettingsProcessor::new, ReferenceFieldsProcessor::new, FastAccessValidator::new, - ReservedMacroNames::new, + ReservedFunctionNames::new, RankingExpressionTypeValidator::new, - - // These should be last. + // These should be last: IndexingValidation::new, IndexingValues::new); } @@ -91,19 +90,18 @@ public class Processing { * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} * @param queryProfiles The query profiles contained in the application this search is part of. */ - public static void process(Search search, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, - boolean validate) { + public void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { + Collection<ProcessorFactory> factories = processors(); search.process(); factories.stream() .map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles)) - .forEach(processor -> processor.process(validate)); + .forEach(processor -> processor.process(validate, documentsOnly)); } @FunctionalInterface public interface ProcessorFactory { Processor create(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index b938e40d9a2..6bfd0ef29ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -53,8 +53,10 @@ public abstract class Processor { * @param validate true to throw exceptions on validation errors, false to make the best possible effort * at completing processing without throwing an exception. * If we are not validating, emitting warnings have no effect and can (but must not) be skipped. + * @param documentsOnly true to skip processing (including validation, regardless of the validate setting) + * of aspects not relating to document definitions (e.g rank profiles) */ - public abstract void process(boolean validate); + public abstract void process(boolean validate, boolean documentsOnly); /** * Convenience method for adding a no-strings-attached implementation field for a regular field diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index 81455991cc9..102d1910360 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -33,8 +33,9 @@ public class RankingExpressionTypeValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { @@ -48,7 +49,6 @@ public class RankingExpressionTypeValidator extends Processor { /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */ private void validate(RankProfile profile) { - profile.parseExpressions(); TypeContext context = profile.typeContext(queryProfiles); profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java index 76afcd5d520..0418538922b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java @@ -28,7 +28,7 @@ public class ReferenceFieldsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { clearSummaryAttributeAspectForConcreteFields(); clearSummaryAttributeAspectForExplicitSummaryFields(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java index f2aa31bb9c3..805cbaced0f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java @@ -28,7 +28,7 @@ public class ReservedDocumentNames extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String docName = search.getDocument().getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java index adcebed9254..d7099215f17 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedMacroNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -13,29 +13,30 @@ import java.util.Set; import java.util.logging.Level; /** - * Issues a warning if some macro has a reserved name. This is not necessarily - * an error, as a macro can shadow a built-in function. + * Issues a warning if some function has a reserved name. This is not necessarily + * an error, as a rank profile function can shadow a built-in function. * * @author lesters */ -public class ReservedMacroNames extends Processor { +public class ReservedFunctionNames extends Processor { private static Set<String> reservedNames = getReservedNames(); - public ReservedMacroNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public ReservedFunctionNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; + if (documentsOnly) return; for (RankProfile rp : rankProfileRegistry.all()) { - for (String macroName : rp.getMacros().keySet()) { - if (reservedNames.contains(macroName)) { - deployLogger.log(Level.WARNING, "Macro \"" + macroName + "\" " + - "in rank profile \"" + rp.getName() + "\" " + - "has a reserved name. This might mean that the macro shadows " + + for (String functionName : rp.getFunctions().keySet()) { + if (reservedNames.contains(functionName)) { + deployLogger.log(Level.WARNING, "Function '" + functionName + "' " + + "in rank profile '" + rp.getName() + "' " + + "has a reserved name. This might mean that the function shadows " + "the built-in function with the same name." ); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java index 403de1253b4..2d8eaff7762 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java @@ -19,7 +19,7 @@ public class SearchMustHaveDocument extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; if (search.getDocument() == null) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java index a0c884d25f9..8a4795c4dd2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java @@ -24,7 +24,8 @@ public class SetLanguage extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; List<String> textFieldsWithoutLanguage = new ArrayList<>(); for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java index a19ea8d7068..715828f808a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java @@ -20,7 +20,7 @@ public class SetRankTypeEmptyOnFilters extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getRanking().isFilter()) { field.setRankType(RankType.EMPTY); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java index 6426c724a07..defcf761649 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java @@ -21,7 +21,7 @@ public class SortingSettings extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java index d56b0272f06..e133f88f45c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java @@ -16,7 +16,7 @@ public class StringSettingsOnNonStringFields extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java index a952d3732b3..d2c4968ca26 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -26,7 +26,7 @@ public class SummaryConsistency extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (DocumentSummary summary : search.getSummaries().values()) { if (summary.getName().equals("default")) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java index 647a433f201..b8d170c07f6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java @@ -26,7 +26,7 @@ public class SummaryDynamicStructsArrays extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java index 7bcbd9a267c..9b51c7c473e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -22,7 +22,7 @@ public class SummaryFieldsMustHaveValidSource extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (DocumentSummary summary : search.getSummaries().values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java index 23569cf39ae..678d5324e38 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java @@ -26,7 +26,7 @@ public class SummaryNamesFieldCollisions extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java index 177fc7f2326..79b7a6067b9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java @@ -25,7 +25,7 @@ public class TagType extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) implementTagType(field); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java index 08571168336..8e54d7c00d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -22,7 +22,7 @@ public class TensorFieldProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index 645ed5121ea..74f30d6a730 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -34,7 +34,7 @@ public class TextMatch extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.getMatching().getType() != Matching.Type.TEXT) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java index ac376982cfa..d81fdf70d20 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -28,7 +28,7 @@ public class UriHack extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if (field.doesIndexing()) { DataType fieldType = field.getDataType(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java index ed813b42fff..c6b83349691 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java @@ -18,7 +18,7 @@ public class UrlFieldValidator extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; for (SDField field : search.allConcreteFields()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java index 54ad9f13f6f..21b7f1d2675 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -28,7 +28,7 @@ public class ValidateFieldTypes extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; String searchName = search.getName(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java index a0b204a25f2..408d60e1cff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -21,7 +21,7 @@ public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; Matching defaultMatching = new Matching(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java index 892bcdad12c..13fe3f24d69 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -25,7 +25,7 @@ public class WordMatch extends Processor { super(search, deployLogger, rankProfileRegistry, queryProfiles); } - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { for (SDField field : search.allConcreteFields()) { if ( ! field.getMatching().getType().equals(Matching.Type.WORD)) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index cc1638347f6..ec4cbdfe58b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.Optional; /** - * Class that processes a search instance and sets type settings on all rank profiles. + * This processes a search instance and sets type settings on all rank profiles. * * Currently, type settings are limited to the type of tensor attribute fields and tensor query features. * @@ -35,7 +35,9 @@ public class RankProfileTypeSettingsProcessor extends Processor { } @Override - public void process(boolean validate) { + public void process(boolean validate, boolean documentsOnly) { + if (documentsOnly) return; + processAttributeFields(); processImportedFields(); processQueryProfileTypes(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 282e5a29962..4b70b1b5ae2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -32,6 +32,7 @@ import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel; @@ -236,7 +237,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), model.name(), profile, queryProfiles, model); for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue()); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } @@ -248,7 +249,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { - profile.addMacro(entry.getKey(), false).setRankingExpression(entry.getValue()); + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 09990c7b9de..11736256d1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -2,9 +2,12 @@ package com.yahoo.vespa.model.container; import ai.vespa.models.evaluation.ModelsEvaluator; +import ai.vespa.models.handler.ModelsEvaluationHandler; +import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.model.container.component.Handler; import java.util.List; import java.util.Objects; @@ -16,12 +19,17 @@ import java.util.Objects; */ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, RankingConstantsConfig.Producer { + private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); + private final static String REST_HANDLER_NAME = ModelsEvaluationHandler.class.getName(); + private final static String BUNDLE_NAME = "model-evaluation"; + /** Global rank profiles, aka models */ private final RankProfileList rankProfileList; public ContainerModelEvaluation(ContainerCluster cluster, RankProfileList rankProfileList) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); - cluster.addSimpleComponent(ModelsEvaluator.class.getName(), null, "model-evaluation"); + cluster.addSimpleComponent(EVALUATOR_NAME, null, BUNDLE_NAME); + cluster.addComponent(ContainerModelEvaluation.getHandler()); } public void prepare(List<Container> containers) { @@ -38,4 +46,14 @@ public class ContainerModelEvaluation implements RankProfilesConfig.Producer, Ra rankProfileList.getConfig(builder); } + public static Handler<?> getHandler() { + Handler<?> handler = new Handler<>(new ComponentModel(REST_HANDLER_NAME, null, BUNDLE_NAME)); + String binding = ModelsEvaluationHandler.API_ROOT + "/" + ModelsEvaluationHandler.VERSION_V1; + handler.addServerBindings("http://*/" + binding, + "https://*/" + binding, + "http://*/" + binding + "/*", + "https://*/" + binding + "/*"); + return handler; + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index e2236feb336..adf5c81283e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -13,6 +13,7 @@ import com.yahoo.searchdefinition.FeatureNames; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; @@ -139,7 +140,7 @@ public class ConvertedModel { public ExpressionNode expression(FeatureArguments arguments, RankProfileTransformContext context) { RankingExpression expression = selectExpression(arguments); if (sourceModel.isPresent()) // we can verify - verifyRequiredMacros(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); + verifyRequiredFunctions(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); return expression.getRoot(); } @@ -183,41 +184,41 @@ public class ConvertedModel { QueryProfileRegistry queryProfiles, ModelStore store) { // Add constants - Set<String> constantsReplacedByMacros = new HashSet<>(); + Set<String> constantsReplacedByFunctions = new HashSet<>(); model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); model.largeConstants().forEach((k, v) -> transformLargeConstant(store, profile, queryProfiles, - constantsReplacedByMacros, k, v)); + constantsReplacedByFunctions, k, v)); - // Add macros - addGeneratedMacros(model, profile); + // Add functions + addGeneratedFunctions(model, profile); // Add expressions Map<String, RankingExpression> expressions = new HashMap<>(); for (Pair<String, RankingExpression> output : model.outputExpressions()) { addExpression(output.getSecond(), output.getFirst(), - constantsReplacedByMacros, + constantsReplacedByFunctions, model, store, profile, queryProfiles, expressions); } - // Transform and save macro - must come after reading expressions due to optimization transforms - // and must use the macro expression added to the profile, which may differ from the one saved in the model, + // Transform and save function - must come after reading expressions due to optimization transforms + // and must use the function expression added to the profile, which may differ from the one saved in the model, // after rewrite - model.macros().forEach((k, v) -> transformGeneratedMacro(store, constantsReplacedByMacros, k, - profile.getMacros().get(k).getRankingExpression())); + model.functions().forEach((k, v) -> transformGeneratedFunction(store, constantsReplacedByFunctions, k, + profile.getFunctions().get(k).function().getBody())); return expressions; } private static void addExpression(RankingExpression expression, String expressionName, - Set<String> constantsReplacedByMacros, + Set<String> constantsReplacedByFunctions, ImportedModel model, ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, Map<String, RankingExpression> expressions) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); reduceBatchDimensions(expression, model, profile, queryProfiles); store.writeExpression(expressionName, expression); expressions.put(expressionName, expression); @@ -232,8 +233,8 @@ public class ConvertedModel { profile.rankingConstants().add(constant); } - for (Pair<String, RankingExpression> macro : store.readMacros()) { - addGeneratedMacroToProfile(profile, macro.getFirst(), macro.getSecond()); + for (Pair<String, RankingExpression> function : store.readFunctions()) { + addGeneratedFunctionToProfile(profile, function.getFirst(), function.getSecond()); } return store.readExpressions(); @@ -247,16 +248,16 @@ public class ConvertedModel { private static void transformLargeConstant(ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, - Set<String> constantsReplacedByMacros, + Set<String> constantsReplacedByFunctions, String constantName, Tensor constantValue) { - RankProfile.Macro macroOverridingConstant = profile.getMacros().get(constantName); - if (macroOverridingConstant != null) { - TensorType macroType = macroOverridingConstant.getRankingExpression().type(profile.typeContext(queryProfiles)); - if ( ! macroType.equals(constantValue.type())) - throw new IllegalArgumentException("Macro '" + constantName + "' replaces the constant with this name. " + - typeMismatchExplanation(constantValue.type(), macroType)); - constantsReplacedByMacros.add(constantName); // will replace constant(constantName) by constantName later + RankProfile.RankingExpressionFunction rankingExpressionFunctionOverridingConstant = profile.getFunctions().get(constantName); + if (rankingExpressionFunctionOverridingConstant != null) { + TensorType functionType = rankingExpressionFunctionOverridingConstant.function().getBody().type(profile.typeContext(queryProfiles)); + if ( ! functionType.equals(constantValue.type())) + throw new IllegalArgumentException("Function '" + constantName + "' replaces the constant with this name. " + + typeMismatchExplanation(constantValue.type(), functionType)); + constantsReplacedByFunctions.add(constantName); // will replace constant(constantName) by constantName later } else { Path constantPath = store.writeLargeConstant(constantName, constantValue); @@ -267,79 +268,75 @@ public class ConvertedModel { } } - private static void transformGeneratedMacro(ModelStore store, - Set<String> constantsReplacedByMacros, - String macroName, - RankingExpression expression) { + private static void transformGeneratedFunction(ModelStore store, + Set<String> constantsReplacedByFunctions, + String functionName, + RankingExpression expression) { - expression = replaceConstantsByMacros(expression, constantsReplacedByMacros); - store.writeMacro(macroName, expression); + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); + store.writeFunction(functionName, expression); } - private static void addGeneratedMacroToProfile(RankProfile profile, String macroName, RankingExpression expression) { - if (profile.getMacros().containsKey(macroName)) { - if ( ! profile.getMacros().get(macroName).getRankingExpression().equals(expression)) - throw new IllegalArgumentException("Generated macro '" + macroName + "' already exists in " + profile + + private static void addGeneratedFunctionToProfile(RankProfile profile, String functionName, RankingExpression expression) { + if (profile.getFunctions().containsKey(functionName)) { + if ( ! profile.getFunctions().get(functionName).function().getBody().equals(expression)) + throw new IllegalArgumentException("Generated function '" + functionName + "' already exists in " + profile + " - with a different definition" + - ": Has\n" + profile.getMacros().get(macroName).getRankingExpression() + + ": Has\n" + profile.getFunctions().get(functionName).function().getBody() + "\nwant to add " + expression + "\n"); return; } - RankProfile.Macro macro = profile.addMacro(macroName, false); // TODO: Inline if only used once - macro.setRankingExpression(expression); - macro.setTextualExpression(expression.getRoot().toString()); + profile.addFunction(new ExpressionFunction(functionName, expression), false); // TODO: Inline if only used once } /** - * Verify that the macros referred in the given expression exists in the given rank profile, - * and return tensors of the types specified in requiredMacros. + * Verify that the functions referred in the given expression exists in the given rank profile, + * and return tensors of the types specified in requiredFunctions. */ - private static void verifyRequiredMacros(RankingExpression expression, ImportedModel model, - RankProfile profile, QueryProfileRegistry queryProfiles) { - Set<String> macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - TensorType requiredType = model.requiredMacros().get(macroName); - if (requiredType == null) continue; // Not a required macro - - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) - throw new IllegalArgumentException("Model refers input '" + macroName + - "' of type " + requiredType + " but this macro is not present in " + + private static void verifyRequiredFunctions(RankingExpression expression, ImportedModel model, + RankProfile profile, QueryProfileRegistry queryProfiles) { + Set<String> functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + TensorType requiredType = model.requiredFunctions().get(functionName); + if (requiredType == null) continue; // Not a required function + + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); + if (rankingExpressionFunction == null) + throw new IllegalArgumentException("Model refers input '" + functionName + + "' of type " + requiredType + " but this function is not present in " + profile); // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second // phase and summary features), as it may only resolve correctly given those bindings - // Or, probably better, annotate the macros with type constraints here and verify during general + // Or, probably better, annotate the functions with type constraints here and verify during general // type verification - TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); + TensorType actualType = rankingExpressionFunction.function().getBody().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) - throw new IllegalArgumentException("Model refers input '" + macroName + + throw new IllegalArgumentException("Model refers input '" + functionName + "' of type " + requiredType + - " which must be produced by a macro in the rank profile, but " + - "this macro references a feature which is not declared"); + " which must be produced by a function in the rank profile, but " + + "this function references a feature which is not declared"); if ( ! actualType.isAssignableTo(requiredType)) - throw new IllegalArgumentException("Model refers input '" + macroName + "'. " + + throw new IllegalArgumentException("Model refers input '" + functionName + "'. " + typeMismatchExplanation(requiredType, actualType)); } } private static String typeMismatchExplanation(TensorType requiredType, TensorType actualType) { - return "The required type of this is " + requiredType + ", but this macro returns " + actualType + + return "The required type of this is " + requiredType + ", but this function returns " + actualType + (actualType.rank() == 0 ? ". This is often due to missing declaration of query tensor features " + "in query profile types - see the documentation." : ""); } - /** - * Add the generated macros to the rank profile - */ - private static void addGeneratedMacros(ImportedModel model, RankProfile profile) { - model.macros().forEach((k, v) -> addGeneratedMacroToProfile(profile, k, v.copy())); + /** Add the generated functions to the rank profile */ + private static void addGeneratedFunctions(ImportedModel model, RankProfile profile) { + model.functions().forEach((k, v) -> addGeneratedFunctionToProfile(profile, k, v.copy())); } /** * Check if batch dimensions of inputs can be reduced out. If the input - * macro specifies that a single exemplar should be evaluated, we can + * function specifies that a single exemplar should be evaluated, we can * reduce the batch dimension out. */ private static void reduceBatchDimensions(RankingExpression expression, ImportedModel model, @@ -347,19 +344,19 @@ public class ConvertedModel { TypeContext<Reference> typeContext = profile.typeContext(queryProfiles); TensorType typeBeforeReducing = expression.getRoot().type(typeContext); - // Check generated macros for inputs to reduce - Set<String> macroNames = new HashSet<>(); - addMacroNamesIn(expression.getRoot(), macroNames, model); - for (String macroName : macroNames) { - if ( ! model.macros().containsKey(macroName)) continue; + // Check generated functions for inputs to reduce + Set<String> functionNames = new HashSet<>(); + addFunctionNamesIn(expression.getRoot(), functionNames, model); + for (String functionName : functionNames) { + if ( ! model.functions().containsKey(functionName)) continue; - RankProfile.Macro macro = profile.getMacros().get(macroName); - if (macro == null) { - throw new IllegalArgumentException("Model refers to generated macro '" + macroName + - "but this macro is not present in " + profile); + RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); + if (rankingExpressionFunction == null) { + throw new IllegalArgumentException("Model refers to generated function '" + functionName + + "but this function is not present in " + profile); } - RankingExpression macroExpression = macro.getRankingExpression(); - macroExpression.setRoot(reduceBatchDimensionsAtInput(macroExpression.getRoot(), model, typeContext)); + RankingExpression functionExpression = rankingExpressionFunction.function().getBody(); + functionExpression.setRoot(reduceBatchDimensionsAtInput(functionExpression.getRoot(), model, typeContext)); } // Check expression for inputs to reduce @@ -378,7 +375,7 @@ public class ConvertedModel { List<ExpressionNode> children = ((TensorFunctionNode)node).children(); if (children.size() == 1 && children.get(0) instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) children.get(0); - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(tensorFunction, typeContext); } } @@ -386,7 +383,7 @@ public class ConvertedModel { } if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) node; - if (model.requiredMacros().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext); } } @@ -447,47 +444,47 @@ public class ConvertedModel { } /** - * If a constant c is overridden by a macro, we need to replace instances of "constant(c)" by "c" in expressions. + * If a constant c is overridden by a function, we need to replace instances of "constant(c)" by "c" in expressions. * This method does that for the given expression and returns the result. */ - private static RankingExpression replaceConstantsByMacros(RankingExpression expression, - Set<String> constantsReplacedByMacros) { - if (constantsReplacedByMacros.isEmpty()) return expression; + private static RankingExpression replaceConstantsByFunctions(RankingExpression expression, + Set<String> constantsReplacedByFunctions) { + if (constantsReplacedByFunctions.isEmpty()) return expression; return new RankingExpression(expression.getName(), - replaceConstantsByMacros(expression.getRoot(), constantsReplacedByMacros)); + replaceConstantsByFunctions(expression.getRoot(), constantsReplacedByFunctions)); } - private static ExpressionNode replaceConstantsByMacros(ExpressionNode node, Set<String> constantsReplacedByMacros) { + private static ExpressionNode replaceConstantsByFunctions(ExpressionNode node, Set<String> constantsReplacedByFunctions) { if (node instanceof ReferenceNode) { Reference reference = ((ReferenceNode)node).reference(); if (FeatureNames.isSimpleFeature(reference) && reference.name().equals("constant")) { String argument = reference.simpleArgument().get(); - if (constantsReplacedByMacros.contains(argument)) + if (constantsReplacedByFunctions.contains(argument)) return new ReferenceNode(argument); } } if (node instanceof CompositeNode) { // not else: this matches some of the same nodes as the outer if above CompositeNode composite = (CompositeNode)node; return composite.setChildren(composite.children().stream() - .map(child -> replaceConstantsByMacros(child, constantsReplacedByMacros)) + .map(child -> replaceConstantsByFunctions(child, constantsReplacedByFunctions)) .collect(Collectors.toList())); } return node; } - private static void addMacroNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) { + private static void addFunctionNamesIn(ExpressionNode node, Set<String> names, ImportedModel model) { if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode)node; - if (referenceNode.getOutput() == null) { // macro references cannot specify outputs + if (referenceNode.getOutput() == null) { // function references cannot specify outputs names.add(referenceNode.getName()); - if (model.macros().containsKey(referenceNode.getName())) { - addMacroNamesIn(model.macros().get(referenceNode.getName()).getRoot(), names, model); + if (model.functions().containsKey(referenceNode.getName())) { + addFunctionNamesIn(model.functions().get(referenceNode.getName()).getRoot(), names, model); } } } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode)node).children()) - addMacroNamesIn(child, names, model); + addFunctionNamesIn(child, names, model); } } @@ -551,19 +548,19 @@ public class ConvertedModel { return expressions; } - /** Adds this macro expression to the application package so it can be read later. */ - void writeMacro(String name, RankingExpression expression) { - application.getFile(modelFiles.macrosPath()).appendFile(name + "\t" + - expression.getRoot().toString() + "\n"); + /** Adds this function expression to the application package so it can be read later. */ + void writeFunction(String name, RankingExpression expression) { + application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" + + expression.getRoot().toString() + "\n"); } - /** Reads the previously stored macro expressions for these arguments */ - List<Pair<String, RankingExpression>> readMacros() { + /** Reads the previously stored function expressions for these arguments */ + List<Pair<String, RankingExpression>> readFunctions() { try { - ApplicationFile file = application.getFile(modelFiles.macrosPath()); + ApplicationFile file = application.getFile(modelFiles.functionsPath()); if ( ! file.exists()) return Collections.emptyList(); - List<Pair<String, RankingExpression>> macros = new ArrayList<>(); + List<Pair<String, RankingExpression>> functions = new ArrayList<>(); BufferedReader reader = new BufferedReader(file.createReader()); String line; while (null != (line = reader.readLine())) { @@ -571,13 +568,13 @@ public class ConvertedModel { String name = parts[0]; try { RankingExpression expression = new RankingExpression(parts[0], parts[1]); - macros.add(new Pair<>(name, expression)); + functions.add(new Pair<>(name, expression)); } catch (ParseException e) { throw new IllegalStateException("Could not parse " + name, e); } } - return macros; + return functions; } catch (IOException e) { throw new UncheckedIOException(e); @@ -725,9 +722,9 @@ public class ConvertedModel { return storedModelReplicatedPath().append("constants"); } - /** Path to the macros file */ - public Path macrosPath() { - return storedModelReplicatedPath().append("macros.txt"); + /** Path to the functions file */ + public Path functionsPath() { + return storedModelReplicatedPath().append("functions.txt"); } } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 63d3926afad..813d1d47533 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -34,6 +34,7 @@ import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferen import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.DocumentsOnlyRankProfile; import com.yahoo.searchdefinition.DefaultRankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; @@ -58,6 +59,8 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.query.ranking.Diversity; import java.util.Map; +import java.util.List; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.logging.Level; import org.apache.commons.lang.StringUtils; @@ -73,6 +76,7 @@ public class SDParser { private ApplicationPackage app; private DeployLogger deployLogger; private RankProfileRegistry rankProfileRegistry; + private boolean documentsOnly; /** For testing only */ public SDParser(String input, DeployLogger deployLogger) { @@ -81,17 +85,24 @@ public class SDParser { /** For testing only */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger) { - this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry()); + this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry(), false); } + /** + * Creates a parser + * + * @param documentsOnly true to only parse the document aspect of a search definition (e.g skip rank profiles) + */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger, ApplicationPackage applicationPackage, - RankProfileRegistry rankProfileRegistry) { + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) { this(stream); this.deployLogger = deployLogger; this.app = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; } /** @@ -1804,6 +1815,7 @@ void rankingConstant(Search search) : } lbrace() (rankingConstantItem(constant) (<NL>)*)+ <RBRACE> ) { + if (documentsOnly) return; search.rankingConstants().add(constant); } } @@ -1859,7 +1871,10 @@ void rankProfile(Search search) : { ( <RANKPROFILE> name = identifier() { - if ("default".equals(name)) { + if (documentsOnly) { + profile = new DocumentsOnlyRankProfile(name, search, rankProfileRegistry); + } + else if ("default".equals(name)) { profile = rankProfileRegistry.get(search, "default"); } else { profile = new RankProfile(name, search, rankProfileRegistry); @@ -1868,6 +1883,7 @@ void rankProfile(Search search) : [inheritsRankProfile(profile)] lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> ) { + if (documentsOnly) return; rankProfileRegistry.add(profile); } } @@ -1885,7 +1901,7 @@ Object rankProfileItem(RankProfile profile) : { } | fieldRankFilter(profile) | firstPhase(profile) | matchPhase(profile) - | macro(profile) + | function(profile) | ignoreRankFeatures(profile) | numThreadsPerSearch(profile) | minHitsPerThread(profile) @@ -1910,7 +1926,8 @@ void inheritsRankProfile(RankProfile profile) : String str; } { - <INHERITS> str = identifier() { profile.setInherited(str); } + <INHERITS> str = identifier() + { profile.setInherited(str); } } /** @@ -1918,19 +1935,20 @@ void inheritsRankProfile(RankProfile profile) : * * @param profile The profile to modify. */ -void macro(RankProfile profile) : +void function(RankProfile profile) : { - String macro, param, expr; + String name, expression, parameter; + List parameters = new ArrayList(); boolean inline = false; } { - ( <MACRO> inline = inline() macro = identifier() [ "$" { macro = macro + token.image; } ] - "(" { profile.addMacro(macro, inline); } - [ param = identifier() { profile.getMacros().get(macro).addParam(param); } - ( <COMMA> param = identifier() { profile.getMacros().get(macro).addParam(param); } )* ] + ( ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ] + "(" + [ parameter = identifier() { parameters.add(parameter); } + ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ] ")" - lbrace() expr = expression() (<NL>)* <RBRACE> ) - { profile.getMacros().get(macro).setTextualExpression(expr); } + lbrace() expression = expression() (<NL>)* <RBRACE> ) + { profile.addFunction(name, parameters, expression, inline); } } boolean inline() : @@ -2034,7 +2052,7 @@ Object firstPhaseItem(RankProfile profile) : double dropLimit; } { - ( expression = expression() { profile.setFirstPhaseRankingString(expression); } + ( expression = expression() { profile.setFirstPhaseRanking(expression); } | (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); } | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } ) @@ -2063,7 +2081,7 @@ Object secondPhaseItem(RankProfile profile) : int rerankCount; } { - ( expression = expression() { profile.setSecondPhaseRankingString(expression); } + ( expression = expression() { profile.setSecondPhaseRanking(expression); } | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } ) { return null; } diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy index 8d2d65b5551..4ed8311d7ae 100755 --- a/config-model/src/main/perl/vespa-deploy +++ b/config-model/src/main/perl/vespa-deploy @@ -240,6 +240,14 @@ sub usage { print "Usage: vespa-deploy [-h] [-v] [-f] [-t] [-p] [-V] [<command>] [args]\n"; print "Supported commands: 'upload', 'prepare', 'activate', 'fetch' and 'help'\n"; print "Supported options: '-h' (help), '-v' (verbose), '-f' (force/ignore validation errors), '-t' (timeout in seconds), '-p' (config server http port)\n"; + print " '-h' (help)\n"; + print " '-v' (verbose)\n"; + print " '-n' (dry-run)\n"; + print " '-f' (force/ignore validation errors)\n"; + print " '-t <timeout>' (timeout in seconds)\n"; + print " '-c <server>' (config server hostname)\n"; + print " '-p <port>' (config server http port)\n\n"; + print "Try 'vespa-deploy help <command>' to get more help\n"; } } diff --git a/config-model/src/test/derived/gemini2/gemini.sd b/config-model/src/test/derived/gemini2/gemini.sd index 18be346a758..01e20c1b30a 100644 --- a/config-model/src/test/derived/gemini2/gemini.sd +++ b/config-model/src/test/derived/gemini2/gemini.sd @@ -6,19 +6,19 @@ search gemini { rank-profile test { - macro wrapper2(x) { + function wrapper2(x) { expression: x } - macro wrapper1(x) { + function wrapper1(x) { expression: wrapper2(x) } - macro toplevel() { + function toplevel() { expression: wrapper1(attribute(right)) } - macro interfering() { + function interfering() { expression: wrapper1(attribute(wrong)) } diff --git a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd index 7f09095c5e7..6e399c03a2c 100644 --- a/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd +++ b/config-model/src/test/examples/rankingexpressionfunction/rankingexpressionfunction.sd @@ -22,11 +22,11 @@ search rankexpression { } rank-profile macros { - macro titlematch$(var1, var2) { + function titlematch$(var1, var2) { expression: file: titlematch } - macro artistmatch() { + function artistmatch() { expression: 78+closeness(distance) } diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index 96b0fa98098..0435ea439df 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -121,7 +121,7 @@ search simple { second-phase { rerank-count: 99 } - macro openTicket() { + function openTicket() { expression: if(attribute(status) == "accepted",1, if(attribute(status) == "new",1,if(attribute(status) == "reopened",1,0))) } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 03fa92f5cb9..07a36832094 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -5,10 +5,12 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.io.IOException; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,9 +29,9 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT new DerivedConfiguration(search, registry, new QueryProfileRegistry(), new ImportedModels()); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { - e.printStackTrace(); - assertTrue(e.getCause().getMessage().contains("Could not read ranking expression file")); - assertTrue(e.getCause().getMessage().contains("wrongending.expr.expression")); + String message = Exceptions.toMessageString(e); + assertTrue(message.contains("Could not read ranking expression file")); + assertTrue(message.contains("wrongending.expr.expression")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java index 28559f351ac..02ec597c3ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java @@ -4,6 +4,8 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; @@ -17,6 +19,7 @@ import static org.junit.Assert.assertNull; * @author Ulf Lilleengen */ public class RankProfileRegistryTest { + private static final String TESTDIR = "src/test/cfg/search/data/v2/inherited_rankprofiles"; @Test @@ -43,11 +46,11 @@ public class RankProfileRegistryTest { RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { - assertNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); - RankProfile rankProfileWithAddedMacro = new RankProfile(rankProfileName, search, rankProfileRegistry); - rankProfileWithAddedMacro.addMacro("foo", true); - rankProfileRegistry.add(rankProfileWithAddedMacro); - assertNotNull(rankProfileRegistry.get(search, rankProfileName).getMacros().get("foo")); + assertNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); + RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, search, rankProfileRegistry); + rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); + rankProfileRegistry.add(rankProfileWithAddedFunction); + assertNotNull(rankProfileRegistry.get(search, rankProfileName).getFunctions().get("foo")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index a524a26cbef..150469cc928 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -59,7 +59,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -76,7 +76,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile child2 = rankProfileRegistry.get(s, "child2").compile(queryProfileRegistry, new ImportedModels()); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); - assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); + assertEquals("foo: 14.0", child2.getFunctions().get("foo").function().getBody().toString()); List<Pair<String, String>> rankProperties = new RawRankProfile(child2, queryProfileRegistry, new ImportedModels(), @@ -101,7 +101,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " c: 7 \n" + " }\n" + - " macro c() {\n" + + " function c() {\n" + " expression: p2*p1\n" + " }\n" + " }\n" + @@ -114,7 +114,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase fail("Should have caused an exception"); } catch (IllegalArgumentException e) { - assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and macro named 'c'", + assertEquals("Rank profile 'test' is invalid: Cannot have both a constant and function named 'c'", Exceptions.toMessageString(e)); } } @@ -132,7 +132,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, -9.21034037)\n" + " }\n" + " }\n" + @@ -141,8 +141,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test @@ -161,7 +160,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " constants {\n" + " myValue: -9.21034037\n" + " }\n" + - " macro POP_SLOW_SCORE() {\n" + + " function POP_SLOW_SCORE() {\n" + " expression: safeLog(popShareSlowDecaySignal, myValue)\n" + " }\n" + " }\n" + @@ -170,14 +169,13 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); - profile.parseExpressions(); // TODO: Do differently - assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("POP_SLOW_SCORE").function().getBody().getRoot().toString()); } @Test - public void testConstantDivisorInMacro() throws ParseException { + public void testConstantDivisorInFunction() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -186,7 +184,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: k1 + (k2 + k3) / 100000000.0\n\n" + " }\n" + " }\n" + @@ -196,7 +194,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("k1 + (k2 + k3) / 100000000.0", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } @Test @@ -212,7 +210,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro rank_default(){\n" + + " function rank_default(){\n" + " expression: 0.5+50*(attribute(rating_yelp)-3)\n\n" + " }\n" + " }\n" + @@ -222,7 +220,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.get(s, "test"); assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", - profile.compile(new QueryProfileRegistry(), new ImportedModels()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + profile.compile(new QueryProfileRegistry(), new ImportedModels()).getFunctions().get("rank_default").function().getBody().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index b13ffabda77..e507a6c48e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -22,7 +22,7 @@ import static org.junit.Assert.fail; public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase { @Test - public void testMacroInliningPreserveArithemticOrdering() throws ParseException { + public void testFunctionInliningPreserveArithemticOrdering() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -44,18 +44,18 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: p1 * add\n" + " }\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 3 + attribute(a) + attribute(b) * mul3\n" + " }\n" + - " macro inline mul3() {\n" + + " function inline mul3() {\n" + " expression: attribute(a) * 3 + singleif\n" + " }\n" + - " macro inline singleif() {\n" + + " function inline singleif() {\n" + " expression: if (p1 < attribute(a), 1, 2) == 0\n" + " }\n" + " }\n" + " rank-profile child inherits parent {\n" + - " macro inline add() {\n" + + " function inline add() {\n" + " expression: 9 + attribute(a)\n" + " }\n" + " }\n" + @@ -95,7 +95,7 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " second-phase {\n" + " expression: p2 * foo\n" + " }\n" + - " macro inline foo() {\n" + + " function inline foo() {\n" + " expression: 3 + p1 + p2\n" + " }\n" + " }\n" + @@ -106,16 +106,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " constants {\n" + " p2: 2.0 \n" + " }\n" + - " macro bar() {\n" + + " function bar() {\n" + " expression: p2*p1\n" + " }\n" + - " macro inline baz() {\n" + + " function inline baz() {\n" + " expression: p2+p1+boz\n" + " }\n" + - " macro inline boz() {\n" + + " function inline boz() {\n" + " expression: 3.0\n" + " }\n" + - " macro inline arg(a1) {\n" + + " function inline arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -162,16 +162,16 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase " first-phase {\n" + " expression: A + C + D\n" + " }\n" + - " macro inline D() {\n" + + " function inline D() {\n" + " expression: B + 1\n" + " }\n" + - " macro C() {\n" + + " function C() {\n" + " expression: A + B\n" + " }\n" + - " macro inline B() {\n" + + " function inline B() {\n" + " expression: attribute(b)\n" + " }\n" + - " macro inline A() {\n" + + " function inline A() {\n" + " expression: attribute(a)\n" + " }\n" + " }\n" + @@ -187,8 +187,8 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase } /** - * Expression evaluation has no stack so macro arguments are bound at config time creating a separate version of - * each macro for each binding, using hashes to name the bound variants of the macro. + * Expression evaluation has no stack so function arguments are bound at config time creating a separate version of + * each function for each binding, using hashes to name the bound variants of the function. * This method censors those hashes for string comparison. */ private String censorBindingHash(String s) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index df9a40d29e2..17bebcba70e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -29,7 +29,7 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: foo\n" + " }\n" + " }\n" + @@ -61,10 +61,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(5)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: foo + a1*2\n" + " }\n" + " }\n" + @@ -96,10 +96,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo\n" + " }\n" + - " macro foo() {\n" + + " function foo() {\n" + " expression: arg(foo)\n" + " }\n" + - " macro arg(a1) {\n" + + " function arg(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -131,10 +131,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -159,10 +159,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(3)\n" + " }\n" + - " macro foo(a1) {\n" + + " function foo(a1) {\n" + " expression: bar(3) + bar(a1)\n" + " }\n" + - " macro bar(a1) {\n" + + " function bar(a1) {\n" + " expression: a1*2\n" + " }\n" + " }\n" + @@ -183,10 +183,10 @@ public class RankingExpressionLoopDetectionTestCase { " first-phase {\n" + " expression: foo(bar(2))\n" + " }\n" + - " macro foo(x) {\n" + + " function foo(x) {\n" + " expression: x * x\n" + " }\n" + - " macro bar(x) {\n" + + " function bar(x) {\n" + " expression: x + x\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 1ece2355a92..e15d4075b19 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase { @Test - public void testBasicMacroShadowing() throws ParseException { + public void testBasicFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -31,7 +31,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -57,7 +57,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMultiLevelMacroShadowing() throws ParseException { + public void testMultiLevelFunctionShadowing() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -69,13 +69,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro tan(x) {\n" + + " function tan(x) {\n" + " expression: x * x\n" + " }\n" + - " macro cos(x) {\n" + + " function cos(x) {\n" + " expression: tan(x)\n" + " }\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: cos(x)\n" + " }\n" + " first-phase {\n" + @@ -113,7 +113,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase @Test - public void testMacroShadowingArguments() throws ParseException { + public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( @@ -125,7 +125,7 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -168,13 +168,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " }\n" + " \n" + " rank-profile test {\n" + - " macro relu(x) {\n" + // relu is a built in function, redefined here + " function relu(x) {\n" + // relu is a built in function, redefined here " expression: max(1.0, x)\n" + " }\n" + - " macro hidden_layer() {\n" + + " function hidden_layer() {\n" + " expression: relu(sum(query(q) * constant(W_hidden), input) + constant(b_input))\n" + " }\n" + - " macro final_layer() {\n" + + " function final_layer() {\n" + " expression: sigmoid(sum(hidden_layer * constant(W_final), hidden) + constant(b_final))\n" + " }\n" + " second-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index 3fe3a7c3de1..5e649c2e551 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -5,9 +5,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.yolean.Exceptions; import org.junit.Ignore; import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -29,9 +31,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success - // TODO: Where's the "com.yahoo.searchdefinition.parser.ParseException:" nonsense coming from? - assertTrue("Got unexpected error message: " + e.getCause().getMessage(), - e.getCause().getMessage().startsWith("com.yahoo.searchdefinition.parser.ParseException: Could not parse ranking expression '" + expression + "'")); + assertTrue(Exceptions.toMessageString(e).startsWith("Illegal first phase ranking function: Could not parse ranking expression '" + expression + "' in default, firstphase.:")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 8cdfdd51637..82c03c02f61 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -34,20 +34,19 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { assertEquals("simple",search.getName()); assertTrue(search.hasDocument()); - SDDocumentType document=search.getDocument(); - assertEquals("simple",document.getName()); - assertEquals(12,document.getFieldCount()); + SDDocumentType document = search.getDocument(); + assertEquals("simple", document.getName()); + assertEquals(12, document.getFieldCount()); SDField field; Attribute attribute; - new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true); + new MakeAliases(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()).process(true, false); // First field field=(SDField) document.getField("title"); assertEquals(DataType.STRING,field.getDataType()); - assertEquals("{ summary | index; }", - field.getIndexingScript().toString()); + assertEquals("{ summary | index; }", field.getIndexingScript().toString()); assertTrue(!search.getIndex("default").isPrefix()); assertTrue(search.getIndex("title").isPrefix()); Iterator<String> titleAliases=search.getIndex("title").aliasIterator(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java index 274cf06aa37..f87a26c6f79 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java @@ -31,7 +31,7 @@ public class IdTestCase extends AbstractExportingTestCase { uri.parseIndexingScript("{ summary | index }"); document.addField(uri); - Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); assertNull(document.getField("uri")); assertNull(document.getField("Uri")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index 94d0bf6329a..2bae285301c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -41,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); // Check attribute fields diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index 0e02970280e..ba19a8312f6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -69,14 +69,14 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { assertTrue(!transforms.hasNext()); } @Test - public void testPositionDeriving() throws IOException, ParseException { + public void testPositionDeriving() { Search search = new Search("store", null); SDDocumentType document = new SDDocumentType("store"); search.addDocument(document); String fieldName = "location"; SDField field = document.addField(fieldName, PositionDataType.INSTANCE); field.parseIndexingScript("{ attribute | summary }"); - Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); SummaryMap summaryMap = new SummaryMap(search, new Summaries(search, new BaseDeployLogger())); Iterator transforms = summaryMap.resultTransformIterator(); @@ -141,7 +141,7 @@ public class SummaryMapTestCase extends SearchDefinitionTestCase { } @Test - public void testFailOnSummaryFieldSourceCollision() throws IOException, ParseException { + public void testFailOnSummaryFieldSourceCollision() { try { Search search = SearchBuilder.buildFromFile("src/test/examples/summaryfieldcollision.sd"); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index 26b100a2d96..8941b07101d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -33,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase { a.parseIndexingScript("{ index }"); document.addField(a); - Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles(), true, false); DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedModels()); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index a0556a156b5..48adc0eefc5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -36,7 +36,7 @@ public class AddAttributeTransformToSummaryOfImportedFieldsTest { AddAttributeTransformToSummaryOfImportedFields processor = new AddAttributeTransformToSummaryOfImportedFields( search,null,null,null); - processor.process(true); + processor.process(true, false); SummaryField summaryField = search.getSummaries().get(SUMMARY_NAME).getSummaryField(IMPORTED_FIELD_NAME); SummaryTransform actualTransform = summaryField.getTransform(); assertEquals(SummaryTransform.ATTRIBUTE, actualTransform); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java index 8f1cc9c3de8..3a0fedfd550 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributePropertiesTestCase.java @@ -24,7 +24,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { public void testInvalidAttributeProperties() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties1.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail("attribute property should not be set"); } catch (RuntimeException e) { // empty @@ -34,7 +34,7 @@ public class AttributePropertiesTestCase extends SearchDefinitionTestCase { @Test public void testValidAttributeProperties() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/attributeproperties2.sd"); - new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new AttributeProperties(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java index 4dee939e3da..1ab8b054cb7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java @@ -24,7 +24,7 @@ public class BoldingTestCase extends SearchDefinitionTestCase { public void testBoldingNonString() throws IOException, ParseException { try { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/processing/boldnonstring.sd"); - new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new Bolding(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); fail(); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("'bolding: on' for non-text field")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java index 7230f176048..de08bf66548 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java @@ -188,7 +188,7 @@ public class ImportedFieldsResolverTestCase { private static ImportedFields resolve(Search search) { assertNotNull(search.temporaryImportedFields().get()); assertFalse(search.importedFields().isPresent()); - new ImportedFieldsResolver(search, null, null, null).process(true); + new ImportedFieldsResolver(search, null, null, null).process(true, false); assertFalse(search.temporaryImportedFields().isPresent()); assertNotNull(search.importedFields().get()); return search.importedFields().get(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index 876e852aea1..e078d91f248 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -155,7 +155,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { sdoc.addField(unprocessedField); Search search = new Search("test", null); search.addDocument(sdoc); - Processing.process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true); + new Processing().process(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles(), true, false); return unprocessedField.getIndexingScript(); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java index 1e0270f293d..29bba224f46 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java @@ -19,11 +19,12 @@ import static org.junit.Assert.assertTrue; * @author baldersheim */ public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase { + @Test public void testIntegerIndex2Attribute() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/integerindex2attribute.sd"); search.process(); - new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); + new IntegerIndex2Attribute(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); SDField f; f = search.getConcreteField("s1"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index f67c85e2881..cff9abb08ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -72,9 +72,9 @@ class RankProfileSearchFixture { assertEquals(expValue, rankPropertyList.get(0).getValue()); } - public void assertMacro(String expexctedExpression, String macroName, String rankProfile) { + public void assertFunction(String expexctedExpression, String functionName, String rankProfile) { assertEquals(expexctedExpression, - compiledRankProfile(rankProfile).getMacros().get(macroName).getRankingExpression().getRoot().toString()); + compiledRankProfile(rankProfile).getFunctions().get(functionName).function().getBody().getRoot().toString()); } public RankProfile compileRankProfile(String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java index d8eb4368b57..0d8cbbf2e6a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java @@ -109,7 +109,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testMacroInvocationTypes() throws Exception { + public void testFunctionInvocationTypes() throws Exception { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString(joinLines( @@ -123,7 +123,7 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro macro1(attribute_to_use) {", + " function macro1(attribute_to_use) {", " expression: attribute(attribute_to_use)", " }", " summary-features {", @@ -143,7 +143,7 @@ public class RankingExpressionTypeValidatorTestCase { } @Test - public void testTensorMacroInvocationTypes_Nested() throws Exception { + public void testTensorFunctionInvocationTypes_Nested() throws Exception { SearchBuilder builder = new SearchBuilder(); builder.importString(joinLines( "search test {", @@ -156,16 +156,16 @@ public class RankingExpressionTypeValidatorTestCase { " }", " }", " rank-profile my_rank_profile {", - " macro return_a() {", + " function return_a() {", " expression: return_first(attribute(a), attribute(b))", " }", - " macro return_b() {", + " function return_b() {", " expression: return_second(attribute(a), attribute(b))", " }", - " macro return_first(e1, e2) {", + " function return_first(e1, e2) {", " expression: e1", " }", - " macro return_second(e1, e2) {", + " function return_second(e1, e2) {", " expression: return_first(e2, e1)", " }", " summary-features {", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index b046d60f948..8944409e1e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -128,7 +128,7 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testOnnxReferenceMissingMacro() throws ParseException { + public void testOnnxReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -145,14 +145,14 @@ public class RankingExpressionWithOnnxTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testOnnxReferenceWithWrongMacroType() { + public void testOnnxReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "onnx('mnist_softmax.onnx')"); @@ -163,7 +163,7 @@ public class RankingExpressionWithOnnxTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use Onnx model from " + "onnx('mnist_softmax.onnx'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -213,13 +213,13 @@ public class RankingExpressionWithOnnxTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstant() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstant() throws IOException { String rankProfile = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_Variable() {\n" + + " function " + name + "_Variable() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -234,7 +234,7 @@ public class RankingExpressionWithOnnxTestCase { search.compileRankProfile("my_profile", applicationDir.append("models")); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get( name + "_Variable")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -247,7 +247,7 @@ public class RankingExpressionWithOnnxTestCase { RankProfileSearchFixture searchFromStored = uncompiledFixtureWith(rankProfile, storedApplication); searchFromStored.compileRankProfile("my_profile", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get( name + "_Variable")); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -275,19 +275,19 @@ public class RankingExpressionWithOnnxTestCase { } } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 14632a568ea..cba931e81f0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -160,7 +160,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceMissingMacro() throws ParseException { + public void testTensorFlowReferenceMissingFunction() throws ParseException { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), @@ -177,14 +177,14 @@ public class RankingExpressionWithTensorFlowTestCase { catch (IllegalArgumentException expected) { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + - "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this macro is " + + "Model refers input 'Placeholder' of type tensor(d0[],d1[784]) but this function is " + "not present in rank profile 'my_profile'", Exceptions.toMessageString(expected)); } } @Test - public void testTensorFlowReferenceWithWrongMacroType() { + public void testTensorFlowReferenceWithWrongFunctionType() { try { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d5[10])(0.0)", "tensorflow('mnist_softmax/saved')"); @@ -195,7 +195,7 @@ public class RankingExpressionWithTensorFlowTestCase { assertEquals("Rank profile 'my_profile' is invalid: Could not use tensorflow model from " + "tensorflow('mnist_softmax/saved'): " + "Model refers input 'Placeholder'. The required type of this is tensor(d0[],d1[784]), " + - "but this macro returns tensor(d0[2],d5[10])", + "but this function returns tensor(d0[2],d5[10])", Exceptions.toMessageString(expected)); } } @@ -261,13 +261,13 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testImportingFromStoredExpressionsWithMacroOverridingConstantAndInheritance() throws IOException { + public void testImportingFromStoredExpressionsWithFunctionOverridingConstantAndInheritance() throws IOException { String rankProfiles = " rank-profile my_profile {\n" + - " macro Placeholder() {\n" + + " function Placeholder() {\n" + " expression: tensor(d0[2],d1[784])(0.0)\n" + " }\n" + - " macro " + name + "_layer_Variable_read() {\n" + + " function " + name + "_layer_Variable_read() {\n" + " expression: tensor(d1[10],d2[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -285,7 +285,7 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", search.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); // At this point the expression is stored - copy application to another location which do not have a models dir @@ -300,7 +300,7 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.compileRankProfile("my_profile_child", applicationDir.append("models")); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); - assertNull("Constant overridden by macro is not added", + assertNull("Constant overridden by function is not added", searchFromStored.search().rankingConstants().get("mnist_softmax_saved_layer_Variable_read")); } finally { @@ -317,11 +317,11 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testMacroGeneration() { + public void testFunctionGeneration() { final String name = "mnist_saved"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWith("tensor(d0[1],d1[784])(0.0)", "tensorflow('mnist/saved')", @@ -330,8 +330,8 @@ public class RankingExpressionWithTensorFlowTestCase { "input", new StoringApplicationPackage(applicationDir)); search.assertFirstPhaseExpression(expression, "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); } @Test @@ -339,7 +339,7 @@ public class RankingExpressionWithTensorFlowTestCase { final String name = "mnist_saved"; final String rankProfiles = " rank-profile my_profile {\n" + - " macro input() {\n" + + " function input() {\n" + " expression: tensor(d0[1],d1[784])(0.0)\n" + " }\n" + " first-phase {\n" + @@ -349,9 +349,9 @@ public class RankingExpressionWithTensorFlowTestCase { " rank-profile my_profile_child inherits my_profile {\n" + " }"; - final String expression = "join(join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; - final String macroExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; - final String macroExpression2 = "join(reduce(join(join(join(imported_ml_macro_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_macro_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; + final String expression = "join(join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden2_add, reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; + final String functionExpression2 = "join(reduce(join(join(join(imported_ml_function_" + name + "_dnn_hidden1_add, 0.009999999776482582, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; RankProfileSearchFixture search = fixtureWithUncompiled(rankProfiles, new StoringApplicationPackage(applicationDir)); search.compileRankProfile("my_profile", applicationDir.append("models")); @@ -359,10 +359,10 @@ public class RankingExpressionWithTensorFlowTestCase { search.assertFirstPhaseExpression(expression, "my_profile"); search.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - search.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - search.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + search.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + search.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); // At this point the expression is stored - copy application to another location which do not have a models dir Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); @@ -377,10 +377,10 @@ public class RankingExpressionWithTensorFlowTestCase { searchFromStored.assertFirstPhaseExpression(expression, "my_profile"); searchFromStored.assertFirstPhaseExpression(expression, "my_profile_child"); assertSmallConstant(name + "_dnn_hidden1_mul_x", TensorType.fromSpec("tensor()"), search); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile"); - searchFromStored.assertMacro(macroExpression1, "imported_ml_macro_" + name + "_dnn_hidden1_add", "my_profile_child"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile"); - searchFromStored.assertMacro(macroExpression2, "imported_ml_macro_" + name + "_dnn_hidden2_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile"); + searchFromStored.assertFunction(functionExpression1, "imported_ml_function_" + name + "_dnn_hidden1_add", "my_profile_child"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile"); + searchFromStored.assertFunction(functionExpression2, "imported_ml_function_" + name + "_dnn_hidden2_add", "my_profile_child"); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); @@ -429,19 +429,19 @@ public class RankingExpressionWithTensorFlowTestCase { new StoringApplicationPackage(applicationDir)); } - private RankProfileSearchFixture fixtureWith(String macroExpression, + private RankProfileSearchFixture fixtureWith(String functionExpression, String firstPhaseExpression, String constant, String field, - String macroName, + String functionName, StoringApplicationPackage application) { try { RankProfileSearchFixture fixture = new RankProfileSearchFixture( application, application.getQueryProfiles(), " rank-profile my_profile {\n" + - " macro " + macroName + "() {\n" + - " expression: " + macroExpression + + " function " + functionName + "() {\n" + + " expression: " + functionExpression + " }\n" + " first-phase {\n" + " expression: " + firstPhaseExpression + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java index 0866d3192cf..48f2bf43e81 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java @@ -95,10 +95,10 @@ public class RankingExpressionWithTensorTestCase { } @Test - public void requireThatConstantTensorsCanBeUsedInMacro() throws ParseException { + public void requireThatConstantTensorsCanBeUsedInFunction() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + - " macro my_macro() {\n" + + " function my_macro() {\n" + " expression: sum(my_tensor)\n" + " }\n" + " first-phase {\n" + @@ -112,7 +112,7 @@ public class RankingExpressionWithTensorTestCase { " }"); f.compileRankProfile("my_profile"); f.assertFirstPhaseExpression("5.0 + my_macro", "my_profile"); - f.assertMacro("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); + f.assertFunction("reduce(constant(my_tensor), sum)", "my_macro", "my_profile"); f.assertRankProperty("{{x:1}:1.0}", "constant(my_tensor).value", "my_profile"); f.assertRankProperty("tensor(x{})", "constant(my_tensor).type", "my_profile"); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 86127a260c5..fd048737b43 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -20,24 +20,24 @@ import static org.junit.Assert.assertEquals; public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test - public void testMacros() throws IOException, ParseException { + public void testFunctions() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", rankProfileRegistry, new QueryProfileRegistry()).getSearch(); - final RankProfile macrosRankProfile = rankProfileRegistry.get(search, "macros"); - macrosRankProfile.parseExpressions(); - final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros(); - assertEquals(2, macros.get("titlematch$").getFormalParams().size()); - assertEquals("var1", macros.get("titlematch$").getFormalParams().get(0)); - assertEquals("var2", macros.get("titlematch$").getFormalParams().get(1)); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getTextualExpression().trim()); - assertEquals("var1 * var2 + 890", macros.get("titlematch$").getRankingExpression().getRoot().toString()); - assertEquals("0.8+0.2*titlematch$(4,5)+0.8*titlematch$(7,8)*closeness(distance)", macrosRankProfile.getFirstPhaseRankingString().trim()); - assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); - assertEquals(0, macros.get("artistmatch").getFormalParams().size()); + RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros"); + Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions(); + assertEquals(2, functions.get("titlematch$").function().arguments().size()); + assertEquals("var1", functions.get("titlematch$").function().arguments().get(0)); + assertEquals("var2", functions.get("titlematch$").function().arguments().get(1)); + assertEquals("var1 * var2 + 890", functions.get("titlematch$").function().getBody().getRoot().toString()); + assertEquals("0.8 + 0.2 * titlematch$(4,5) + 0.8 * titlematch$(7,8) * closeness(distance)", + functionsRankProfile.getFirstPhaseRanking().getRoot().toString()); + assertEquals("78 + closeness(distance)", + functions.get("artistmatch").function().getBody().getRoot().toString()); + assertEquals(0, functions.get("artistmatch").function().arguments().size()); - List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, + List<Pair<String, String>> rankProperties = new RawRankProfile(functionsRankProfile, new QueryProfileRegistry(), new ImportedModels(), new AttributeFields(search)).configProperties(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java index 8a07e99101c..b39c48b67bf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedMacroNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -16,10 +16,10 @@ import static org.junit.Assert.assertTrue; /** * @author lesters */ -public class ReservedMacroNamesTestCase { +public class ReservedRankingExpressionFunctionNamesTestCase { @Test - public void requireThatMacrosWithReservedNamesIssueAWarning() throws ParseException { + public void requireThatFunctionsWithReservedNamesIssueAWarning() throws ParseException { TestDeployLogger deployLogger = new TestDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); @@ -32,10 +32,10 @@ public class ReservedMacroNamesTestCase { " }\n" + " \n" + " rank-profile test_rank_profile {\n" + - " macro not_a_reserved_name(x) {\n" + + " function not_a_reserved_name(x) {\n" + " expression: x + x\n" + " }\n" + - " macro sigmoid(x) {\n" + + " function sigmoid(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + @@ -43,7 +43,7 @@ public class ReservedMacroNamesTestCase { " }\n" + " }\n" + " rank-profile test_rank_profile_2 inherits test_rank_profile {\n" + - " macro sin(x) {\n" + + " function sin(x) {\n" + " expression: x * x\n" + " }\n" + " first-phase {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java index 85bee61c1b4..d0c1bf8b0ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -12,6 +12,7 @@ import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase { @@ -20,41 +21,44 @@ public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTe Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'nonexistingfield'.", e.getMessage()); } } + @Test public void requireThatInvalidImplicitSourceIsCaught() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidimplicitsummarysource.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidsummarysource', summary class 'baz', summary field 'cox': there is no valid source 'cox'.", e.getMessage()); } } + @Test public void requireThatInvalidSelfReferingSingleSource() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/invalidselfreferringsummary.sd"); search.process(); try { - new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true); - assertTrue("This should throw and never get here", false); + new SummaryFieldsMustHaveValidSource(search, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfiles()).process(true, false); + fail("This should throw and never get here"); } catch (IllegalArgumentException e) { assertEquals("For search 'invalidselfreferringsummary', summary class 'withid', summary field 'w': there is no valid source 'w'.", e.getMessage()); } } + @Test public void requireThatDocumentIdIsAllowedToPass() throws IOException, ParseException { Search search = UnprocessingSearchBuilder.buildUnprocessedFromFile("src/test/examples/documentidinsummary.sd"); search.process(); BaseDeployLogger deployLogger = new BaseDeployLogger(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true); + new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, new QueryProfiles()).process(true, false); assertEquals("documentid", search.getSummary("withid").getSummaryField("w").getSingleSource()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 76c50821cb9..8e721dbe503 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -113,7 +113,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { } @Test - public void requireThatMaxAndMinWithTensorsReturnedFromMacrosAreReplaced() throws ParseException { + public void requireThatMaxAndMinWithTensorsReturnedFromFunctionsAreReplaced() throws ParseException { assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", "max(returns_tensor,x)"); assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", @@ -171,7 +171,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:0 }\n" + " }\n" + " }\n" + - " macro base_tensor() {\n" + + " function base_tensor() {\n" + " expression: constant(base_constant_tensor)\n" + " }\n" + " }\n" + @@ -181,19 +181,19 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " value: { {x:0}:1 }\n" + " }\n" + " }\n" + - " macro returns_tensor_with_arg(arg1) {\n" + + " function returns_tensor_with_arg(arg1) {\n" + " expression: 2.0 * arg1\n" + " }\n" + - " macro wraps_returns_tensor() {\n" + + " function wraps_returns_tensor() {\n" + " expression: returns_tensor\n" + " }\n" + - " macro returns_tensor() {\n" + + " function returns_tensor() {\n" + " expression: attribute(tensor_field_2)\n" + " }\n" + - " macro tensor_inheriting() {\n" + + " function tensor_inheriting() {\n" + " expression: base_tensor\n" + " }\n" + - " macro testexpression() {\n" + + " function testexpression() {\n" + " expression: " + expression + "\n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java index ab3bb113727..d0b6524a7e1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java @@ -40,7 +40,7 @@ public class ValidateFieldTypesTest { exceptionRule.expectMessage( "For search '" + DOCUMENT_NAME + "', field '" + IMPORTED_FIELD_NAME + "': Incompatible types. " + "Expected int for summary field '" + IMPORTED_FIELD_NAME + "', got string."); - validator.process(true); + validator.process(true, false); } private static Search createSearchWithDocument(String documentName) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index b7b3fc99e20..9e26caf2cb4 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.ml; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.evaluation.RankProfilesConfigImporter; +import ai.vespa.models.handler.ModelsEvaluationHandler; import com.yahoo.component.ComponentId; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationPackage; @@ -80,6 +81,10 @@ public class ModelEvaluationTest { ContainerCluster cluster = model.getContainerClusters().get("container"); assertNotNull(cluster.getComponentsMap().get(new ComponentId(ModelsEvaluator.class.getName()))); + assertNotNull(cluster.getComponentsMap().get(new ComponentId(ModelsEvaluationHandler.class.getName()))); + assertTrue(cluster.getHandlers().stream() + .anyMatch(h -> h.getComponentId().toString().equals(ModelsEvaluationHandler.class.getName()))); + RankProfilesConfig.Builder b = new RankProfilesConfig.Builder(); cluster.getConfig(b); RankProfilesConfig config = new RankProfilesConfig(b); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java index 334228333c5..ea2ce324a27 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/TenantName.java @@ -6,8 +6,7 @@ import java.util.Objects; /** * Represents a tenant in the provision API. * - * @author lulf - * @since 5.12 + * @author Ulf Lilleengen */ public class TenantName implements Comparable<TenantName> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 6a55fb77933..6c67f730f60 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -483,10 +483,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Logs ---------------------------------------------------------------- - public HttpResponse getLogs(ApplicationId applicationId) { - String logServerHostName = getLogServerURI(applicationId); + public HttpResponse getLogs(ApplicationId applicationId, String apiParams) { + String logServerURI = getLogServerURI(applicationId) + apiParams; LogRetriever logRetriever = new LogRetriever(); - return logRetriever.getLogs(logServerHostName); + return logRetriever.getLogs(logServerURI); } // ---------------- Session operations ---------------------------------------------------------------- diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index b65cb370f93..528575f4f27 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -97,7 +97,9 @@ public class ApplicationHandler extends HttpHandler { } if (isLogRequest(request)) { - return applicationRepository.getLogs(applicationId); + String apiParams = request.getUri().getQuery(); + apiParams = apiParams == null ? "" : "?" + apiParams; + return applicationRepository.getLogs(applicationId, apiParams); } return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 120119f35bb..2ae8917e905 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -118,7 +118,7 @@ public class ApplicationRepositoryTest { } @Test - public void getLogs(){ + public void getLogs() { WireMockServer wireMock = new WireMockServer(wireMockConfig().port(8080)); wireMock.start(); WireMock.configureFor("localhost", wireMock.port()); @@ -127,15 +127,15 @@ public class ApplicationRepositoryTest { .withStatus(200))); wireMock.start(); deployApp(testAppLogServerWithContainer); - HttpResponse response = applicationRepository.getLogs(applicationId()); - assertEquals(response.getStatus(),200); + HttpResponse response = applicationRepository.getLogs(applicationId(), ""); + assertEquals(200, response.getStatus()); wireMock.stop(); } @Test(expected = IllegalArgumentException.class) public void getLogsNoContainerOnLogServerHostShouldThrowException() { deployApp(testApp); - applicationRepository.getLogs(applicationId()); + applicationRepository.getLogs(applicationId(), ""); } @Test diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 4183b642af1..4c12bacf145 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -10,11 +10,13 @@ import org.json.JSONObject; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.util.Arrays; +import java.util.HashMap; import java.util.concurrent.Executor; public class LogHandler extends ThreadedHttpRequestHandler { - private static final String LOG_DIRECTORY = "/home/y/logs/vespa/"; + private static final String LOG_DIRECTORY = "/home/y/logs/vespa/logarchive/"; @Inject public LogHandler(Executor executor) { @@ -23,10 +25,15 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - JSONObject logJson; + JSONObject responseJSON = new JSONObject(); + HashMap<String, String> apiParams = getParameters(request); + long earliestLogThreshold = getEarliestThreshold(apiParams); + long latestLogThreshold = getLatestThreshold(apiParams); + LogReader logReader= new LogReader(earliestLogThreshold, latestLogThreshold); try { - logJson = LogReader.readLogs(LOG_DIRECTORY); + JSONObject logJson = logReader.readLogs(LOG_DIRECTORY); + responseJSON.put("logs", logJson.toString()); } catch (IOException | JSONException e) { return new HttpResponse(404) { @Override @@ -37,9 +44,33 @@ public class LogHandler extends ThreadedHttpRequestHandler { @Override public void render(OutputStream outputStream) throws IOException { OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream); - outputStreamWriter.write(logJson.toString()); + outputStreamWriter.write(responseJSON.toString()); outputStreamWriter.close(); } }; } + + private HashMap<String, String> getParameters(HttpRequest request) { + String query = request.getUri().getQuery(); + HashMap<String, String> keyValPair = new HashMap<>(); + Arrays.stream(query.split("&")).forEach(pair -> { + String[] splitPair = pair.split("="); + keyValPair.put(splitPair[0], splitPair[1]); + }); + return keyValPair; + } + + private long getEarliestThreshold(HashMap<String, String> map) { + if (map.containsKey("from")) { + return Long.valueOf(map.get("from")); + } + return Long.MIN_VALUE; + } + + private long getLatestThreshold(HashMap<String, String> map) { + if (map.containsKey("to")) { + return Long.valueOf(map.get("to")); + } + return Long.MAX_VALUE; + } } diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java index eb00446dd0e..2483f2497d0 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogReader.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogReader.java @@ -10,23 +10,34 @@ import java.nio.file.Files; public class LogReader { - protected static JSONObject readLogs(String logDirectory) throws IOException, JSONException { + long earliestLogThreshold; + long latestLogThreshold; + + public LogReader(long earliestLogThreshold, long latestLogThreshold) { + this.earliestLogThreshold = earliestLogThreshold; + this.latestLogThreshold = latestLogThreshold; + } + + protected JSONObject readLogs(String logDirectory) throws IOException, JSONException { JSONObject json = new JSONObject(); File root = new File(logDirectory); - traverse_folder(root, json); + traverse_folder(root, json, ""); return json; } - private static void traverse_folder(File root, JSONObject json) throws IOException, JSONException { - for(File child : root.listFiles()) { + private void traverse_folder(File root, JSONObject json, String filename) throws IOException, JSONException { + File[] files = root.listFiles(); + for(File child : files) { + File temp = child; JSONObject childJson = new JSONObject(); - if(child.isFile()) { - json.put(child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); + long logTime = child.lastModified(); + if(child.isFile() && earliestLogThreshold < logTime && logTime < latestLogThreshold) { + json.put(filename + child.getName(), DatatypeConverter.printBase64Binary(Files.readAllBytes(child.toPath()))); } - else { - json.put(child.getName(), childJson); - traverse_folder(child, childJson); + else if (!child.isFile()){ + traverse_folder(child, json, filename + child.getName() + "-"); } } } + } diff --git a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java index e5302ee43ee..534026f89ac 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/LogReaderTest.java @@ -20,8 +20,19 @@ public class LogReaderTest { @Test public void testThatFilesAreWrittenCorrectlyToOutputStream() throws Exception{ String logDirectory = "src/test/resources/logfolder/"; - JSONObject json = LogReader.readLogs(logDirectory); - String expected = "{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; + LogReader logReader = new LogReader(21, Long.MAX_VALUE); + JSONObject json = logReader.readLogs(logDirectory); + String expected = "{\"subfolder-log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\",\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}"; + String actual = json.toString(); + assertEquals(expected, actual); + } + + @Test + public void testThatLogsOutsideRangeAreExcluded() throws Exception { + String logDirectory = "src/test/resources/logfolder/"; + LogReader logReader = new LogReader(Long.MAX_VALUE, Long.MIN_VALUE); + JSONObject json = logReader.readLogs(logDirectory); + String expected = "{}"; String actual = json.toString(); assertEquals(expected, actual); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index eb10c78f891..5dacaf9b0db 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -9,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -42,7 +42,7 @@ public interface ConfigServer { Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath); - HttpResponse getLogs(DeploymentId deployment); + Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters); /** * Set new status on en endpoint in one zone. * diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java new file mode 100644 index 00000000000..223d3e88f13 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Logs.java @@ -0,0 +1,16 @@ +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import java.util.Map; + +public class Logs { + + private final Map<String, String> logs; + + public Logs(Map<String, String> logs) { + this.logs = logs; + } + + public Map<String, String> logs() { + return this.logs; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 034db3d487d..154c4e632de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; @@ -87,7 +88,9 @@ import java.net.URISyntaxException; import java.security.Principal; import java.time.DayOfWeek; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -168,7 +171,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.getUri().getQuery()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); @@ -346,13 +349,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region) { + private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, String query) { ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); ZoneId zone = ZoneId.from(environment, region); DeploymentId deployment = new DeploymentId(application, zone); - return controller.configServer().getLogs(deployment); + HashMap<String, String> queryParameters = getParameters(query); + Optional<Logs> response = controller.configServer().getLogs(deployment, queryParameters); + Slime slime = new Slime(); + Cursor object = slime.setObject(); + if (response.isPresent()) { + response.get().logs().entrySet().stream().forEach(entry -> object.setString(entry.getKey(), entry.getValue())); + } + return new SlimeJsonResponse(slime); } + private HashMap<String, String> getParameters(String query) { + HashMap<String, String> keyValPair = new HashMap<>(); + Arrays.stream(query.split("&")).forEach(pair -> { + String[] splitPair = pair.split("="); + keyValPair.put(splitPair[0], splitPair[1]); + }); + return keyValPair; + } private void toSlime(Cursor object, Application application, HttpRequest request) { object.setString("application", application.id().application().value()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index bd65465633e..aea809de365 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; @@ -298,14 +299,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public HttpResponse getLogs(DeploymentId deployment) { - return new HttpResponse(200) { - @Override - public void render(OutputStream outputStream) throws IOException { - outputStream.write("{\"subfolder\":{\"log2.log\":\"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl\"},\"log1.log\":\"VGhpcyBpcyBvbmUgbG9nIGZpbGU=\"}".getBytes()); - } - }; - + public Optional<Logs> getLogs(DeploymentId deployment, HashMap<String, String> queryParameters) { + HashMap<String, String> logs = new HashMap<>(); + logs.put("subfolder-log2.log", "VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl"); + logs.put("log1.log", "VGhpcyBpcyBvbmUgbG9nIGZpbGU="); + return Optional.of(new Logs(logs)); } public static class Application { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 0ea23ae1b78..30c81a0721a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -331,7 +331,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("application1-recursive.json")); // GET logs - tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs", GET).userIdentity(USER_ID), new File("logs.json")); + tester.assertResponse(request("/application/v4/tenant/tenant2/application//application1/environment/prod/region/corp-us-east-1/instance/default/logs?from=1233&to=3214", GET).userIdentity(USER_ID), new File("logs.json")); // DELETE (cancel) ongoing change tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json index 398a62758ee..69fc0f88ea6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/logs.json @@ -1,5 +1,4 @@ { - "subfolder": { - "log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl"}, + "subfolder-log2.log":"VGhpcyBpcyBhbm90aGVyIGxvZyBmaWxl", "log1.log":"VGhpcyBpcyBvbmUgbG9nIGZpbGU=" }
\ No newline at end of file diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java index 517f44cb983..a0516a62bd9 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java @@ -10,7 +10,6 @@ import com.yahoo.document.Field; import com.yahoo.document.serialization.DocumentUpdateWriter; import com.yahoo.document.update.FieldUpdate; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -19,7 +18,6 @@ import java.util.Map; * * @author vegardh */ -// TODO Vespa 7 Remove all deprecated methods public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOperationWrapper { private DocumentUpdate docU; @@ -43,12 +41,10 @@ public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOpera @Override public FieldUpdate getFieldUpdate(Field field) { - return docU.getFieldUpdate(field); + return getFieldUpdate(field.getName()); } @Override - @Deprecated - @SuppressWarnings( "deprecation" ) public FieldUpdate getFieldUpdate(int index) { return docU.getFieldUpdate(index); } @@ -64,15 +60,10 @@ public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOpera } @Override - @Deprecated - @SuppressWarnings( "deprecation" ) public List<FieldUpdate> getFieldUpdates() { return docU.getFieldUpdates(); } - @Override - public Collection<FieldUpdate> fieldUpdates() { - return docU.fieldUpdates(); - } + @Override public DocumentId getId() { return docU.getId(); diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java index a89dbfcc782..9a3a29e55b1 100644 --- a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java @@ -37,10 +37,8 @@ public class ProcessingUpdateTestCase { @Test public void testProcessingUpdates() { DocumentType articleType = new DocumentType("article"); - Field bodyField = new Field("body", DataType.STRING, true); - Field titleField = new Field("title", DataType.STRING, true); - articleType.addField(bodyField); - articleType.addField(titleField); + articleType.addField(new Field("body", DataType.STRING, true)); + articleType.addField(new Field("title", DataType.STRING, true)); dtm = new DocumentTypeManager(); dtm.registerDocumentType(articleType); @@ -71,12 +69,12 @@ public class ProcessingUpdateTestCase { assertEquals(new StringFieldValue("body blah blah blah "), first.getFieldValue("title")); DocumentUpdate second = (DocumentUpdate) operations.get(1); - FieldUpdate firstUpd = second.getFieldUpdate(bodyField); + FieldUpdate firstUpd = second.getFieldUpdate(0); assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, firstUpd.getValueUpdate(0).getValueUpdateClassID()); assertEquals(new StringFieldValue("this is the updated body of the article, blahdi blahdi blahdi"), firstUpd.getValueUpdate(0) .getValue()); - FieldUpdate secondUpd = second.getFieldUpdate(titleField); + FieldUpdate secondUpd = second.getFieldUpdate(1); assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, secondUpd.getValueUpdate(0).getValueUpdateClassID()); assertEquals(new StringFieldValue("body blahdi blahdi blahdi "), secondUpd.getValueUpdate(0).getValue()); } diff --git a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java index e6de3190156..05a03480173 100644 --- a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java +++ b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java @@ -328,17 +328,16 @@ public class SchemaMappingAndAccessesTest { Document doc = getDoc(); DocumentType type = doc.getDataType(); DocumentUpdate dud = new DocumentUpdate(type, new DocumentId("doc:map:test:1")); - com.yahoo.document.Field title = type.getField("title"); - FieldUpdate assignSingle = FieldUpdate.createAssign(title, new StringFieldValue("something")); + FieldUpdate assignSingle = FieldUpdate.createAssign(type.getField("title"), new StringFieldValue("something")); Map<String, String> fieldMap = new HashMap<>(); fieldMap.put("t", "title"); fieldMap.put("a", "artist"); ProxyDocumentUpdate pup = new ProxyDocumentUpdate(dud, fieldMap); pup.addFieldUpdate(assignSingle); - assertEquals(pup.fieldUpdates().toString(), dud.fieldUpdates().toString()); + assertEquals(pup.getFieldUpdates(), dud.getFieldUpdates()); assertEquals(pup.getDocumentType(), dud.getDocumentType()); - assertEquals(pup.getFieldUpdate(title).size(), 1); - assertEquals(pup.getFieldUpdate(title), dud.fieldUpdates().iterator().next()); + assertEquals(pup.getFieldUpdate(new com.yahoo.document.Field("title")).size(), 1); + assertEquals(pup.getFieldUpdate(0), dud.getFieldUpdate(0)); assertEquals(pup.getFieldUpdate("title"), dud.getFieldUpdate("title")); assertEquals(pup.getId(), dud.getId()); assertEquals(pup.getType(), dud.getType()); diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java index 4905f3d9dad..f25603deee9 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java @@ -9,6 +9,7 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.MapFieldValue; import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.StructuredFieldValue; import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; @@ -19,11 +20,7 @@ import com.yahoo.document.update.ValueUpdate; import com.yahoo.vespa.indexinglanguage.AdapterFactory; import com.yahoo.vespa.indexinglanguage.expressions.Expression; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * @author Simon Thoresen Hult @@ -51,13 +48,13 @@ public class DocumentScript { } public DocumentUpdate execute(AdapterFactory adapterFactory, DocumentUpdate update) { - for (FieldUpdate fieldUpdate : update.fieldUpdates()) { + for (FieldUpdate fieldUpdate : update.getFieldUpdates()) { requireThatFieldIsDeclaredInDocument(fieldUpdate.getField()); for (ValueUpdate<?> valueUpdate : fieldUpdate.getValueUpdates()) { removeAnyLinguisticsSpanTree(valueUpdate); } } - for (FieldPathUpdate fieldUpdate : update.fieldPathUpdates()) { + for (FieldPathUpdate fieldUpdate : update.getFieldPathUpdates()) { requireThatFieldIsDeclaredInDocument(fieldUpdate.getFieldPath().get(0).getFieldRef()); if (fieldUpdate instanceof AssignFieldPathUpdate) { removeAnyLinguisticsSpanTree(((AssignFieldPathUpdate)fieldUpdate).getFieldValue()); diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java index cfaee10a07d..a47762bfbf3 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/DocumentScriptTestCase.java @@ -187,8 +187,8 @@ public class DocumentScriptTestCase { FieldPathUpdate executeWithUpdateAndExpectFieldPath(String fieldName, FieldPathUpdate updateIn) { DocumentUpdate update = executeWithUpdate(fieldName, updateIn); - assertEquals(1, update.fieldPathUpdates().size()); - return update.fieldPathUpdates().iterator().next(); + assertEquals(1, update.getFieldPathUpdates().size()); + return update.getFieldPathUpdates().get(0); } } @@ -229,10 +229,10 @@ public class DocumentScriptTestCase { StringFieldValue newTitleValue = new StringFieldValue("iron moose 4, moose with a vengeance"); DocumentUpdate update = f.executeWithUpdate("structfield", new AssignFieldPathUpdate(f.type, "structfield.title", newTitleValue)); - assertEquals(1, update.fieldPathUpdates().size()); - assertEquals(0, update.fieldUpdates().size()); - assertTrue(update.fieldPathUpdates().iterator().next() instanceof AssignFieldPathUpdate); - AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)update.fieldPathUpdates().iterator().next(); + assertEquals(1, update.getFieldPathUpdates().size()); + assertEquals(0, update.getFieldUpdates().size()); + assertTrue(update.getFieldPathUpdates().get(0) instanceof AssignFieldPathUpdate); + AssignFieldPathUpdate assignUpdate = (AssignFieldPathUpdate)update.getFieldPathUpdates().get(0); assertEquals("structfield.title", assignUpdate.getOriginalFieldPath()); assertEquals(newTitleValue, assignUpdate.getFieldValue()); } diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java index 5979672524d..cef020cd828 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.update.AssignValueUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.ValueUpdate; +import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import org.junit.Test; @@ -62,15 +63,15 @@ public class IndexingProcessorTestCase { assertTrue(output instanceof DocumentUpdate); DocumentUpdate docUpdate = (DocumentUpdate) output; - assertEquals(3, docUpdate.fieldUpdates().size()); + assertEquals(3, docUpdate.getFieldUpdates().size()); { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("song"); + FieldUpdate fieldUpdate = docUpdate.getFieldUpdate(0); assertEquals("song", fieldUpdate.getField().getName()); assertEquals(1, fieldUpdate.getValueUpdates().size()); ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); assertTrue(valueUpdate instanceof AssignValueUpdate); assertEquals(new StringFieldValue("isbnmarker"), valueUpdate.getValue()); - fieldUpdate = docUpdate.getFieldUpdate("title"); + fieldUpdate = docUpdate.getFieldUpdate(1); assertEquals("title", fieldUpdate.getField().getName()); assertEquals(1, fieldUpdate.getValueUpdates().size()); valueUpdate = fieldUpdate.getValueUpdate(0); @@ -79,14 +80,14 @@ public class IndexingProcessorTestCase { } { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("title"); + FieldUpdate fieldUpdate = docUpdate.getFieldUpdate(1); ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); assertEquals("title", fieldUpdate.getField().getName()); assertTrue(valueUpdate instanceof AssignValueUpdate); assertEquals(new StringFieldValue("69"), valueUpdate.getValue()); } { - FieldUpdate fieldUpdate = docUpdate.getFieldUpdate("isbn"); + FieldUpdate fieldUpdate = docUpdate.getFieldUpdate(2); ValueUpdate<?> valueUpdate = fieldUpdate.getValueUpdate(0); assertEquals("isbn", fieldUpdate.getField().getName()); assertTrue(valueUpdate instanceof AssignValueUpdate); diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index 028215c0d6c..ad93942c1c0 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -13,12 +13,9 @@ import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; /** @@ -40,17 +37,14 @@ import java.util.Optional; * @see com.yahoo.document.update.FieldUpdate * @see com.yahoo.document.update.ValueUpdate */ -//TODO Vespa 7 Remove all deprecated methods - public class DocumentUpdate extends DocumentOperation implements Iterable<FieldPathUpdate> { //see src/vespa/document/util/identifiableid.h public static final int CLASSID = 0x1000 + 6; private DocumentId docId; - private final List<FieldUpdate> fieldUpdates; - private final Map<Integer, FieldUpdate> id2FieldUpdateMap; - private final List<FieldPathUpdate> fieldPathUpdates; + private List<FieldUpdate> fieldUpdates; + private List<FieldPathUpdate> fieldPathUpdates; private DocumentType documentType; private Optional<Boolean> createIfNonExistent = Optional.empty(); @@ -61,7 +55,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param docType the document type that this update is valid for */ public DocumentUpdate(DocumentType docType, DocumentId docId) { - this(docType, docId, new HashMap<>()); + this(docType, docId, new ArrayList<FieldUpdate>()); } /** @@ -71,7 +65,6 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP docId = null; documentType = null; fieldUpdates = new ArrayList<>(); - id2FieldUpdateMap = new HashMap<>(); fieldPathUpdates = new ArrayList<>(); reader.read(this); } @@ -86,11 +79,10 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP this(docType, new DocumentId(docId)); } - private DocumentUpdate(DocumentType docType, DocumentId docId, Map<Integer, FieldUpdate> id2fieldUpdateMap) { + private DocumentUpdate(DocumentType docType, DocumentId docId, List<FieldUpdate> fieldUpdates) { this.docId = docId; this.documentType = docType; - this.fieldUpdates = new ArrayList<>(id2fieldUpdateMap.values()); - id2FieldUpdateMap = id2fieldUpdateMap; + this.fieldUpdates = fieldUpdates; this.fieldPathUpdates = new ArrayList<>(); } @@ -122,7 +114,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate applyTo(Document doc) { verifyType(doc); - for (FieldUpdate fieldUpdate : id2FieldUpdateMap.values()) { + for (FieldUpdate fieldUpdate : fieldUpdates) { fieldUpdate.applyTo(doc); } for (FieldPathUpdate fieldPathUpdate : fieldPathUpdates) { @@ -140,9 +132,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate prune(Document doc) { verifyType(doc); - for (Iterator<Map.Entry<Integer, FieldUpdate>> iter = id2FieldUpdateMap.entrySet().iterator(); iter.hasNext();) { - Map.Entry<Integer, FieldUpdate> entry = iter.next(); - FieldUpdate update = entry.getValue(); + for (Iterator<FieldUpdate> iter = fieldUpdates.iterator(); iter.hasNext();) { + FieldUpdate update = iter.next(); if (!update.isEmpty()) { ValueUpdate last = update.getValueUpdate(update.size() - 1); if (last instanceof AssignValueUpdate) { @@ -171,42 +162,20 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * Get an unmodifiable list of all field updates that this document update specifies. * * @return a list of all FieldUpdates in this DocumentUpdate - * @deprecated Use fieldUpdates() instead. */ - @Deprecated public List<FieldUpdate> getFieldUpdates() { return Collections.unmodifiableList(fieldUpdates); } /** - * Get an unmodifiable collection of all field updates that this document update specifies. - * - * @return a collection of all FieldUpdates in this DocumentUpdate - */ - public Collection<FieldUpdate> fieldUpdates() { - return Collections.unmodifiableCollection(id2FieldUpdateMap.values()); - } - - /** * Get an unmodifiable list of all field path updates this document update specifies. * * @return Returns a list of all field path updates in this document update. - * @deprecated Use fieldPathUpdates() instead. */ - @Deprecated public List<FieldPathUpdate> getFieldPathUpdates() { return Collections.unmodifiableList(fieldPathUpdates); } - /** - * Get an unmodifiable collection of all field path updates that this document update specifies. - * - * @return a collection of all FieldPathUpdates in this DocumentUpdate - */ - public Collection<FieldPathUpdate> fieldPathUpdates() { - return Collections.unmodifiableCollection(fieldPathUpdates); - } - /** Returns the type of the document this updates * * @return The documentype of the document @@ -229,9 +198,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param index the index of the FieldUpdate to return * @return the FieldUpdate at the specified index * @throws IndexOutOfBoundsException if index is out of range - * @deprecated use getFieldUpdate(Field field) instead. */ - @Deprecated public FieldUpdate getFieldUpdate(int index) { return fieldUpdates.get(index); } @@ -243,16 +210,9 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param upd the FieldUpdate to be stored at the specified position * @return the FieldUpdate previously at the specified position * @throws IndexOutOfBoundsException if index is out of range - * @deprecated Use removeFieldUpdate/addFieldUpdate instead */ - @Deprecated public FieldUpdate setFieldUpdate(int index, FieldUpdate upd) { - FieldUpdate old = fieldUpdates.get(index); - fieldUpdates.set(index, upd); - id2FieldUpdateMap.remove(old.getField().getId()); - id2FieldUpdateMap.put(upd.getField().getId(), upd); - - return old; + return fieldUpdates.set(index, upd); } /** @@ -262,13 +222,12 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return the update for the field, or null if that field has no update in this */ public FieldUpdate getFieldUpdate(Field field) { - return getFieldUpdateById(field.getId()); + return getFieldUpdate(field.getName()); } /** Removes all field updates from the list for field updates. */ public void clearFieldUpdates() { fieldUpdates.clear(); - id2FieldUpdateMap.clear(); } /** @@ -278,34 +237,27 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return the update for the field, or null if that field has no update in this */ public FieldUpdate getFieldUpdate(String fieldName) { - Field field = documentType.getField(fieldName); - return field != null ? getFieldUpdate(field) : null; - } - private FieldUpdate getFieldUpdateById(Integer fieldId) { - return id2FieldUpdateMap.get(fieldId); + for (FieldUpdate fieldUpdate : fieldUpdates) { + if (fieldUpdate.getField().getName().equals(fieldName)) { + return fieldUpdate; + } + } + return null; } /** * Assigns the field updates of this document update. * This document update receives ownership of the list - it can not be subsequently used - * by the caller. Also note that there no assumptions can be made on the order of items - * after this call. They might have been joined if for the same field or reordered. + * by the caller. The list may not be unmodifiable. * * @param fieldUpdates the new list of updates of this * @throws NullPointerException if the argument passed is null */ - public void setFieldUpdates(Collection<FieldUpdate> fieldUpdates) { + public void setFieldUpdates(List<FieldUpdate> fieldUpdates) { if (fieldUpdates == null) { throw new NullPointerException("The field updates of a document update can not be null"); } - clearFieldUpdates(); - addFieldUpdates(fieldUpdates); - } - - public void addFieldUpdates(Collection<FieldUpdate> fieldUpdates) { - for (FieldUpdate fieldUpdate : fieldUpdates) { - addFieldUpdate(fieldUpdate); - } + this.fieldUpdates = fieldUpdates; } /** @@ -314,7 +266,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return the size of the List of FieldUpdates */ public int size() { - return id2FieldUpdateMap.size(); + return fieldUpdates.size(); } /** @@ -327,17 +279,17 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * field. */ public DocumentUpdate addFieldUpdate(FieldUpdate update) { - Integer fieldId = update.getField().getId(); - if (documentType.getField(fieldId) == null) { - throw new IllegalArgumentException("Document type '" + documentType.getName() + "' does not have field '" + update.getField().getName() + "'."); + String fieldName = update.getField().getName(); + if (!documentType.hasField(fieldName)) { + throw new IllegalArgumentException("Document type '" + documentType.getName() + "' does not have field '" + + fieldName + "'."); } - FieldUpdate prevUpdate = getFieldUpdateById(fieldId); + FieldUpdate prevUpdate = getFieldUpdate(fieldName); if (prevUpdate != update) { if (prevUpdate != null) { prevUpdate.addAll(update); } else { fieldUpdates.add(update); - id2FieldUpdateMap.put(fieldId, update); } } return this; @@ -353,6 +305,12 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP return this; } + // TODO: Remove this when we figure out correct behaviour. + + public void addFieldUpdateNoCheck(FieldUpdate fieldUpdate) { + fieldUpdates.add(fieldUpdate); + } + /** * Adds all the field- and field path updates of the given document update to this. If the given update refers to a * different document or document type than this, this method throws an exception. @@ -371,7 +329,9 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP if (!documentType.equals(update.documentType)) { throw new IllegalArgumentException("Expected " + documentType + ", got " + update.documentType + "."); } - addFieldUpdates(update.fieldUpdates()); + for (FieldUpdate fieldUpd : update.fieldUpdates) { + addFieldUpdate(fieldUpd); + } for (FieldPathUpdate pathUpd : update.fieldPathUpdates) { addFieldPathUpdate(pathUpd); } @@ -383,22 +343,9 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @param index the index of the FieldUpdate to remove * @return the FieldUpdate previously at the specified position * @throws IndexOutOfBoundsException if index is out of range - * @deprecated use removeFieldUpdate(Field field) instead. */ - @Deprecated public FieldUpdate removeFieldUpdate(int index) { - FieldUpdate prev = getFieldUpdate(index); - fieldUpdates.remove(index); - return removeFieldUpdate(prev.getField()); - } - - public FieldUpdate removeFieldUpdate(Field field) { - return id2FieldUpdateMap.remove(field.getId()); - } - - public FieldUpdate removeFieldUpdate(String fieldName) { - Field field = documentType.getField(fieldName); - return field != null ? removeFieldUpdate(field) : null; + return fieldUpdates.remove(index); } /** @@ -429,7 +376,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP if (documentType != null ? !documentType.equals(that.documentType) : that.documentType != null) return false; if (fieldPathUpdates != null ? !fieldPathUpdates.equals(that.fieldPathUpdates) : that.fieldPathUpdates != null) return false; - if (id2FieldUpdateMap != null ? !id2FieldUpdateMap.equals(that.id2FieldUpdateMap) : that.id2FieldUpdateMap != null) return false; + if (fieldUpdates != null ? !fieldUpdates.equals(that.fieldUpdates) : that.fieldUpdates != null) return false; if (this.getCreateIfNonExistent() != ((DocumentUpdate) o).getCreateIfNonExistent()) return false; return true; @@ -438,7 +385,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP @Override public int hashCode() { int result = docId != null ? docId.hashCode() : 0; - result = 31 * result + (id2FieldUpdateMap != null ? id2FieldUpdateMap.hashCode() : 0); + result = 31 * result + (fieldUpdates != null ? fieldUpdates.hashCode() : 0); result = 31 * result + (fieldPathUpdates != null ? fieldPathUpdates.hashCode() : 0); result = 31 * result + (documentType != null ? documentType.hashCode() : 0); return result; @@ -455,7 +402,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP string.append(": "); string.append("["); - for (Iterator<FieldUpdate> i = id2FieldUpdateMap.values().iterator(); i.hasNext();) { + for (Iterator<FieldUpdate> i = fieldUpdates.iterator(); i.hasNext();) { FieldUpdate fieldUpdate = i.next(); string.append(fieldUpdate); if (i.hasNext()) { @@ -485,7 +432,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return True if this update is empty. */ public boolean isEmpty() { - return id2FieldUpdateMap.isEmpty() && fieldPathUpdates.isEmpty(); + return fieldUpdates.isEmpty() && fieldPathUpdates.isEmpty(); } /** diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 6adae27cadc..b9b273d691f 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -47,7 +47,6 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.regex.Matcher; @@ -106,11 +105,11 @@ public class DocumentUpdateJsonSerializer } generator.writeObjectFieldStart("fields"); - for (FieldUpdate up : update.fieldUpdates()) { + for (FieldUpdate up : update.getFieldUpdates()) { up.serialize(this); } - update.fieldPathUpdates().stream() + update.getFieldPathUpdates().stream() .collect(Collectors.groupingBy(FieldPathUpdate::getFieldPath)) .forEach((fieldPath, fieldPathUpdates) -> wrapIOException(() -> write(fieldPath, fieldPathUpdates, generator))); @@ -121,7 +120,7 @@ public class DocumentUpdateJsonSerializer }); } - private void write(FieldPath fieldPath, Collection<FieldPathUpdate> fieldPathUpdates, JsonGenerator generator) throws IOException { + private void write(FieldPath fieldPath, List<FieldPathUpdate> fieldPathUpdates, JsonGenerator generator) throws IOException { generator.writeObjectFieldStart(fieldPath.toString()); for (FieldPathUpdate update : fieldPathUpdates) { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index 3f885844987..a048ea349eb 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -472,7 +472,7 @@ public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 imple for (int i = 0; i < size; i++) { // TODO: Should use checked method, but doesn't work according to test now. - update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), serializationVersion)); + update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), serializationVersion)); } } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java index 4f8a26d3777..7bdc6fd5355 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -29,7 +29,7 @@ public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 { for (int i = 0; i < size; i++) { // TODO: Should use checked method, but doesn't work according to test now. - update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), 8)); + update.addFieldUpdateNoCheck(new FieldUpdate(this, update.getDocumentType(), 8)); } int sizeAndFlags = getInt(null); diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index a9f77cb5eb0..15319985591 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -1,19 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.document.datatypes.Array; -import com.yahoo.document.datatypes.FloatFieldValue; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.datatypes.*; import com.yahoo.document.fieldpathupdate.FieldPathUpdate; -import com.yahoo.document.serialization.DocumentDeserializer; -import com.yahoo.document.serialization.DocumentDeserializerFactory; -import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.document.serialization.DocumentSerializerFactory; -import com.yahoo.document.serialization.DocumentUpdateFlags; -import com.yahoo.document.serialization.DocumentUpdateWriter; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.serialization.*; import com.yahoo.document.update.AssignValueUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.ValueUpdate; @@ -268,8 +259,8 @@ public class DocumentUpdateTestCase { update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(1))); update.addFieldUpdate(FieldUpdate.createAssign(field, new IntegerFieldValue(2))); - assertEquals(1, update.fieldUpdates().size()); - FieldUpdate fieldUpdate = update.getFieldUpdate(field); + assertEquals(1, update.getFieldUpdates().size()); + FieldUpdate fieldUpdate = update.getFieldUpdate(0); assertNotNull(fieldUpdate); assertEquals(field, fieldUpdate.getField()); assertEquals(2, fieldUpdate.getValueUpdates().size()); @@ -351,16 +342,13 @@ public class DocumentUpdateTestCase { assertEquals(new DocumentId("doc:update:test"), upd.getId()); assertEquals(type, upd.getType()); - FieldUpdate serAssignFU = upd.getFieldUpdate(type.getField("intfield")); + FieldUpdate serAssignFU = upd.getFieldUpdate(0); assertEquals(type.getField("intfield"), serAssignFU.getField()); ValueUpdate serAssign = serAssignFU.getValueUpdate(0); assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, serAssign.getValueUpdateClassID()); assertEquals(new IntegerFieldValue(4), serAssign.getValue()); - ValueUpdate serArith = serAssignFU.getValueUpdate(1); - assertEquals(ValueUpdate.ValueUpdateClassID.ARITHMETIC, serArith.getValueUpdateClassID()); - - FieldUpdate serAddFU = upd.getFieldUpdate(type.getField("arrayoffloatfield")); + FieldUpdate serAddFU = upd.getFieldUpdate(2); assertEquals(type.getField("arrayoffloatfield"), serAddFU.getField()); ValueUpdate serAdd1 = serAddFU.getValueUpdate(0); assertEquals(ValueUpdate.ValueUpdateClassID.ADD, serAdd1.getValueUpdateClassID()); @@ -375,7 +363,12 @@ public class DocumentUpdateTestCase { FloatFieldValue addparam3 = (FloatFieldValue)serAdd3.getValue(); assertEquals(new FloatFieldValue(-1.00f), addparam3); - FieldUpdate wsetFU = upd.getFieldUpdate(type.getField("wsfield")); + FieldUpdate arithFU = upd.getFieldUpdate(3); + assertEquals(type.getField("intfield"), serAssignFU.getField()); + ValueUpdate serArith = arithFU.getValueUpdate(0); + assertEquals(ValueUpdate.ValueUpdateClassID.ARITHMETIC, serArith.getValueUpdateClassID()); + + FieldUpdate wsetFU = upd.getFieldUpdate(4); assertEquals(type.getField("wsfield"), wsetFU.getField()); assertEquals(2, wsetFU.size()); ValueUpdate mapUpd = wsetFU.getValueUpdate(0); @@ -427,8 +420,8 @@ public class DocumentUpdateTestCase { barUpdate.addFieldUpdate(barField); fooUpdate.addAll(barUpdate); - assertEquals(1, fooUpdate.fieldUpdates().size()); - FieldUpdate fieldUpdate = fooUpdate.getFieldUpdate(field); + assertEquals(1, fooUpdate.getFieldUpdates().size()); + FieldUpdate fieldUpdate = fooUpdate.getFieldUpdate(0); assertNotNull(fieldUpdate); assertEquals(field, fieldUpdate.getField()); assertEquals(2, fieldUpdate.getValueUpdates().size()); @@ -442,32 +435,6 @@ public class DocumentUpdateTestCase { } @Test - public void testGetAndRemoveByName() { - DocumentType docType = new DocumentType("my_type"); - Field my_int = new Field("my_int", DataType.INT); - Field your_int = new Field("your_int", DataType.INT); - docType.addField(my_int); - docType.addField(your_int); - DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:this:is:a:test")); - - update.addFieldUpdate(FieldUpdate.createAssign(my_int, new IntegerFieldValue(2))); - assertNull(update.getFieldUpdate("none-existing-field")); - assertNull(update.removeFieldUpdate("none-existing-field")); - assertNull(update.getFieldUpdate("your_int")); - assertEquals(new IntegerFieldValue(2), update.getFieldUpdate("my_int").getValueUpdate(0).getValue()); - assertNull(update.removeFieldUpdate("your_int")); - assertEquals(new IntegerFieldValue(2), update.removeFieldUpdate("my_int").getValueUpdate(0).getValue()); - assertNull(update.getFieldUpdate("my_int")); - - update.addFieldUpdate(FieldUpdate.createAssign(my_int, new IntegerFieldValue(2))); - assertNull(update.getFieldUpdate(your_int)); - assertEquals(new IntegerFieldValue(2), update.getFieldUpdate(my_int).getValueUpdate(0).getValue()); - assertNull(update.removeFieldUpdate(your_int)); - assertEquals(new IntegerFieldValue(2), update.removeFieldUpdate(my_int).getValueUpdate(0).getValue()); - assertNull(update.getFieldUpdate(my_int)); - } - - @Test public void testInstantiationAndEqualsHashCode() { DocumentType type = new DocumentType("doo"); DocumentUpdate d1 = new DocumentUpdate(type, new DocumentId("doc:this:is:a:test")); @@ -498,7 +465,6 @@ public class DocumentUpdateTestCase { } @Test - @SuppressWarnings("deprecation") public void testFieldUpdatesInDocUp() { DocumentType t1 = new DocumentType("doo"); Field f1 = new Field("field1", DataType.STRING); @@ -527,6 +493,14 @@ public class DocumentUpdateTestCase { assertSame(fu1, documentUpdate.getFieldUpdate(f1)); + assertSame(fu1, documentUpdate.getFieldUpdate(0)); + assertSame(fu2, documentUpdate.getFieldUpdate(1)); + + documentUpdate.setFieldUpdate(0, fu2); + documentUpdate.setFieldUpdate(1, fu1); + assertEquals(2, documentUpdate.size()); + assertSame(fu1, documentUpdate.getFieldUpdate(1)); + assertSame(fu2, documentUpdate.getFieldUpdate(0)); try { documentUpdate.setFieldUpdates(null); @@ -541,12 +515,12 @@ public class DocumentUpdateTestCase { documentUpdate.setFieldUpdates(fus); assertEquals(2, documentUpdate.size()); - assertSame(fu1, documentUpdate.getFieldUpdate(fu1.getField())); - assertSame(fu2, documentUpdate.getFieldUpdate(fu2.getField())); + assertSame(fu1, documentUpdate.getFieldUpdate(0)); + assertSame(fu2, documentUpdate.getFieldUpdate(1)); - documentUpdate.removeFieldUpdate(fu2.getField()); + documentUpdate.removeFieldUpdate(1); assertEquals(1, documentUpdate.size()); - assertSame(fu1, documentUpdate.getFieldUpdate(fu1.getField())); + assertSame(fu1, documentUpdate.getFieldUpdate(0)); documentUpdate.toString(); diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 32f63e6c0b3..1fd45cb07c4 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -292,8 +292,8 @@ public class JsonReaderTestCase { + "\"skuggsjaa\": {" + "\"assign\": { \"sandra\": \"person\"," + " \"cloud\": \"another person\"}}}}"); - assertEquals(1, put.fieldUpdates().size()); - FieldUpdate fu = put.fieldUpdates().iterator().next(); + assertEquals(1, put.getFieldUpdates().size()); + FieldUpdate fu = put.getFieldUpdate(0); assertEquals(1, fu.getValueUpdates().size()); ValueUpdate vu = fu.getValueUpdate(0); assertTrue(vu instanceof AssignValueUpdate); @@ -315,8 +315,8 @@ public class JsonReaderTestCase { + " \"fields\": { " + "\"skuggsjaa\": {" + "\"assign\": { }}}}"); - assertEquals(1, put.fieldUpdates().size()); - FieldUpdate fu = put.fieldUpdates().iterator().next(); + assertEquals(1, put.getFieldUpdates().size()); + FieldUpdate fu = put.getFieldUpdate(0); assertEquals(1, fu.getValueUpdates().size()); ValueUpdate vu = fu.getValueUpdate(0); assertTrue(vu instanceof AssignValueUpdate); diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java index dcdea0975ad..ea954f0da40 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java @@ -46,8 +46,8 @@ public class UriParserTestCase { DocumentUpdate upd = nextUpdate(it); assertNotNull(upd); - assertEquals(1, upd.fieldUpdates().size()); - FieldUpdate fieldUpd = upd.fieldUpdates().iterator().next(); + assertEquals(1, upd.getFieldUpdates().size()); + FieldUpdate fieldUpd = upd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_arr"), fieldUpd.getField()); assertEquals(1, fieldUpd.getValueUpdates().size()); diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java index 1aad59f4c56..4c64f7c35cb 100755 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java @@ -5,7 +5,6 @@ import com.yahoo.document.*; import com.yahoo.document.datatypes.*; import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; -import com.yahoo.document.fieldpathupdate.FieldPathUpdate; import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; import com.yahoo.document.serialization.DeserializationException; import com.yahoo.document.update.AddValueUpdate; @@ -17,7 +16,6 @@ import org.junit.Before; import org.junit.Test; import java.io.ByteArrayInputStream; -import java.util.Iterator; import java.util.List; import static org.junit.Assert.*; @@ -689,105 +687,103 @@ public class VespaXMLReaderTestCase { DocumentUpdate docUpdate = op.getDocumentUpdate(); - assertEquals(20, docUpdate.fieldPathUpdates().size()); + assertEquals(20, docUpdate.getFieldPathUpdates().size()); - Iterator<FieldPathUpdate> updates = docUpdate.fieldPathUpdates().iterator(); { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(0); assertEquals("url", ass.getOriginalFieldPath()); assertEquals(new StringFieldValue("assignUrl"), ass.getNewValue()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(1); assertEquals("title", ass.getOriginalFieldPath()); assertEquals(new StringFieldValue("assignTitle"), ass.getNewValue()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(2); assertEquals("last_downloaded", ass.getOriginalFieldPath()); assertEquals("1", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(3); assertEquals("value_long", ass.getOriginalFieldPath()); assertEquals("2", ass.getExpression()); } - updates.next(); // Skip number 5 { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(5); assertEquals("stringarr", ass.getOriginalFieldPath()); assertEquals("[assignString1, assignString2]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(6); assertEquals("intarr", ass.getOriginalFieldPath()); assertEquals("[3, 4]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(7); assertEquals("longarr", ass.getOriginalFieldPath()); assertEquals("[5, 6]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(8); assertEquals("bytearr", ass.getOriginalFieldPath()); assertEquals("[7, 8]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(9); assertEquals("floatarr", ass.getOriginalFieldPath()); assertEquals("[9.0, 10.0]", ass.getNewValue().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(10); assertEquals("weightedsetint", ass.getOriginalFieldPath()); WeightedSet set = (WeightedSet)ass.getNewValue(); assertEquals(Integer.valueOf(11), set.get(new IntegerFieldValue(11))); assertEquals(Integer.valueOf(12), set.get(new IntegerFieldValue(12))); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(11); assertEquals("weightedsetstring", ass.getOriginalFieldPath()); WeightedSet set = (WeightedSet)ass.getNewValue(); assertEquals(Integer.valueOf(13), set.get(new StringFieldValue("assign13"))); assertEquals(Integer.valueOf(14), set.get(new StringFieldValue("assign14"))); } { - AddFieldPathUpdate ass = (AddFieldPathUpdate)updates.next(); + AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(12); assertEquals("stringarr", ass.getOriginalFieldPath()); assertEquals("[addString1, addString2]", ass.getNewValues().toString()); } { - AddFieldPathUpdate ass = (AddFieldPathUpdate)updates.next(); + AddFieldPathUpdate ass = (AddFieldPathUpdate)docUpdate.getFieldPathUpdates().get(13); assertEquals("longarr", ass.getOriginalFieldPath()); assertEquals("[5]", ass.getNewValues().toString()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(14); assertEquals("weightedsetint{13}", ass.getOriginalFieldPath()); assertEquals("13", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(15); assertEquals("weightedsetint{14}", ass.getOriginalFieldPath()); assertEquals("14", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(16); assertEquals("weightedsetstring{add13}", ass.getOriginalFieldPath()); assertEquals("1", ass.getExpression()); } { - AssignFieldPathUpdate ass = (AssignFieldPathUpdate)updates.next(); + AssignFieldPathUpdate ass = (AssignFieldPathUpdate)docUpdate.getFieldPathUpdates().get(17); assertEquals("weightedsetstring{assign13}", ass.getOriginalFieldPath()); assertEquals("130", ass.getExpression()); } { - RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)updates.next(); + RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(18); assertEquals("weightedsetstring{assign14}", ass.getOriginalFieldPath()); } { - RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)updates.next(); + RemoveFieldPathUpdate ass = (RemoveFieldPathUpdate)docUpdate.getFieldPathUpdates().get(19); assertEquals("bytearr", ass.getOriginalFieldPath()); } Document doc = new Document(manager.getDocumentType("news"), new DocumentId("doc:test:test:test")); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java index 2ccd8abffda..252f6c5bd12 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java @@ -57,7 +57,7 @@ public class SimpleAdapterFactory implements AdapterFactory { ret.add(new IdentityFieldPathUpdateAdapter(fieldUpd, newDocumentAdapter(complete, true))); } } - for (FieldUpdate fieldUpd : upd.fieldUpdates()) { + for (FieldUpdate fieldUpd : upd.getFieldUpdates()) { Field field = fieldUpd.getField(); for (ValueUpdate valueUpd : fieldUpd.getValueUpdates()) { if (FieldUpdateHelper.isComplete(field, valueUpd)) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java index 3c5eb9ea1c5..459f3ce827c 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToPathUpdateTestCase.java @@ -29,8 +29,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldPathUpdates().size()); - assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("my_int", upd.getOriginalFieldPath()); @@ -49,8 +49,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldPathUpdates().size()); - assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("my_str", upd.getOriginalFieldPath()); @@ -77,8 +77,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldPathUpdates().size()); - assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("a", upd.getOriginalFieldPath()); @@ -103,8 +103,8 @@ public class DocumentToPathUpdateTestCase { DocumentUpdate docUpd = new FieldPathUpdateAdapter(new SimpleDocumentAdapter(null, doc), upd).getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldPathUpdates().size()); - assertNotNull(upd = docUpd.fieldPathUpdates().iterator().next()); + assertEquals(1, docUpd.getFieldPathUpdates().size()); + assertNotNull(upd = docUpd.getFieldPathUpdates().get(0)); assertTrue(upd instanceof AssignFieldPathUpdate); assertEquals("a.b", upd.getOriginalFieldPath()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java index 83947b5f64d..de090163b7b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentToValueUpdateTestCase.java @@ -40,8 +40,8 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); - assertEquals("my_int", docUpd.fieldUpdates().iterator().next().getField().getName()); + assertEquals(1, docUpd.getFieldUpdates().size()); + assertEquals("my_int", docUpd.getFieldUpdate(0).getField().getName()); } @Test @@ -56,9 +56,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_int"), fieldUpd.getField()); @@ -80,9 +80,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_int"), fieldUpd.getField()); @@ -103,9 +103,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_str"), fieldUpd.getField()); @@ -136,9 +136,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("a"), fieldUpd.getField()); @@ -166,9 +166,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_arr"), fieldUpd.getField()); @@ -196,9 +196,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_arr"), fieldUpd.getField()); @@ -231,9 +231,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("a"), fieldUpd.getField()); @@ -261,9 +261,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_wset"), fieldUpd.getField()); @@ -294,9 +294,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_wset"), fieldUpd.getField()); @@ -326,9 +326,9 @@ public class DocumentToValueUpdateTestCase { UpdateAdapter adapter = FieldUpdateAdapter.fromPartialUpdate(new SimpleDocumentAdapter(null, doc), valueUpd); DocumentUpdate docUpd = adapter.getOutput(); assertNotNull(docUpd); - assertEquals(1, docUpd.fieldUpdates().size()); + assertEquals(1, docUpd.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpd.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpd.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_wset"), fieldUpd.getField()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java index dc8ffcd8d10..bdb8dbedf78 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/DocumentUpdateTestCase.java @@ -32,10 +32,10 @@ public class DocumentUpdateTestCase { docUpdate = Expression.execute(Expression.fromString("input my_str | for_each { to_pos } | index my_pos"), docUpdate); assertNotNull(docUpdate); - assertEquals(0, docUpdate.fieldPathUpdates().size()); - assertEquals(1, docUpdate.fieldUpdates().size()); + assertEquals(0, docUpdate.getFieldPathUpdates().size()); + assertEquals(1, docUpdate.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpdate.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_pos"), fieldUpd.getField()); assertEquals(1, fieldUpd.getValueUpdates().size()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java index 63a2cc66a97..033034fed1f 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GuardTestCase.java @@ -72,10 +72,10 @@ public class GuardTestCase { docUpdate.addFieldUpdate(FieldUpdate.createAssign(docType.getField("my_str"), new StringFieldValue("69"))); assertNotNull(docUpdate = Expression.execute(Expression.fromString("guard { input my_str | to_int | attribute my_lng }"), docUpdate)); - assertEquals(0, docUpdate.fieldPathUpdates().size()); - assertEquals(1, docUpdate.fieldUpdates().size()); + assertEquals(0, docUpdate.getFieldPathUpdates().size()); + assertEquals(1, docUpdate.getFieldUpdates().size()); - FieldUpdate fieldUpd = docUpdate.fieldUpdates().iterator().next(); + FieldUpdate fieldUpd = docUpdate.getFieldUpdate(0); assertNotNull(fieldUpd); assertEquals(docType.getField("my_lng"), fieldUpd.getField()); assertEquals(1, fieldUpd.getValueUpdates().size()); diff --git a/model-evaluation/pom.xml b/model-evaluation/pom.xml index 328d475c501..7c7410df833 100644 --- a/model-evaluation/pom.xml +++ b/model-evaluation/pom.xml @@ -72,6 +72,18 @@ <artifactId>guava</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_http_service</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jdisc_jetty</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java index e08b9f77d15..1412936d4a0 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java @@ -56,6 +56,6 @@ public class FunctionEvaluator { return function.getBody().evaluate(context).asTensor(); } - LazyArrayContext context() { return context; } + public LazyArrayContext context() { return context; } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java index beaa36b898f..c7d0cbd8f30 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java @@ -26,7 +26,7 @@ import java.util.Set; * * @author bratseth */ -final class LazyArrayContext extends Context implements ContextIndex { +public final class LazyArrayContext extends Context implements ContextIndex { private final IndexedBindings indexedBindings; diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index d2fca309a19..7bea2d0825a 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -81,11 +81,11 @@ public class RankProfilesConfigImporter { referencedFunctions.put(reference.get(), new ExpressionFunction(reference.get().serialForm(), arguments, expression)); } - else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to macros + else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to functions firstPhase = new ExpressionFunction("firstphase", new ArrayList<>(), new RankingExpression("first-phase", property.value())); } - else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to macros + else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to functions secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(), new RankingExpression("second-phase", property.value())); } diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index 78c46864d7b..1c995c255f5 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -1,47 +1,213 @@ package ai.vespa.models.handler; +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.serialization.JsonFormat; import java.io.IOException; import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.Charset; +import java.util.Optional; +import java.util.concurrent.Executor; -public class ModelsEvaluationHandler extends LoggingRequestHandler { +public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler { + + public static final String API_ROOT = "model-evaluation"; + public static final String VERSION_V1 = "v1"; + public static final String EVALUATE = "eval"; private final ModelsEvaluator modelsEvaluator; - public ModelsEvaluationHandler(ModelsEvaluator modelsEvaluator, Context context) { - super(context); + public ModelsEvaluationHandler(ModelsEvaluator modelsEvaluator, Executor executor) { + super(executor); this.modelsEvaluator = modelsEvaluator; } @Override public HttpResponse handle(HttpRequest request) { - Tensor result = modelsEvaluator.evaluatorOf(property("model", "serving_default", request), - request.getProperty("function")) - .evaluate(); - return new RawResponse(JsonFormat.encode(result)); + Path path = new Path(request); + Optional<String> apiName = path.segment(0); + Optional<String> version = path.segment(1); + Optional<String> modelName = path.segment(2); + + if ( ! apiName.isPresent() || ! apiName.get().equalsIgnoreCase(API_ROOT)) { + return new ErrorResponse(404, "unknown API"); + } + if ( ! version.isPresent() || ! version.get().equalsIgnoreCase(VERSION_V1)) { + return new ErrorResponse(404, "unknown API version"); + } + if ( ! modelName.isPresent()) { + return listAllModels(request); + } + if ( ! modelsEvaluator.models().containsKey(modelName.get())) { + // TODO: Replace by catching IllegalArgumentException and passing that error message + return new ErrorResponse(404, "no model with name '" + modelName.get() + "' found"); + } + + Model model = modelsEvaluator.models().get(modelName.get()); + + // The following logic follows from the spec, in that signature and + // output are optional if the model only has a single function. + // TODO: Try to avoid recreating that logic here + + if (path.segments() == 3) { + if (model.functions().size() > 1) { + return listModelDetails(request, modelName.get()); + } + return listTypeDetails(request, modelName.get()); + } + + if (path.segments() == 4) { + if ( ! path.segment(3).get().equalsIgnoreCase(EVALUATE)) { + return listTypeDetails(request, modelName.get(), path.segment(3).get()); + } + if (model.functions().stream().anyMatch(f -> f.getName().equalsIgnoreCase(EVALUATE))) { + return listTypeDetails(request, modelName.get(), path.segment(3).get()); // model has a function "eval" + } + if (model.functions().size() <= 1) { + return evaluateModel(request, modelName.get()); + } + // TODO: Replace by catching IllegalArgumentException and passing that error message + return new ErrorResponse(404, "attempt to evaluate model without specifying function"); + } + + if (path.segments() == 5) { + if (path.segment(4).get().equalsIgnoreCase(EVALUATE)) { + return evaluateModel(request, modelName.get(), path.segment(3).get()); + } + } + + return new ErrorResponse(404, "unrecognized request"); + } + + private HttpResponse listAllModels(HttpRequest request) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + for (String modelName: modelsEvaluator.models().keySet()) { + root.setString(modelName, baseUrl(request) + modelName); + } + return new Response(200, com.yahoo.slime.JsonFormat.toJsonBytes(slime)); + } + + private HttpResponse listModelDetails(HttpRequest request, String modelName) { + Model model = modelsEvaluator.models().get(modelName); + Slime slime = new Slime(); + Cursor root = slime.setObject(); + for (ExpressionFunction func : model.functions()) { + root.setString(func.getName(), baseUrl(request) + modelName + "/" + func.getName()); + } + return new Response(200, com.yahoo.slime.JsonFormat.toJsonBytes(slime)); + } + + private HttpResponse listTypeDetails(HttpRequest request, String modelName) { + return listTypeDetails(request, modelsEvaluator.evaluatorOf(modelName)); + } + + private HttpResponse listTypeDetails(HttpRequest request, String modelName, String signatureAndOutput) { + return listTypeDetails(request, modelsEvaluator.evaluatorOf(modelName, signatureAndOutput)); + } + + private HttpResponse listTypeDetails(HttpRequest request, FunctionEvaluator evaluator) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor bindings = root.setArray("bindings"); + for (String bindingName : evaluator.context().names()) { + // TODO: Use an API which exposes only the external binding names instead of this + if (bindingName.startsWith("constant(")) { + continue; + } + if (bindingName.startsWith("rankingExpression(")) { + continue; + } + Cursor binding = bindings.addObject(); + binding.setString("name", bindingName); + binding.setString("type", ""); // TODO: implement type information when available + } + return new Response(200, com.yahoo.slime.JsonFormat.toJsonBytes(slime)); + } + + private HttpResponse evaluateModel(HttpRequest request, String modelName) { + return evaluateModel(request, modelsEvaluator.evaluatorOf(modelName)); + } + + private HttpResponse evaluateModel(HttpRequest request, String modelName, String signatureAndOutput) { + return evaluateModel(request, modelsEvaluator.evaluatorOf(modelName, signatureAndOutput)); + } + + private HttpResponse evaluateModel(HttpRequest request, FunctionEvaluator evaluator) { + for (String bindingName : evaluator.context().names()) { + property(request, bindingName).ifPresent(s -> evaluator.bind(bindingName, Tensor.from(s))); + } + Tensor result = evaluator.evaluate(); + return new Response(200, JsonFormat.encode(result)); + } + + private Optional<String> property(HttpRequest request, String name) { + return Optional.ofNullable(request.getProperty(name)); } - private String property(String name, String defaultValue, HttpRequest request) { - String value = request.getProperty(name); - if (value == null) return defaultValue; - return value; + private String baseUrl(HttpRequest request) { + URI uri = request.getUri(); + StringBuilder sb = new StringBuilder(); + sb.append(uri.getScheme()).append("://").append(uri.getHost()); + if (uri.getPort() >= 0) { + sb.append(":").append(uri.getPort()); + } + sb.append("/").append(API_ROOT).append("/").append(VERSION_V1).append("/"); + return sb.toString(); } - private static class RawResponse extends HttpResponse { + private static class Path { + + private final String[] segments; + + public Path(HttpRequest httpRequest) { + segments = splitPath(httpRequest); + } + + Optional<String> segment(int index) { + return (index < 0 || index >= segments.length) ? Optional.empty() : Optional.of(segments[index]); + } + + int segments() { + return segments.length; + } + + private static String[] splitPath(HttpRequest request) { + String path = request.getUri().getPath().toLowerCase(); + if (path.startsWith("/")) { + path = path.substring("/".length()); + } + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + return path.split("/"); + } + + } + + private static class Response extends HttpResponse { private final byte[] data; - RawResponse(byte[] data) { - super(200); + Response(int code, byte[] data) { + super(code); this.data = data; } + Response(int code, String data) { + this(code, data.getBytes(Charset.forName(DEFAULT_CHARACTER_ENCODING))); + } + @Override public String getContentType() { return "application/json"; @@ -53,5 +219,11 @@ public class ModelsEvaluationHandler extends LoggingRequestHandler { } } + private static class ErrorResponse extends Response { + ErrorResponse(int code, String data) { + super(code, "{\"error\":\"" + data + "\"}"); + } + } + } diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/package-info.java b/model-evaluation/src/main/java/ai/vespa/models/handler/package-info.java new file mode 100644 index 00000000000..7978abf2632 --- /dev/null +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/package-info.java @@ -0,0 +1,4 @@ +@ExportPackage +package ai.vespa.models.handler; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java index 0aceaccc3e0..9a3e59aed80 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelTester.java @@ -65,7 +65,7 @@ public class ModelTester { } /** Allows us to provide canned tensor constants during import since file distribution does not work in tests */ - private static class RankProfilesConfigImporterWithMockedConstants extends RankProfilesConfigImporter { + public static class RankProfilesConfigImporterWithMockedConstants extends RankProfilesConfigImporter { private static final Logger log = Logger.getLogger(RankProfilesConfigImporterWithMockedConstants.class.getName()); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java new file mode 100644 index 00000000000..5f045a2feb4 --- /dev/null +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -0,0 +1,201 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.handler; + +import ai.vespa.models.evaluation.ModelTester; +import ai.vespa.models.evaluation.ModelsEvaluator; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.FileSource; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; +import com.yahoo.path.Path; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +public class ModelsEvaluationHandlerTest { + + private static ModelsEvaluationHandler handler; + + @BeforeClass + static public void setUp() { + Executor executor = Executors.newSingleThreadExecutor(); + ModelsEvaluator models = createModels("src/test/resources/config/models/"); + handler = new ModelsEvaluationHandler(models, executor); + } + + @Test + public void testUnknownAPI() { + assertResponse("http://localhost/wrong-api-binding", 404); + } + + @Test + public void testUnknownVersion() { + assertResponse("http://localhost/model-evaluation/v0", 404); + } + + @Test + public void testNonExistingModel() { + assertResponse("http://localhost/model-evaluation/v1/non-existing-model", 404); + } + + @Test + public void testListModels() { + String url = "http://localhost/model-evaluation/v1"; + String expected = "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\"}"; + assertResponse(url, 200, expected); + } + + @Test + public void testXgBoostEvaluationWithoutBindings() { + String url = "http://localhost/model-evaluation/v1/xgboost_2_2/eval"; // only has a single function + String expected = "{\"cells\":[{\"address\":{},\"value\":-8.17695}]}"; + assertResponse(url, 200, expected); + } + + @Test + public void testXgBoostEvaluationWithBindings() { + Map<String, String> properties = new HashMap<>(); + properties.put("f29", "-1.0"); + properties.put("f56", "0.2"); + properties.put("f60", "0.3"); + properties.put("f109", "0.4"); + properties.put("non-existing-binding", "-1"); + String url = "http://localhost/model-evaluation/v1/xgboost_2_2/eval"; + String expected = "{\"cells\":[{\"address\":{},\"value\":-7.936679999999999}]}"; + assertResponse(url, properties, 200, expected); + } + + @Test + public void testMnistSoftmaxDetails() { + String url = "http://localhost:8080/model-evaluation/v1/mnist_softmax"; + String expected = "{\"bindings\":[{\"name\":\"Placeholder\",\"type\":\"\"}]}"; // only has a single function + assertResponse(url, 200, expected); + } + + @Test + public void testMnistSoftmaxTypeDetails() { + String url = "http://localhost/model-evaluation/v1/mnist_softmax/default.add/"; + String expected = "{\"bindings\":[{\"name\":\"Placeholder\",\"type\":\"\"}]}"; + assertResponse(url, 200, expected); + } + + @Test + public void testMnistSoftmaxEvaluateDefaultFunctionWithoutBindings() { + String url = "http://localhost/model-evaluation/v1/mnist_softmax/eval"; + String expected = "{\"cells\":[{\"address\":{\"d1\":\"0\"},\"value\":-0.3546536862850189},{\"address\":{\"d1\":\"1\"},\"value\":0.3759574592113495},{\"address\":{\"d1\":\"2\"},\"value\":0.06054411828517914},{\"address\":{\"d1\":\"3\"},\"value\":-0.251544713973999},{\"address\":{\"d1\":\"4\"},\"value\":0.017951013520359993},{\"address\":{\"d1\":\"5\"},\"value\":1.2899067401885986},{\"address\":{\"d1\":\"6\"},\"value\":-0.10389615595340729},{\"address\":{\"d1\":\"7\"},\"value\":0.6367976665496826},{\"address\":{\"d1\":\"8\"},\"value\":-1.4136744737625122},{\"address\":{\"d1\":\"9\"},\"value\":-0.2573896050453186}]}"; + assertResponse(url, 200, expected); + } + + @Test + public void testMnistSoftmaxEvaluateSpecificFunctionWithoutBindings() { + String url = "http://localhost/model-evaluation/v1/mnist_softmax/default.add/eval"; + String expected = "{\"cells\":[{\"address\":{\"d1\":\"0\"},\"value\":-0.3546536862850189},{\"address\":{\"d1\":\"1\"},\"value\":0.3759574592113495},{\"address\":{\"d1\":\"2\"},\"value\":0.06054411828517914},{\"address\":{\"d1\":\"3\"},\"value\":-0.251544713973999},{\"address\":{\"d1\":\"4\"},\"value\":0.017951013520359993},{\"address\":{\"d1\":\"5\"},\"value\":1.2899067401885986},{\"address\":{\"d1\":\"6\"},\"value\":-0.10389615595340729},{\"address\":{\"d1\":\"7\"},\"value\":0.6367976665496826},{\"address\":{\"d1\":\"8\"},\"value\":-1.4136744737625122},{\"address\":{\"d1\":\"9\"},\"value\":-0.2573896050453186}]}"; + assertResponse(url, 200, expected); + } + + @Test + public void testMnistSoftmaxEvaluateDefaultFunctionWithBindings() { + Map<String, String> properties = new HashMap<>(); + properties.put("Placeholder", "{1.0}"); + String url = "http://localhost/model-evaluation/v1/mnist_softmax/eval"; + String expected = "{\"cells\":[{\"address\":{\"d1\":\"0\"},\"value\":2.7147769462592217},{\"address\":{\"d1\":\"1\"},\"value\":-19.710327346521872},{\"address\":{\"d1\":\"2\"},\"value\":9.496512226053643},{\"address\":{\"d1\":\"3\"},\"value\":13.11241075176957},{\"address\":{\"d1\":\"4\"},\"value\":-12.355567088005559},{\"address\":{\"d1\":\"5\"},\"value\":10.39812446509341},{\"address\":{\"d1\":\"6\"},\"value\":-1.3739236534397499},{\"address\":{\"d1\":\"7\"},\"value\":-3.4260787871386995},{\"address\":{\"d1\":\"8\"},\"value\":6.471120687192041},{\"address\":{\"d1\":\"9\"},\"value\":-5.327024804970982}]}"; + assertResponse(url, properties, 200, expected); + } + + @Test + public void testMnistSoftmaxEvaluateSpecificFunctionWithBindings() { + Map<String, String> properties = new HashMap<>(); + properties.put("Placeholder", "{1.0}"); + String url = "http://localhost/model-evaluation/v1/mnist_softmax/default.add/eval"; + String expected = "{\"cells\":[{\"address\":{\"d1\":\"0\"},\"value\":2.7147769462592217},{\"address\":{\"d1\":\"1\"},\"value\":-19.710327346521872},{\"address\":{\"d1\":\"2\"},\"value\":9.496512226053643},{\"address\":{\"d1\":\"3\"},\"value\":13.11241075176957},{\"address\":{\"d1\":\"4\"},\"value\":-12.355567088005559},{\"address\":{\"d1\":\"5\"},\"value\":10.39812446509341},{\"address\":{\"d1\":\"6\"},\"value\":-1.3739236534397499},{\"address\":{\"d1\":\"7\"},\"value\":-3.4260787871386995},{\"address\":{\"d1\":\"8\"},\"value\":6.471120687192041},{\"address\":{\"d1\":\"9\"},\"value\":-5.327024804970982}]}"; + assertResponse(url, properties, 200, expected); + } + + @Test + public void testMnistSavedDetails() { + String url = "http://localhost:8080/model-evaluation/v1/mnist_saved"; + String expected = "{\"imported_ml_macro_mnist_saved_dnn_hidden1_add\":\"http://localhost:8080/model-evaluation/v1/mnist_saved/imported_ml_macro_mnist_saved_dnn_hidden1_add\",\"serving_default.y\":\"http://localhost:8080/model-evaluation/v1/mnist_saved/serving_default.y\"}"; + assertResponse(url, 200, expected); + } + + @Test + public void testMnistSavedTypeDetails() { + String url = "http://localhost/model-evaluation/v1/mnist_saved/serving_default.y/"; + String expected = "{\"bindings\":[{\"name\":\"input\",\"type\":\"\"}]}"; + assertResponse(url, 200, expected); + } + + @Test + public void testMnistSavedEvaluateDefaultFunctionShouldFail() { + String url = "http://localhost/model-evaluation/v1/mnist_saved/eval"; + String expected = "{\"error\":\"attempt to evaluate model without specifying function\"}"; + assertResponse(url, 404, expected); + } + + @Test + public void testMnistSavedEvaluateSpecificFunction() { + Map<String, String> properties = new HashMap<>(); + properties.put("input", "-1.0"); + String url = "http://localhost/model-evaluation/v1/mnist_saved/serving_default.y/eval"; + String expected = "{\"cells\":[{\"address\":{\"d1\":\"0\"},\"value\":-2.72208123403445},{\"address\":{\"d1\":\"1\"},\"value\":6.465137496457595},{\"address\":{\"d1\":\"2\"},\"value\":-7.078050386283122},{\"address\":{\"d1\":\"3\"},\"value\":-10.485296462655546},{\"address\":{\"d1\":\"4\"},\"value\":0.19508378636937004},{\"address\":{\"d1\":\"5\"},\"value\":6.348870746681019},{\"address\":{\"d1\":\"6\"},\"value\":10.756191852397258},{\"address\":{\"d1\":\"7\"},\"value\":1.476101533270058},{\"address\":{\"d1\":\"8\"},\"value\":-17.778398655804875},{\"address\":{\"d1\":\"9\"},\"value\":-2.0597690508530295}]}"; + assertResponse(url, properties, 200, expected); + } + + static private void assertResponse(String url, int expectedCode) { + assertResponse(url, Collections.emptyMap(), expectedCode, null); + } + + static private void assertResponse(String url, int expectedCode, String expectedResult) { + assertResponse(url, Collections.emptyMap(), expectedCode, expectedResult); + } + + static private void assertResponse(String url, Map<String, String> properties, int expectedCode, String expectedResult) { + HttpRequest getRequest = HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET, null, properties); + HttpRequest postRequest = HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.POST, null, properties); + assertResponse(getRequest, expectedCode, expectedResult); + assertResponse(postRequest, expectedCode, expectedResult); + } + + static private void assertResponse(HttpRequest request, int expectedCode, String expectedResult) { + HttpResponse response = handler.handle(request); + assertEquals("application/json", response.getContentType()); + assertEquals(expectedCode, response.getStatus()); + if (expectedResult != null) { + assertEquals(expectedResult, getContents(response)); + } + } + + static private String getContents(HttpResponse response) { + try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) { + response.render(stream); + return stream.toString(); + } catch (IOException e) { + throw new Error(e); + } + } + + static private ModelsEvaluator createModels(String path) { + Path configDir = Path.fromString(path); + RankProfilesConfig config = new ConfigGetter<>(new FileSource(configDir.append("rank-profiles.cfg").toFile()), + RankProfilesConfig.class).getConfig(""); + RankingConstantsConfig constantsConfig = new ConfigGetter<>(new FileSource(configDir.append("ranking-constants.cfg").toFile()), + RankingConstantsConfig.class).getConfig(""); + ModelTester.RankProfilesConfigImporterWithMockedConstants importer = + new ModelTester.RankProfilesConfigImporterWithMockedConstants(Path.fromString(path).append("constants"), + MockFileAcquirer.returnFile(null)); + return new ModelsEvaluator(importer.importFrom(config, constantsConfig)); + } + +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index c6d8f70fde8..da34ab8822d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,13 +3,16 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.text.Utf8; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.*; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * A function defined by a ranking expression @@ -24,6 +27,16 @@ public class ExpressionFunction { private final RankingExpression body; /** + * Constructs a new function with no arguments + * + * @param name the name of this function + * @param body the ranking expression that defines this function + */ + public ExpressionFunction(String name, RankingExpression body) { + this(name, Collections.emptyList(), body); + } + + /** * Constructs a new function * * @param name the name of this function diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index ed82ba20fbe..722520fea08 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -250,12 +250,12 @@ public class RankingExpression implements Serializable { /** * Creates the necessary rank properties required to implement this expression. * - * @param macros the expression macros to expand. - * @return a list of named rank properties required to implement this expression. + * @param functions the expression functions to expand + * @return a list of named rank properties required to implement this expression */ - public Map<String, String> getRankProperties(List<ExpressionFunction> macros) { + public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { Deque<String> path = new LinkedList<>(); - SerializationContext context = new SerializationContext(macros); + SerializationContext context = new SerializationContext(functions); String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString(); Map<String, String> serializedExpressions = context.serializedFunctions(); serializedExpressions.put(propertyName(name), serializedRoot); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java index ac5eefcc5b2..282a4c5e0a9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java @@ -30,8 +30,8 @@ public class ImportedModel { private final Map<String, Tensor> smallConstants = new HashMap<>(); private final Map<String, Tensor> largeConstants = new HashMap<>(); private final Map<String, RankingExpression> expressions = new HashMap<>(); - private final Map<String, RankingExpression> macros = new HashMap<>(); - private final Map<String, TensorType> requiredMacros = new HashMap<>(); + private final Map<String, RankingExpression> functions = new HashMap<>(); + private final Map<String, TensorType> requiredFunctions = new HashMap<>(); /** * Creates a new imported model. @@ -77,13 +77,13 @@ public class ImportedModel { public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); } /** - * Returns an immutable map of macros that are part of this model. - * Note that the macros themselves are *not* copies and *not* immutable - they must be copied before modification. + * Returns an immutable map of the functions that are part of this model. + * Note that the functions themselves are *not* copies and *not* immutable - they must be copied before modification. */ - public Map<String, RankingExpression> macros() { return Collections.unmodifiableMap(macros); } + public Map<String, RankingExpression> functions() { return Collections.unmodifiableMap(functions); } - /** Returns an immutable map of the macros that must be provided by the environment running this model */ - public Map<String, TensorType> requiredMacros() { return Collections.unmodifiableMap(requiredMacros); } + /** Returns an immutable map of the functions that must be provided by the environment running this model */ + public Map<String, TensorType> requiredFunctions() { return Collections.unmodifiableMap(requiredFunctions); } /** Returns an immutable map of the signatures of this */ public Map<String, Signature> signatures() { return Collections.unmodifiableMap(signatures); } @@ -100,8 +100,8 @@ public class ImportedModel { void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); } void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); } void expression(String name, RankingExpression expression) { expressions.put(name, expression); } - void macro(String name, RankingExpression expression) { macros.put(name, expression); } - void requiredMacro(String name, TensorType type) { requiredMacros.put(name, type); } + void function(String name, RankingExpression expression) { functions.put(name, expression); } + void requiredFunction(String name, TensorType type) { requiredFunctions.put(name, type); } /** * Returns all the output expressions of this indexed by name. The names consist of one or two parts diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java index 2ae107a5770..d25502fd149 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java @@ -24,7 +24,7 @@ import java.util.logging.Logger; * ranking expressions. The general mechanism for import is for the * specific ML platform import implementations to create an * IntermediateGraph. This class offers common code to convert the - * IntermediateGraph to Vespa ranking expressions and macros. + * IntermediateGraph to Vespa ranking expressions and functions. * * @author lesters */ @@ -122,7 +122,7 @@ public abstract class ModelImporter { importExpressionInputs(operation, model); importRankingExpression(operation, model); importArgumentExpression(operation, model); - importMacroExpression(operation, model); + importFunctionExpression(operation, model); return operation.function(); } @@ -188,15 +188,15 @@ public abstract class ModelImporter { // All inputs must have dimensions with standard naming convention: d0, d1, ... OrderedTensorType standardNamingConvention = OrderedTensorType.standardType(operation.type().get()); model.argument(operation.vespaName(), standardNamingConvention.type()); - model.requiredMacro(operation.vespaName(), standardNamingConvention.type()); + model.requiredFunction(operation.vespaName(), standardNamingConvention.type()); } } - private static void importMacroExpression(IntermediateOperation operation, ImportedModel model) { - if (operation.macro().isPresent()) { - TensorFunction function = operation.macro().get(); + private static void importFunctionExpression(IntermediateOperation operation, ImportedModel model) { + if (operation.rankingExpressionFunction().isPresent()) { + TensorFunction function = operation.rankingExpressionFunction().get(); try { - model.macro(operation.macroName(), new RankingExpression(operation.macroName(), function.toString())); + model.function(operation.rankingExpressionFunctionName(), new RankingExpression(operation.rankingExpressionFunctionName(), function.toString())); } catch (ParseException e) { throw new RuntimeException("Tensorflow function " + function + diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java index 43de29cedd5..34f5f1365a1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/IntermediateOperation.java @@ -29,7 +29,7 @@ import java.util.function.Function; */ public abstract class IntermediateOperation { - private final static String MACRO_PREFIX = "imported_ml_macro_"; + private final static String FUNCTION_PREFIX = "imported_ml_function_"; protected final String name; protected final String modelName; @@ -38,7 +38,7 @@ public abstract class IntermediateOperation { protected OrderedTensorType type; protected TensorFunction function; - protected TensorFunction macro = null; + protected TensorFunction rankingExpressionFunction = null; private final List<String> importWarnings = new ArrayList<>(); private Value constantValue = null; @@ -71,8 +71,8 @@ public abstract class IntermediateOperation { ExpressionNode constant = new ReferenceNode(Reference.simple("constant", vespaName())); function = new TensorFunctionNode.TensorFunctionExpressionNode(constant); } else if (outputs.size() > 1) { - macro = lazyGetFunction(); - function = new VariableTensor(macroName(), type.type()); + rankingExpressionFunction = lazyGetFunction(); + function = new VariableTensor(rankingExpressionFunctionName(), type.type()); } else { function = lazyGetFunction(); } @@ -86,11 +86,13 @@ public abstract class IntermediateOperation { /** Return unmodifiable list of inputs */ public List<IntermediateOperation> inputs() { return inputs; } - /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a macro. */ + /** Return unmodifiable list of outputs. If a node has multiple outputs, consider adding a function. */ public List<IntermediateOperation> outputs() { return Collections.unmodifiableList(outputs); } - /** Returns a Vespa ranking expression that should be added as a macro */ - public Optional<TensorFunction> macro() { return Optional.ofNullable(macro); } + /** Returns a function that should be added as a ranking expression function */ + public Optional<TensorFunction> rankingExpressionFunction() { + return Optional.ofNullable(rankingExpressionFunction); + } /** Add dimension name constraints for this operation */ public void addDimensionNameConstraints(DimensionRenamer renamer) { } @@ -131,8 +133,10 @@ public abstract class IntermediateOperation { public String vespaName() { return vespaName(name); } public String vespaName(String name) { return name != null ? namePartOf(name).replace('/', '_') : null; } - /** Retrieve the valid Vespa name of this node if it is a macro */ - public String macroName() { return vespaName() != null ? MACRO_PREFIX + modelName + "_" + vespaName() : null; } + /** Retrieve the valid Vespa name of this node if it is a ranking expression function */ + public String rankingExpressionFunctionName() { + return vespaName() != null ? FUNCTION_PREFIX + modelName + "_" + vespaName() : null; + } /** Retrieve the list of warnings produced during its lifetime */ public List<String> warnings() { return Collections.unmodifiableList(importWarnings); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java index 9299ae9be12..b335fd7e1c5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/importer/operations/PlaceholderWithDefault.java @@ -26,13 +26,13 @@ public class PlaceholderWithDefault extends IntermediateOperation { if (!allInputFunctionsPresent(1)) { return null; } - // This should be a call to the macro we add below, but for now + // This should be a call to the function we add below, but for now // we treat this as as identity function and just pass the constant. return inputs.get(0).function().orElse(null); } @Override - public Optional<TensorFunction> macro() { + public Optional<TensorFunction> rankingExpressionFunction() { // For now, it is much more efficient to assume we always will return // the default value, as we can prune away large parts of the expression // tree by having it calculated as a constant. If a case arises where @@ -42,7 +42,7 @@ public class PlaceholderWithDefault extends IntermediateOperation { @Override public boolean isConstant() { - return true; // not true if we add to macro + return true; // not true if we add to function } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index e2dc170c168..eb8d2229a6d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -15,7 +15,7 @@ import java.util.Deque; import java.util.List; /** - * A node referring either to a value in the context or to a named ranking expression (function aka macro). + * A node referring either to a value in the context or to a named ranking expression function. * * @author bratseth */ @@ -64,7 +64,7 @@ public final class ReferenceNode extends CompositeNode { @Override public StringBuilder toString(StringBuilder string, SerializationContext context, Deque<String> path, CompositeNode parent) { - // A reference to a macro argument? + // A reference to a function argument? if (reference.isIdentifier() && context.getBinding(getName()) != null) { // a bound identifier: replace by the value it is bound to return string.append(context.getBinding(getName())); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index 7c929ae24b3..571e1f4d608 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -91,26 +91,26 @@ public class RankingExpressionTestCase { @Test public void testSelfRecursionSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", null, new RankingExpression("foo"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(macros); + exp.getRankProperties(functions); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[]]", e.getMessage()); } } @Test - public void testMacroCycleSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); - macros.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); + public void testFunctionCycleSerialization() throws ParseException { + List<ExpressionFunction> funnctions = new ArrayList<>(); + funnctions.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); + funnctions.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(macros); + exp.getRankProperties(funnctions); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[], bar[]]", e.getMessage()); } @@ -118,11 +118,11 @@ public class RankingExpressionTestCase { @Test public void testSerialization() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); - macros.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); - macros.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); - macros.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", Arrays.asList("arg1", "arg2"), new RankingExpression("min(arg1, pow(arg2, 2))"))); + functions.add(new ExpressionFunction("bar", Arrays.asList("arg1", "arg2"), new RankingExpression("arg1 * arg1 + 2 * arg1 * arg2 + arg2 * arg2"))); + functions.add(new ExpressionFunction("baz", Arrays.asList("arg1", "arg2"), new RankingExpression("foo(1, 2) / bar(arg1, arg2)"))); + functions.add(new ExpressionFunction("cox", null, new RankingExpression("10 + 08 * 1977"))); assertSerialization(Arrays.asList( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(foo@af74e3fd9070bd18.a368ed0a5ba3a5d0) * rankingExpression(foo@dbab346efdad5362.e5c39e42ebd91c30)", @@ -130,19 +130,19 @@ public class RankingExpressionTestCase { "min(6,pow(7,2))", "min(1,pow(2,2))", "min(3,pow(4,2))", - "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", macros); + "min(rankingExpression(foo@84951be88255b0ec.d0303e061b36fab8),pow(8,2))"), "foo(1,2) + foo(3,4) * foo(5, foo(foo(6, 7), 8))", functions); assertSerialization(Arrays.asList( "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) + rankingExpression(bar@af74e3fd9070bd18.a368ed0a5ba3a5d0)", "min(1,pow(2,2))", - "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", macros); + "3 * 3 + 2 * 3 * 4 + 4 * 4"), "foo(1, 2) + bar(3, 4)", functions); assertSerialization(Arrays.asList( "rankingExpression(baz@e2dc17a89864aed0.12232eb692c6c502)", "min(1,pow(2,2))", "rankingExpression(foo@e2dc17a89864aed0.12232eb692c6c502) / rankingExpression(bar@e2dc17a89864aed0.12232eb692c6c502)", - "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", macros); + "1 * 1 + 2 * 1 * 2 + 2 * 2"), "baz(1, 2)", functions); assertSerialization(Arrays.asList( "rankingExpression(cox)", - "10 + 08 * 1977"), "cox", macros + "10 + 08 * 1977"), "cox", functions ); } @@ -159,8 +159,8 @@ public class RankingExpressionTestCase { @Test public void testBug3464208() throws ParseException { - List<ExpressionFunction> macros = new ArrayList<>(); - macros.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("log10tweetage", null, new RankingExpression("69"))); String lhs = "log10(0.01+attribute(user_followers_count)) * log10(socialratio) * " + "log10(userage/(0.01+attribute(user_statuses_count)))"; @@ -172,8 +172,8 @@ public class RankingExpressionTestCase { String expRhs = "(rankingExpression(log10tweetage) * rankingExpression(log10tweetage) * " + "rankingExpression(log10tweetage)) + 5.0 * attribute(ythl)"; - assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, macros); - assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, macros); + assertSerialization(Arrays.asList(expLhs + " + " + expRhs, "69"), lhs + " + " + rhs, functions); + assertSerialization(Arrays.asList(expLhs + " - " + expRhs, "69"), lhs + " - " + rhs, functions); } @Test @@ -295,12 +295,12 @@ public class RankingExpressionTestCase { assertEquals(expected, new RankingExpression(expression).toString()); } - /** Test serialization with no macros */ + /** Test serialization with no functions */ private void assertSerialization(String expectedSerialization, String expressionString) { String serializedExpression; try { RankingExpression expression = new RankingExpression(expressionString); - // No macros -> expect one rank property + // No functions -> expect one rank property serializedExpression = expression.getRankProperties(Collections.emptyList()).values().iterator().next(); assertEquals(expectedSerialization, serializedExpression); } @@ -309,7 +309,7 @@ public class RankingExpressionTestCase { } try { - // No macros -> output should be parseable to a ranking expression + // No functions -> output should be parseable to a ranking expression // (but not the same one due to primitivization) RankingExpression reparsedExpression = new RankingExpression(serializedExpression); // Serializing the primitivized expression should yield the same expression again @@ -323,17 +323,17 @@ public class RankingExpressionTestCase { } private void assertSerialization(List<String> expectedSerialization, String expressionString, - List<ExpressionFunction> macros) { - assertSerialization(expectedSerialization, expressionString, macros, false); + List<ExpressionFunction> functions) { + assertSerialization(expectedSerialization, expressionString, functions, false); } private void assertSerialization(List<String> expectedSerialization, String expressionString, - List<ExpressionFunction> macros, boolean print) { + List<ExpressionFunction> functions, boolean print) { try { if (print) System.out.println("Parsing expression '" + expressionString + "'."); RankingExpression expression = new RankingExpression(expressionString); - Map<String, String> rankProperties = expression.getRankProperties(macros); + Map<String, String> rankProperties = expression.getRankProperties(functions); if (print) { for (String key : rankProperties.keySet()) System.out.println("Property '" + key + "': " + rankProperties.get(key)); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java index a63c7346335..a8f7542f3a4 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java @@ -18,11 +18,11 @@ public class DropoutImportTestCase { public void testDropoutImport() { TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/files/integration/tensorflow/dropout/saved"); - // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("X")); + // Check required functions + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("X")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("X")); + model.get().requiredFunctions().get("X")); ImportedModel.Signature signature = model.get().signature("serving_default"); @@ -32,7 +32,7 @@ public class DropoutImportTestCase { RankingExpression output = signature.outputExpression("y"); assertNotNull(output); assertEquals("outputs/Maximum", output.getName()); - assertEquals("join(join(imported_ml_macro_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_macro_test_outputs_BiasAdd, f(a,b)(max(a,b)))", + assertEquals("join(join(imported_ml_function_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_function_test_outputs_BiasAdd, f(a,b)(max(a,b)))", output.getRoot().toString()); model.assertEqualResult("X", output.getName()); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java index bcfc6ce0a04..e20ac16a691 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java @@ -36,11 +36,11 @@ public class OnnxMnistSoftmaxImportTestCase { constant1.type()); assertEquals(10, constant1.size()); - // Check required macros (inputs) - assertEquals(1, model.requiredMacros().size()); - assertTrue(model.requiredMacros().containsKey("Placeholder")); + // Check required functions (inputs) + assertEquals(1, model.requiredFunctions().size()); + assertTrue(model.requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.requiredMacros().get("Placeholder")); + model.requiredFunctions().get("Placeholder")); // Check outputs RankingExpression output = model.defaultSignature().outputExpression("add"); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java index dd6c8095e3c..ef28eb4678f 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java @@ -34,14 +34,14 @@ public class TensorFlowMnistSoftmaxImportTestCase { constant1.type()); assertEquals(10, constant1.size()); - // Check (provided) macros - assertEquals(0, model.get().macros().size()); + // Check (provided) functions + assertEquals(0, model.get().functions().size()); - // Check required macros - assertEquals(1, model.get().requiredMacros().size()); - assertTrue(model.get().requiredMacros().containsKey("Placeholder")); + // Check required functions + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().requiredMacros().get("Placeholder")); + model.get().requiredFunctions().get("Placeholder")); // Check signatures assertEquals(1, model.get().signatures().size()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java index 4de3aa5d635..5447e5240f7 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TestableTensorFlowModel.java @@ -48,7 +48,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateFunction(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", @@ -62,7 +62,7 @@ public class TestableTensorFlowModel { Tensor placeholder = placeholderArgument(); context.put(inputName, new TensorValue(placeholder)); - model.macros().forEach((k,v) -> evaluateMacro(context, model, k)); + model.functions().forEach((k, v) -> evaluateFunction(context, model, k)); Tensor vespaResult = model.expressions().get(operationName).evaluate(context).asTensor(); assertEquals("Operation '" + operationName + "' produces equal results", tfResult, vespaResult); @@ -96,24 +96,24 @@ public class TestableTensorFlowModel { return b.build(); } - private void evaluateMacro(Context context, ImportedModel model, String macroName) { - if (!context.names().contains(macroName)) { - RankingExpression e = model.macros().get(macroName); - evaluateMacroDependencies(context, model, e.getRoot()); - context.put(macroName, new TensorValue(e.evaluate(context).asTensor())); + private void evaluateFunction(Context context, ImportedModel model, String functionName) { + if (!context.names().contains(functionName)) { + RankingExpression e = model.functions().get(functionName); + evaluateFunctionDependencies(context, model, e.getRoot()); + context.put(functionName, new TensorValue(e.evaluate(context).asTensor())); } } - private void evaluateMacroDependencies(Context context, ImportedModel model, ExpressionNode node) { + private void evaluateFunctionDependencies(Context context, ImportedModel model, ExpressionNode node) { if (node instanceof ReferenceNode) { String name = node.toString(); - if (model.macros().containsKey(name)) { - evaluateMacro(context, model, name); + if (model.functions().containsKey(name)) { + evaluateFunction(context, model, name); } } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode)node).children()) { - evaluateMacroDependencies(context, model, child); + evaluateFunctionDependencies(context, model, child); } } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java index 84e51835458..1f28f0b0129 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencerTestCase.java @@ -27,7 +27,7 @@ public class ConstantDereferencerTestCase { TransformContext context = new TransformContext(constants); assertEquals("1.0 + 2.0 + 3.5", c.transform(new RankingExpression("a + b + c"), context).toString()); - assertEquals("myMacro(1.0,2.0)", c.transform(new RankingExpression("myMacro(a, b)"), context).toString()); + assertEquals("myFunction(1.0,2.0)", c.transform(new RankingExpression("myFunction(a, b)"), context).toString()); } } diff --git a/vespa-documentgen-plugin/etc/complex/video.sd b/vespa-documentgen-plugin/etc/complex/video.sd index dbaba54cc45..0b0298a162c 100644 --- a/vespa-documentgen-plugin/etc/complex/video.sd +++ b/vespa-documentgen-plugin/etc/complex/video.sd @@ -37,7 +37,7 @@ search video { rank-profile default { first-phase { - expression: nativeRank + expression: file:non-existing.expression } } rank-profile rp1 inherits default { diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index acefa3fa461..d27352b8ea7 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -9,9 +9,7 @@ import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchdefinition.UnprocessingSearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.tensor.TensorType; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; @@ -79,10 +77,10 @@ public class DocumentGenMojo extends AbstractMojo { void execute(File sdDir, File outputDir, String packageName) throws MojoFailureException { if ("".equals(packageName)) throw new IllegalArgumentException("You may not use empty package for generated types."); - searches = new HashMap<String, Search>(); - docTypes = new HashMap<String, String>(); - structTypes = new HashMap<String, String>(); - annotationTypes = new HashMap<String, String>(); + searches = new HashMap<>(); + docTypes = new HashMap<>(); + structTypes = new HashMap<>(); + annotationTypes = new HashMap<>(); outputDir.mkdirs(); SearchBuilder builder = buildSearches(sdDir); @@ -110,7 +108,7 @@ public class DocumentGenMojo extends AbstractMojo { public boolean accept(File dir, String name) { return name.endsWith(".sd"); }}); - SearchBuilder builder = new SearchBuilder(); + SearchBuilder builder = new SearchBuilder(true); for (File f : sdFiles) { try { long modTime = f.lastModified(); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java index 237a7d21aef..297576f1bef 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -36,7 +36,7 @@ public class MockedOperationHandler implements OperationHandler { @Override public void update(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { log.append("UPDATE: " + data.getDocumentUpdate().getId()); - log.append(data.getDocumentUpdate().fieldUpdates().toString()); + log.append(data.getDocumentUpdate().getFieldUpdates().toString()); if (data.getDocumentUpdate().getCreateIfNonExistent()) { log.append("[CREATE IF NON EXISTENT IS TRUE]"); } |