diff options
Diffstat (limited to 'config-model')
33 files changed, 280 insertions, 814 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index bd94f67e4a7..d6b916680d8 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -323,7 +323,7 @@ public class DeployState implements ConfigDefinitionStore { closeIgnoreException(reader.getReader()); } } - builder.build(logger); + builder.build(logger, queryProfiles); return SearchDocumentModel.fromBuilderAndNames(builder, names); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index dc59d9cb3e5..dd03cb8b2a7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -5,10 +5,11 @@ */ package com.yahoo.searchdefinition; -import com.yahoo.searchlib.rankingexpression.Reference; - +import java.util.Arrays; +import java.util.List; import java.util.Optional; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Utility methods for query, document and constant rank feature names @@ -19,16 +20,85 @@ public class FeatureNames { private static final Pattern identifierRegexp = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_-]*"); - public static Reference asConstantFeature(String constantName) { - return Reference.simple("constant", quoteIfNecessary(constantName)); + /** + * <p>Returns the given query, document or constant feature in canonical form. + * A feature name consists of a feature type name (query, attribute or constant), + * followed by one argument enclosed in quotes. + * The argument may be an identifier or any string single or double quoted.</p> + * + * <p>Argument string values may not contain comma, single quote nor double quote characters.</p> + * + * <p><i>The canonical form use no quotes for arguments which are identifiers, and double quotes otherwise.</i></p> + * + * <p>Note that the above definition is not true for features in general, which accept any ranking expression + * as argument.</p> + * + * @throws IllegalArgumentException if the feature name is not valid + */ + // Note that this implementation is more general than what is described above: + // It accepts any number of arguments and an optional output + public static String canonicalize(String feature) { + return canonicalizeIfValid(feature).orElseThrow(() -> + new IllegalArgumentException("A feature name must be on the form query(name), attribute(name) or " + + "constant(name), but was '" + feature + "'" + )); + } + + /** + * Canonicalizes the given argument as in canonicalize, but returns empty instead of throwing an exception if + * the argument is not a valid feature + */ + public static Optional<String> canonicalizeIfValid(String feature) { + int startParenthesis = feature.indexOf('('); + if (startParenthesis < 0) + return Optional.empty(); + int endParenthesis = feature.lastIndexOf(')'); + String featureType = feature.substring(0, startParenthesis); + if ( ! ( featureType.equals("query") || featureType.equals("attribute") || featureType.equals("constant"))) + return Optional.empty(); + if (startParenthesis < 1) return Optional.of(feature); // No arguments + if (endParenthesis < startParenthesis) + return Optional.empty(); + String argumentString = feature.substring(startParenthesis + 1, endParenthesis); + List<String> canonicalizedArguments = + Arrays.stream(argumentString.split(",")) + .map(FeatureNames::canonicalizeArgument) + .collect(Collectors.toList()); + return Optional.of(featureType + "(" + + canonicalizedArguments.stream().collect(Collectors.joining(",")) + + feature.substring(endParenthesis)); + } + + /** Canomicalizes a single argument */ + private static String canonicalizeArgument(String argument) { + if (argument.startsWith("'")) { + if ( ! argument.endsWith("'")) + throw new IllegalArgumentException("Feature arguments starting by a single quote " + + "must end by a single quote, but was \"" + argument + "\""); + argument = argument.substring(1, argument.length() - 1); + } + if (argument.startsWith("\"")) { + if ( ! argument.endsWith("\"")) + throw new IllegalArgumentException("Feature arguments starting by a double quote " + + "must end by a double quote, but was '" + argument + "'"); + argument = argument.substring(1, argument.length() - 1); + } + if (identifierRegexp.matcher(argument).matches()) + return argument; + else + return "\"" + argument + "\""; + } + + public static String asConstantFeature(String constantName) { + return canonicalize("constant(\"" + constantName + "\")"); } - public static Reference asAttributeFeature(String attributeName) { - return Reference.simple("attribute", quoteIfNecessary(attributeName)); + public static String asAttributeFeature(String attributeName) { + return canonicalize("attribute(\"" + attributeName + "\")"); } - public static Reference asQueryFeature(String propertyName) { - return Reference.simple("query", quoteIfNecessary(propertyName)); + public static String asQueryFeature(String propertyName) { + return canonicalize("query(\"" + propertyName + "\")"); } /** @@ -36,21 +106,15 @@ public class FeatureNames { * or empty if it is not a valid query, attribute or constant feature name */ public static Optional<String> argumentOf(String feature) { - Optional<Reference> reference = Reference.simple(feature); - if ( ! reference.isPresent()) return Optional.empty(); - if ( ! ( reference.get().name().equals("attribute") || - reference.get().name().equals("constant") || - reference.get().name().equals("query"))) - return Optional.empty(); - - return Optional.of(reference.get().arguments().expressions().get(0).toString()); - } - - private static String quoteIfNecessary(String s) { - if (identifierRegexp.matcher(s).matches()) - return s; - else - return "\"" + s + "\""; + return canonicalizeIfValid(feature).map(f -> { + int startParenthesis = f.indexOf("("); + int endParenthesis = f.indexOf(")"); + String possiblyQuotedArgument = f.substring(startParenthesis + 1, endParenthesis); + if (possiblyQuotedArgument.startsWith("\"")) + return possiblyQuotedArgument.substring(1, possiblyQuotedArgument.length() - 1); + else + return possiblyQuotedArgument; + }); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java deleted file mode 100644 index cf6d90db7fa..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.rule.Arguments; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; -import com.yahoo.searchlib.rankingexpression.rule.NameNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -/** - * A context which only contains type information. - * This returns empty tensor types (double) for unknown features which are not - * query, attribute or constant features, as we do not have information about which such - * features exist (but we know those that exist are doubles). - * - * @author bratseth - */ -public class MapEvaluationTypeContext extends FunctionReferenceContext implements TypeContext<Reference> { - - private final Map<Reference, TensorType> featureTypes = new HashMap<>(); - - public MapEvaluationTypeContext(Collection<ExpressionFunction> functions) { - super(functions); - } - - public MapEvaluationTypeContext(Map<String, ExpressionFunction> functions, - Map<String, String> bindings, - Map<Reference, TensorType> featureTypes) { - super(functions, bindings); - this.featureTypes.putAll(featureTypes); - } - - public void setType(Reference reference, TensorType type) { - featureTypes.put(reference, type); - } - - @Override - public TensorType getType(String reference) { - throw new UnsupportedOperationException("Not able to parse gereral references from string form"); - } - - @Override - public TensorType getType(Reference reference) { - Optional<String> binding = boundIdentifier(reference); - if (binding.isPresent()) { - try { - // This is not pretty, but changing to bind expressions rather - // than their string values requires deeper changes - return new RankingExpression(binding.get()).type(this); - } - catch (ParseException e) { - throw new IllegalArgumentException(e); - } - } - - if (isSimpleFeature(reference)) { - // The argument may be a local identifier bound to the actual value - String argument = simpleArgument(reference.arguments()).get(); - reference = Reference.simple(reference.name(), bindings.getOrDefault(argument, argument)); - return featureTypes.getOrDefault(reference, defaultTypeOf(reference)); - } - - Optional<ExpressionFunction> function = functionInvocation(reference); - if (function.isPresent()) { - return function.get().getBody().type(this.withBindings(bind(function.get().arguments(), reference.arguments()))); - } - - // We do not know what this is - since we do not have complete knowledge abut the match features - // in Java we must assume this is a match feature and return the double type - which is the type of all - // all match features - return TensorType.empty; - } - - /** - * Returns the default type for this simple feature, or nullif it does not have a default - */ - public TensorType defaultTypeOf(Reference reference) { - if ( ! isSimpleFeature(reference)) - throw new IllegalArgumentException("This can only be called for simple references, not " + reference); - if (reference.name().equals("query")) // we do not require all query features to be declared, only non-doubles - return TensorType.empty; - return null; - } - - /** - * Returns the binding if this reference is a simple identifier which is bound in this context. - * Returns empty otherwise. - */ - private Optional<String> boundIdentifier(Reference reference) { - if ( ! reference.arguments().isEmpty()) return Optional.empty(); - if ( reference.output() != null) return Optional.empty(); - return Optional.ofNullable(bindings.get(reference.name())); - } - - /** - * Return whether the reference (discarding the output) is a simple feature - * ("attribute(name)", "constant(name)" or "query(name)"). - * We disregard the output because all outputs under a simple feature have the same type. - */ - private boolean isSimpleFeature(Reference reference) { - Optional<String> argument = simpleArgument(reference.arguments()); - if ( ! argument.isPresent()) return false; - return reference.name().equals("attribute") || - reference.name().equals("constant") || - reference.name().equals("query"); - } - - /** - * If these arguments contains one simple argument string, it is returned. - * Otherwise null is returned. - */ - private Optional<String> simpleArgument(Arguments arguments) { - if (arguments.expressions().size() != 1) return Optional.empty(); - ExpressionNode argument = arguments.expressions().get(0); - - if ( ! (argument instanceof ReferenceNode)) return Optional.empty(); - ReferenceNode refArgument = (ReferenceNode)argument; - - if ( ! refArgument.reference().isIdentifier()) return Optional.empty(); - - return Optional.of(refArgument.getName()); - } - - private Optional<ExpressionFunction> functionInvocation(Reference reference) { - if (reference.output() != null) return Optional.empty(); - ExpressionFunction function = functions().get(reference.name()); - if (function == null) return Optional.empty(); - if (function.arguments().size() != reference.arguments().size()) return Optional.empty(); - return Optional.of(function); - } - - /** Binds the given list of formal arguments to their actual values */ - private Map<String, String> bind(List<String> formalArguments, - Arguments invocationArguments) { - Map<String, String> bindings = new HashMap<>(formalArguments.size()); - for (int i = 0; i < formalArguments.size(); i++) { - String identifier = invocationArguments.expressions().get(i).toString(); - identifier = super.bindings.getOrDefault(identifier, identifier); - bindings.put(formalArguments.get(i), identifier); - } - return bindings; - } - - public Map<Reference, TensorType> featureTypes() { - return Collections.unmodifiableMap(featureTypes); - } - - @Override - public MapEvaluationTypeContext withBindings(Map<String, String> bindings) { - if (bindings.isEmpty() && this.bindings.isEmpty()) return this; - return new MapEvaluationTypeContext(functions(), bindings, featureTypes); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 064897de8dc..bcbc7cc99e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -2,18 +2,24 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.TensorFieldType; import com.yahoo.search.query.ranking.Diversity; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TypeMapContext; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.TensorType; @@ -33,10 +39,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * Represents a rank profile - a named set of ranking settings @@ -360,14 +363,14 @@ public class RankProfile implements Serializable, Cloneable { /** Returns a read-only view of the summary features to use in this profile. This is never null */ public Set<ReferenceNode> getSummaryFeatures() { - if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - if (getInherited() != null) return getInherited().getSummaryFeatures(); + if (summaryFeatures!=null) return Collections.unmodifiableSet(summaryFeatures); + if (getInherited()!=null) return getInherited().getSummaryFeatures(); return Collections.emptySet(); } public void addSummaryFeature(ReferenceNode feature) { - if (summaryFeatures == null) - summaryFeatures = new LinkedHashSet<>(); + if (summaryFeatures==null) + summaryFeatures=new LinkedHashSet<>(); summaryFeatures.add(feature); } @@ -582,11 +585,8 @@ public class RankProfile implements Serializable, Cloneable { } /** - * Will take the parser-set textual ranking expressions and turn into ranking expression objects, - * if not already done + * Will take the parser-set textual ranking expressions and turn into objects */ - // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions - // until this is called. Simplify by parsing them right away. public void parseExpressions() { try { parseRankingExpressions(); @@ -604,23 +604,20 @@ public class RankProfile implements Serializable, Cloneable { for (Map.Entry<String, Macro> e : getMacros().entrySet()) { String macroName = e.getKey(); Macro macro = e.getValue(); - if (macro.getRankingExpression() == null) { - RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); - macro.setRankingExpression(expr); - macro.setTextualExpression(expr.getRoot().toString()); - } + RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); + macro.setRankingExpression(expr); + macro.setTextualExpression(expr.getRoot().toString()); } } /** * Passes ranking expressions on to parser - * * @throws ParseException if either of the ranking expressions could not be parsed */ private void parseRankingExpressions() throws ParseException { - if (getFirstPhaseRankingString() != null && firstPhaseRanking == null) + if (getFirstPhaseRankingString() != null) setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString())); - if (getSecondPhaseRankingString() != null && secondPhaseRanking == null) + if (getSecondPhaseRankingString() != null) setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); } @@ -751,50 +748,37 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public TypeContext typeContext(QueryProfileRegistry queryProfiles) { - MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() - .map(Macro::asExpressionFunction) - .collect(Collectors.toList())); + TypeMapContext context = new TypeMapContext(); - // Add small and large constants, respectively + // Add small constants getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); + // Add large constants getSearch().getRankingConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.getTensorType())); // Add attributes - getSearch().allFields().forEach(field -> addAttributeFeatureTypes(field, context)); - getSearch().allImportedFields().forEach(field -> addAttributeFeatureTypes(field, context)); + for (SDField field : getSearch().allConcreteFields()) { + field.getAttributes().forEach((k, a) -> context.setType(FeatureNames.asAttributeFeature(k), a.tensorType().orElse(TensorType.empty))); + } // Add query features from rank profile types reached from the "default" profile for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { for (FieldDescription field : queryProfileType.declaredFields().values()) { TensorType type = field.getType().asTensorType(); - Optional<Reference> feature = Reference.simple(field.getName()); - if ( ! feature.isPresent() || ! feature.get().name().equals("query")) continue; - - TensorType existingType = context.getType(feature.get()); - if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) + String feature = FeatureNames.asQueryFeature(field.getName()); + TensorType existingType = context.getType(feature); + if (existingType != null) type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> - new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() + + new IllegalArgumentException(queryProfileType + " contains query feature " + feature + " with type " + field.getType().asTensorType() + ", but this is already defined " + - "in another query profile with type " + - context.getType(feature.get()))); - context.setType(feature.get(), type); + "in another query profile with type " + context.getType(feature))); + context.setType(feature, type); } } return context; } - private void addAttributeFeatureTypes(ImmutableSDField field, MapEvaluationTypeContext context) { - field.getAttributes().forEach((k, a) -> { - String name = k; - if (k.equals(field.getBackingField().getName())) // this attribute should take the fields name - name = field.getName(); // switch to that - it is separate for imported fields - context.setType(FeatureNames.asAttributeFeature(name), - a.tensorType().orElse(TensorType.empty)); - }); - } - /** * A rank setting. The identity of a rank setting is its field name and type (not value). * A rank setting is immutable. @@ -926,7 +910,7 @@ public class RankProfile implements Serializable, Cloneable { */ public static class Macro implements Serializable, Cloneable { - private final String name; + private String name=null; private String textualExpression=null; private RankingExpression expression=null; private List<String> formalParams = new ArrayList<>(); @@ -971,7 +955,7 @@ public class RankProfile implements Serializable, Cloneable { return inline && formalParams.size() == 0; // only inline no-arg macros; } - public ExpressionFunction asExpressionFunction() { + public ExpressionFunction toExpressionMacro() { return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java index 7b4d70d85b1..a075b9d00fa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -16,7 +16,8 @@ import java.util.Set; * Having both of these mappings consolidated here make it easier to remove dependencies on these mappings at * run time, since it is essentially only used when building rank profile config at deployment time. * - * TODO: Rank profiles should be stored under its owning Search instance. + * TODO: Reconsider the difference between local and global maps. Right now, the local maps might better be + * served from a different class owned by SearchBuilder. * * @author Ulf Lilleengen */ @@ -30,6 +31,9 @@ public class RankProfileRegistry { /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */ static final Set<String> overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked")); + public RankProfileRegistry() { + } + public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Search search) { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); rankProfileRegistry.addRankProfile(new DefaultRankProfile(search, rankProfileRegistry)); @@ -43,7 +47,7 @@ public class RankProfileRegistry { * @param rankProfile the rank profile to add */ public void addRankProfile(RankProfile rankProfile) { - if ( ! rankProfiles.containsKey(rankProfile.getSearch())) { + if (!rankProfiles.containsKey(rankProfile.getSearch())) { rankProfiles.put(rankProfile.getSearch(), new LinkedHashMap<>()); } checkForDuplicateRankProfile(rankProfile); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index 1ab76afc9c0..f4a0365e36e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -199,7 +199,9 @@ public class Search implements Serializable, ImmutableSearch { @Override public ImmutableSDField getField(String name) { ImmutableSDField field = getConcreteField(name); - if (field != null) return field; + if (field != null) { + return field; + } return allImportedFields() .filter(f -> f.getName().equals(name)) .findFirst() @@ -246,6 +248,8 @@ public class Search implements Serializable, ImmutableSearch { * Returns a list of all the fields of this search definition, that is all fields in all documents, in the documents * they inherit, and all extra fields. The caller receives ownership to the list - subsequent changes to it will not * impact this + * + * @return the list of fields in this searchdefinition */ public List<SDField> allConcreteFields() { List<SDField> allFields = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index e7cd21ac834..762c0fec838 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -18,7 +18,6 @@ import com.yahoo.searchdefinition.parser.TokenMgrError; import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; -import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; @@ -35,6 +34,7 @@ import java.util.List; * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built * search objects using the {@link #getSearch(String)} method. */ +// TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage public class SearchBuilder { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); @@ -154,7 +154,7 @@ public class SearchBuilder { } catch (TokenMgrError e) { throw new ParseException("Unknown symbol: " + e.getMessage()); } catch (ParseException pe) { - throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); + throw new ParseException(stream.formatException(pe.getMessage())); } return importRawSearch(search); } @@ -196,7 +196,11 @@ public class SearchBuilder { * @throws IllegalStateException Thrown if this method has already been called. */ public void build() { - build(new BaseDeployLogger()); + build(new BaseDeployLogger(), new QueryProfiles()); + } + + public void build(DeployLogger logger) { + build(logger, new QueryProfiles()); } /** @@ -205,10 +209,12 @@ public class SearchBuilder { * * @throws IllegalStateException Thrown if this method has already been called. * @param deployLogger The logger to use during build + * @param queryProfiles The query profiles contained in the application this search is part of. */ - public void build(DeployLogger deployLogger) { - if (isBuilt) throw new IllegalStateException("Model already built"); - + public void build(DeployLogger deployLogger, QueryProfiles queryProfiles) { + if (isBuilt) { + throw new IllegalStateException("Searches already built."); + } List<Search> built = new ArrayList<>(); List<SDDocumentType> sdocs = new ArrayList<>(); sdocs.add(SDDocumentType.VESPA_DOCUMENT); @@ -234,7 +240,7 @@ public class SearchBuilder { for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // These two needed for a couple of old unit tests, ideally these are just read from app - process(search, deployLogger, new QueryProfiles(queryProfileRegistry)); + process(search, deployLogger, queryProfiles); built.add(search); } builder.addToModel(searchList); @@ -248,6 +254,8 @@ public class SearchBuilder { /** * Processes and returns the given {@link Search} object. This method has been factored out of the {@link * #build()} method so that subclasses can choose not to build anything. + * + * @param search The object to build. */ protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) { Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles); @@ -344,7 +352,7 @@ public class SearchBuilder { rankProfileRegistry, queryprofileRegistry); builder.importFile(fileName); - builder.build(deployLogger); + builder.build(deployLogger, new QueryProfiles()); return builder; } @@ -360,7 +368,7 @@ public class SearchBuilder { for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } - builder.build(new BaseDeployLogger()); + builder.build(new BaseDeployLogger(), new QueryProfiles()); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java new file mode 100644 index 00000000000..40e9db1413f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * A context which only contains type information. + * + * @author bratseth + */ +public class TypeMapContext implements TypeContext { + + private final Map<String, TensorType> featureTypes = new HashMap<>(); + + public void setType(String name, TensorType type) { + featureTypes.put(FeatureNames.canonicalize(name), type); + } + + @Override + public TensorType getType(String name) { + return featureTypes.get(FeatureNames.canonicalize(name)); + } + + /** Returns an unmodifiable map of the bindings in this */ + public Map<String, TensorType> bindings() { return Collections.unmodifiableMap(featureTypes); } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index b02362154d9..ea02f960800 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -188,7 +188,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (macros.isEmpty()) return; Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>(); for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) { - expressionMacros.put(macro.getKey(), macro.getValue().asExpressionFunction()); + expressionMacros.put(macro.getKey(), macro.getValue().toExpressionMacro()); } Map<String, String> macroProperties = new LinkedHashMap<>(); @@ -223,7 +223,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { // Is the feature a macro? if (context.getFunction(referenceNode.getName()) != null) { context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), - referenceNode.toString(context, null, null)); + referenceNode.toString(context, null, null)); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java index 4502468379f..8b6df1a87db 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java @@ -63,9 +63,6 @@ public class ImmutableImportedSDField implements ImmutableSDField { } @Override - public ImmutableSDField getBackingField() { return importedField.targetField(); } - - @Override public boolean isIndexStructureField() { return importedField.targetField().isIndexStructureField(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java index 70553d4b57c..152690a6f56 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java @@ -19,7 +19,6 @@ import java.util.Map; * @author bjorncs */ public interface ImmutableSDField { - <T extends Expression> boolean containsExpression(Class<T> searchFor); boolean doesAttributing(); @@ -34,12 +33,6 @@ public interface ImmutableSDField { boolean isImportedField(); - /** - * Returns the field backing this - the field itself if this is a regular field, - * and the target field if this is imported. - */ - ImmutableSDField getBackingField(); - default boolean isConcreteField() { return !isImportedField(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 6e7582a98c8..593edc33370 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -209,9 +209,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } @Override - public ImmutableSDField getBackingField() { return this; } - - @Override public boolean doesAttributing() { return containsExpression(AttributeExpression.class); } @@ -626,7 +623,8 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, public RankType getRankType() { return this.rankType; } /** - * Returns the search-time attribute settings of this field or null if none is set. + * <p>Returns the search-time attribute settings of this field + * or null if none is set.</p> * * <p>TODO: Make unmodifiable.</p> */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index f16697b5ba6..2b997aa25f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -208,10 +208,6 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + " but this macro is not present in " + profile); - // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second - // phase and summary features), as it may only resolve correctly given those bindings - // Or, probably better, annotate the macros with type constraints here and verify during general - // type verification TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) throw new IllegalArgumentException("Model refers Placeholder '" + macroName + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index cc634abef01..ee65c9bec02 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; /** - * @author Simon Thoresen Hult + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> */ public class IndexingValues extends Processor { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 061a803cb48..90183848094 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -76,9 +76,8 @@ public class Processing { ImportedFieldsInSummayValidator::new, FastAccessValidator::new, ReservedMacroNames::new, - RankingExpressionTypeValidator::new, - // These should be last. + // These two should be last. IndexingValidation::new, IndexingValues::new); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java deleted file mode 100644 index baacceea667..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Search; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -/** - * Validates the types of all ranking expressions under a search instance: - * Some operators constrain the types of inputs, and first-and second-phase expressions - * must return scalar values. In addition, the existence of all referred attribute, query and constant - * features is ensured. - * - * @author bratseth - */ -public class RankingExpressionTypeValidator extends Processor { - - private final QueryProfileRegistry queryProfiles; - - public RankingExpressionTypeValidator(Search search, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - this.queryProfiles = queryProfiles.getRegistry(); - } - - @Override - public void process() { - for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) { - try { - validate(profile); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("In " + search + ", " + profile, e); - } - } - } - - /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */ - private void validate(RankProfile profile) { - profile.parseExpressions(); - TypeContext context = profile.typeContext(queryProfiles); - profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context)); - ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); - ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - } - - private TensorType ensureValid(RankingExpression expression, String expressionDescription, TypeContext context) { - if (expression == null) return null; - return ensureValid(expression.getRoot(), expressionDescription, context); - } - - private TensorType ensureValid(ExpressionNode expression, String expressionDescription, TypeContext context) { - TensorType type; - try { - type = expression.type(context); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("The " + expressionDescription + " is invalid", e); - } - if (type == null) // Not expected to happen - throw new IllegalStateException("Could not determine the type produced by " + expressionDescription); - return type; - } - - private void ensureValidDouble(RankingExpression expression, String expressionDescription, TypeContext context) { - if (expression == null) return; - TensorType type = ensureValid(expression, expressionDescription, context); - if ( ! type.equals(TensorType.empty)) - throw new IllegalArgumentException("The " + expressionDescription + " must produce a double " + - "(a tensor with no dimensions), but produces " + type); - } - -} diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg index f5652c31d2a..e890b75770b 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg @@ -24,7 +24,7 @@ rankprofile[0].fef.property[10].value "4" rankprofile[0].fef.property[11].name "vespa.dump.feature" rankprofile[0].fef.property[11].value "attribute(foo1).out" rankprofile[0].fef.property[12].name "vespa.dump.feature" -rankprofile[0].fef.property[12].value "attribute(bar1)" +rankprofile[0].fef.property[12].value "attribute(bar1.out)" rankprofile[0].fef.property[13].name "vespa.dump.feature" rankprofile[0].fef.property[13].value "attribute(foo2).out" rankprofile[0].fef.property[14].name "vespa.dump.feature" @@ -64,7 +64,7 @@ rankprofile[2].fef.property[2].value "10 + feature(arg1).out.out" rankprofile[2].fef.property[3].name "vespa.summary.feature" rankprofile[2].fef.property[3].value "attribute(foo1).out" rankprofile[2].fef.property[4].name "vespa.summary.feature" -rankprofile[2].fef.property[4].value "attribute(bar1)" +rankprofile[2].fef.property[4].value "attribute(bar1.out)" rankprofile[2].fef.property[5].name "vespa.summary.feature" rankprofile[2].fef.property[5].value "attribute(foo2).out" rankprofile[2].fef.property[6].name "vespa.summary.feature" diff --git a/config-model/src/test/derived/rankexpression/rankexpression.sd b/config-model/src/test/derived/rankexpression/rankexpression.sd index d3e0057cfe1..8ed1f2bab4c 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankexpression/rankexpression.sd @@ -5,10 +5,12 @@ search rankexpression { field artist type string { indexing: summary | index + # index-to: artist, default } field title type string { indexing: summary | index + # index-to: title, default } field surl type string { @@ -19,38 +21,6 @@ search rankexpression { indexing: summary | attribute } - field foo1 type int { - indexing: attribute - } - - field foo2 type int { - indexing: attribute - } - - field foo3 type int { - indexing: attribute - } - - field foo4 type int { - indexing: attribute - } - - field bar1 type int { - indexing: attribute - } - - field bar2 type int { - indexing: attribute - } - - field bar3 type int { - indexing: attribute - } - - field bar4 type int { - indexing: attribute - } - } rank-profile default { @@ -63,7 +33,7 @@ search rankexpression { expression: if(3>2,4,2) rerank-count: 10 } - rank-features: attribute(foo1).out attribute(bar1) + rank-features: attribute(foo1).out attribute(bar1.out) rank-features { attribute(foo2).out attribute(bar2).out } rank-features { attribute(foo3).out attribute(bar3).out } @@ -95,7 +65,7 @@ search rankexpression { file:rankexpression } } - summary-features: attribute(foo1).out attribute(bar1) + summary-features: attribute(foo1).out attribute(bar1.out) summary-features { attribute(foo2).out attribute(bar2).out } summary-features { attribute(foo3).out attribute(bar3).out } diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankexpression/summary.cfg index 9752a9f55e3..00df2e87144 100644 --- a/config-model/src/test/derived/rankexpression/summary.cfg +++ b/config-model/src/test/derived/rankexpression/summary.cfg @@ -15,25 +15,9 @@ classes[0].fields[5].name "summaryfeatures" classes[0].fields[5].type "featuredata" classes[0].fields[6].name "documentid" classes[0].fields[6].type "longstring" -classes[1].id 1736696699 +classes[1].id 1787488393 classes[1].name "attributeprefetch" classes[1].fields[0].name "year" -classes[].fields[].type "integer" -classes[].fields[].name "foo1" -classes[].fields[].type "integer" -classes[].fields[].name "foo2" -classes[].fields[].type "integer" -classes[].fields[].name "foo3" -classes[].fields[].type "integer" -classes[].fields[].name "foo4" -classes[].fields[].type "integer" -classes[].fields[].name "bar1" -classes[].fields[].type "integer" -classes[].fields[].name "bar2" -classes[].fields[].type "integer" -classes[].fields[].name "bar3" -classes[].fields[].type "integer" -classes[].fields[].name "bar4" classes[1].fields[0].type "integer" classes[1].fields[1].name "rankfeatures" classes[1].fields[1].type "featuredata" diff --git a/config-model/src/test/derived/rankexpression/summarymap.cfg b/config-model/src/test/derived/rankexpression/summarymap.cfg index 21e6cdf346f..c810f7282ba 100644 --- a/config-model/src/test/derived/rankexpression/summarymap.cfg +++ b/config-model/src/test/derived/rankexpression/summarymap.cfg @@ -7,28 +7,4 @@ override[1].command "rankfeatures" override[1].arguments "" override[2].field "summaryfeatures" override[2].command "summaryfeatures" -override[2].arguments "" -override[].field "foo1" -override[].command "attribute" -override[].arguments "foo1" -override[].field "foo2" -override[].command "attribute" -override[].arguments "foo2" -override[].field "foo3" -override[].command "attribute" -override[].arguments "foo3" -override[].field "foo4" -override[].command "attribute" -override[].arguments "foo4" -override[].field "bar1" -override[].command "attribute" -override[].arguments "bar1" -override[].field "bar2" -override[].command "attribute" -override[].arguments "bar2" -override[].field "bar3" -override[].command "attribute" -override[].arguments "bar3" -override[].field "bar4" -override[].command "attribute" -override[].arguments "bar4"
\ No newline at end of file +override[2].arguments ""
\ No newline at end of file diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index b6ad5372c05..2b231e0cda2 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -35,7 +35,7 @@ rankprofile[3].name "profile2" rankprofile[3].fef.property[0].name "vespa.rank.firstphase" rankprofile[3].fef.property[0].value "rankingExpression(firstphase)" rankprofile[3].fef.property[1].name "rankingExpression(firstphase).rankingScript" -rankprofile[3].fef.property[1].value "reduce(reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" +rankprofile[3].fef.property[1].value "reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x)" rankprofile[3].fef.property[2].name "vespa.type.attribute.f2" rankprofile[3].fef.property[2].value "tensor(x[2],y[])" rankprofile[3].fef.property[3].name "vespa.type.attribute.f3" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index 3d64f6b807e..a6a9a98db3a 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -28,7 +28,7 @@ search tensor { rank-profile profile2 { first-phase { - expression: sum(matmul(attribute(f4), diag(x[2],y[2],z[3]), x)) + expression: matmul(attribute(f4), diag(x[2],y[2],z[3]), x) } } diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd index 28959edbc09..40f9e73f35a 100644 --- a/config-model/src/test/examples/rankpropvars.sd +++ b/config-model/src/test/examples/rankpropvars.sd @@ -18,8 +18,8 @@ first-phase { second-phase { expression { if (attribute(artist) == query(testvar1), - 0.0 * fieldMatch(title) + 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist), - 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) + 0.0 * fieldMatch(title) + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist), + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) } } @@ -42,8 +42,8 @@ first-phase { second-phase { expression { if (attribute(artist) == query(testvar1), - 0.0 * fieldMatch(title) + 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist), - 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) + 0.0 * fieldMatch(title) + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist), + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) } } } diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index 96b0fa98098..4fda7f5039e 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -116,7 +116,7 @@ search simple { first-phase { keep-rank-count:200 rank-score-drop-limit: -13.0 - expression: attribute(popularity) + expression: attribute(year) } second-phase { rerank-count: 99 diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java index aa01070d296..1f60ad870ec 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java @@ -18,6 +18,17 @@ import static org.junit.Assert.assertFalse; public class FeatureNamesTestCase { @Test + public void testCanonicalization() { + assertFalse(FeatureNames.canonicalizeIfValid("foo").isPresent()); + assertEquals("query(bar)", FeatureNames.canonicalize("query(bar)")); + assertEquals("query(bar)", FeatureNames.canonicalize("query('bar')")); + assertEquals("constant(bar)", FeatureNames.canonicalize("constant(\"bar\")")); + assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query(ba.r)")); + assertEquals("query(\"ba.r\")", FeatureNames.canonicalize("query('ba.r')")); + assertEquals("attribute(\"ba.r\")", FeatureNames.canonicalize("attribute(\"ba.r\")")); + } + + @Test public void testArgument() { assertFalse(FeatureNames.argumentOf("foo(bar)").isPresent()); assertFalse(FeatureNames.argumentOf("foo(bar.baz)").isPresent()); @@ -31,20 +42,17 @@ public class FeatureNamesTestCase { @Test public void testConstantFeature() { - assertEquals("constant(\"foo/bar\")", - FeatureNames.asConstantFeature("foo/bar").toString()); + assertEquals("constant(\"foo/bar\")", FeatureNames.asConstantFeature("foo/bar")); } @Test public void testAttributeFeature() { - assertEquals("attribute(foo)", - FeatureNames.asAttributeFeature("foo").toString()); + assertEquals("attribute(foo)", FeatureNames.asAttributeFeature("foo")); } @Test public void testQueryFeature() { - assertEquals("query(\"foo.bar\")", - FeatureNames.asQueryFeature("foo.bar").toString()); + assertEquals("query(\"foo.bar\")", FeatureNames.asQueryFeature("foo.bar")); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 11093d9f008..442c8bd41bd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -135,13 +135,13 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { @Test public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SearchBuilder builder = new SearchBuilder(registry, setupQueryProfileTypes()); + SearchBuilder builder = new SearchBuilder(registry); builder.importString("search test {\n" + " document test { } \n" + " rank-profile p1 {}\n" + " rank-profile p2 {}\n" + "}"); - builder.build(new BaseDeployLogger()); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); Search search = builder.getSearch(); assertEquals(4, registry.allRankProfiles().size()); @@ -151,7 +151,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { assertQueryFeatureTypeSettings(registry.getRankProfile(search, "p2"), search); } - private static QueryProfileRegistry setupQueryProfileTypes() { + private static QueryProfiles setupQueryProfileTypes() { QueryProfileRegistry registry = new QueryProfileRegistry(); QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); @@ -164,7 +164,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { type.addField(new FieldDescription("ranking.features.query(numeric)", FieldType.fromString("integer", typeRegistry)), typeRegistry); typeRegistry.register(type); - return registry; + return new QueryProfiles(registry); } private static void assertQueryFeatureTypeSettings(RankProfile profile, Search search) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 82b9f5ac043..e94880e61c7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -207,9 +207,6 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.importString( "search test {\n" + " document test { \n" + - " field rating_yelp type int {" + - " indexing: attribute" + - " }" + " }\n" + " \n" + " rank-profile test {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index ed1b00e2875..5100ac15c40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -2,10 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; -import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -152,12 +149,11 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase censorBindingHash(testRankProperties.get(4).toString())); } + @Test public void testNeuralNetworkSetup() throws ParseException { - // Note: the type assigned to query profile and constant tensors here is not the correct type RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(x[])"); - SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( "search test {\n" + " document test { \n" + @@ -180,28 +176,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " expression: sum(final_layer)\n" + " }\n" + " }\n" + - " constant W_hidden {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + - " constant b_input {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + - " constant W_final {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + - " constant b_final {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + + "\n" + "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(queryProfiles); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); List<Pair<String, String>> testRankProperties = new RawRankProfile(test, - queryProfiles, + new QueryProfileRegistry(), new AttributeFields(s)).configProperties(); assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(0).toString()); @@ -217,17 +198,6 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase testRankProperties.get(5).toString()); } - private QueryProfileRegistry queryProfileWith(String field, String type) { - QueryProfileType queryProfileType = new QueryProfileType("root"); - queryProfileType.addField(new FieldDescription(field, type)); - QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); - queryProfileRegistry.getTypeRegistry().register(queryProfileType); - QueryProfile profile = new QueryProfile("default"); - profile.setType(queryProfileType); - queryProfileRegistry.register(profile); - return queryProfileRegistry; - } - private String censorBindingHash(String s) { StringBuilder b = new StringBuilder(); boolean areInHash = false; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 0ce6129ef7f..800697b3430 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -38,8 +38,7 @@ class RankProfileSearchFixture { RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, String rankProfiles, String constant, String field) throws ParseException { - this.queryProfileRegistry = queryProfileRegistry; - SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, queryProfileRegistry); + SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, new QueryProfileRegistry()); String sdContent = "search test {\n" + " " + (constant != null ? constant : "") + "\n" + " document test {\n" + @@ -51,6 +50,7 @@ class RankProfileSearchFixture { builder.importString(sdContent); builder.build(); search = builder.getSearch(); + this.queryProfileRegistry = queryProfileRegistry; } public void assertFirstPhaseExpression(String expExpression, String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java deleted file mode 100644 index 5f5b40e545f..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.tensor.TensorType; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; - -import java.util.Map; -import java.util.stream.Collectors; - -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -public class RankingExpressionTypeValidatorTestCase { - - @Test - public void tensorFirstPhaseMustProduceDouble() throws Exception { - try { - SearchBuilder builder = new SearchBuilder(); - builder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: attribute(a)", - " }", - " }", - "}" - )); - builder.build(); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void tensorSecondPhaseMustProduceDouble() throws Exception { - try { - SearchBuilder builder = new SearchBuilder(); - builder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(attribute(a))", - " }", - " second-phase {", - " expression: attribute(a)", - " }", - " }", - "}" - )); - builder.build(); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception { - try { - SearchBuilder searchBuilder = new SearchBuilder(); - searchBuilder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(if(1>0, attribute(a), attribute(b)))", - " }", - " }", - "}" - )); - searchBuilder.build(); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[],y[]) while the 'false' type is tensor(z[10])", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void testMacroInvocationTypes() throws Exception { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SearchBuilder builder = new SearchBuilder(rankProfileRegistry); - builder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " macro macro1(attribute_to_use) {", - " expression: attribute(attribute_to_use)", - " }", - " summary-features {", - " macro1(a)", - " macro1(b)", - " }", - " }", - "}" - )); - builder.build(); - RankProfile profile = - builder.getRankProfileRegistry().getRankProfile(builder.getSearch(), "my_rank_profile"); - assertEquals(TensorType.fromSpec("tensor(x[],y[])"), - summaryFeatures(profile).get("macro1(a)").type(profile.typeContext(builder.getQueryProfileRegistry()))); - assertEquals(TensorType.fromSpec("tensor(z[10])"), - summaryFeatures(profile).get("macro1(b)").type(profile.typeContext(builder.getQueryProfileRegistry()))); - } - - @Test - public void testTensorMacroInvocationTypes_Nested() throws Exception { - SearchBuilder builder = new SearchBuilder(); - builder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " macro return_a() {", - " expression: return_first(attribute(a), attribute(b))", - " }", - " macro return_b() {", - " expression: return_second(attribute(a), attribute(b))", - " }", - " macro return_first(e1, e2) {", - " expression: e1", - " }", - " macro return_second(e1, e2) {", - " expression: return_first(e2, e1)", - " }", - " summary-features {", - " return_a", - " return_b", - " }", - " }", - "}" - )); - builder.build(); - RankProfile profile = - builder.getRankProfileRegistry().getRankProfile(builder.getSearch(), "my_rank_profile"); - assertEquals(TensorType.fromSpec("tensor(x[],y[])"), - summaryFeatures(profile).get("return_a").type(profile.typeContext(builder.getQueryProfileRegistry()))); - assertEquals(TensorType.fromSpec("tensor(z[10])"), - summaryFeatures(profile).get("return_b").type(profile.typeContext(builder.getQueryProfileRegistry()))); - } - - @Test - public void importedFieldsAreAvailable() throws Exception { - SearchBuilder builder = new SearchBuilder(); - builder.importString(joinLines( - "search parent {", - " document parent {", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " }", - "}" - )); - builder.importString(joinLines( - "search child {", - " document child { ", - " field ref type reference<parent> {", - "indexing: attribute | summary", - " }", - " }", - " import field ref.a as imported_a {}", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(attribute(imported_a))", - " }", - " }", - "}" - )); - builder.build(); - } - - @Test - public void undeclaredQueryFeaturesAreAccepted() throws Exception { - SearchBuilder builder = new SearchBuilder(); - builder.importString(joinLines( - "search test {", - " document test { ", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: query(foo)", - " }", - " }", - "}" - )); - builder.build(); - } - - private Map<String, ReferenceNode> summaryFeatures(RankProfile profile) { - return profile.getSummaryFeatures().stream().collect(Collectors.toMap(f -> f.toString(), f -> f)); - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 96795d2b08f..4693ac5cf4d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -73,7 +73,7 @@ public class RankingExpressionWithTensorFlowTestCase { public void testTensorFlowReferenceWithQueryFeature() { String queryProfile = "<query-profile id='default' type='root'/>"; String queryProfileType = "<query-profile-type id='root'>" + - " <field name='query(mytensor)' type='tensor(d0[3],d1[784])'/>" + + " <field name='mytensor' type='tensor(d0[3],d1[784])'/>" + "</query-profile-type>"; StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, @@ -107,7 +107,7 @@ public class RankingExpressionWithTensorFlowTestCase { public void testTensorFlowReferenceWithFeatureCombination() { String queryProfile = "<query-profile id='default' type='root'/>"; String queryProfileType = "<query-profile-type id='root'>" + - " <field name='query(mytensor)' type='tensor(d0[3],d1[784],d2[10])'/>" + + " <field name='mytensor' type='tensor(d0[3],d1[784],d2[10])'/>" + "</query-profile-type>"; StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 054c9220225..b001db69768 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -17,129 +17,98 @@ import com.yahoo.searchdefinition.SearchDefinitionTestCase; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; import java.util.List; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class TensorTransformTestCase extends SearchDefinitionTestCase { @Test public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { - assertTransformedExpression("max(1.0,2.0)", - "max(1.0,2.0)"); - assertTransformedExpression("min(attribute(double_field),x)", - "min(attribute(double_field),x)"); - assertTransformedExpression("max(attribute(double_field),attribute(double_array_field))", - "max(attribute(double_field),attribute(double_array_field))"); - assertTransformedExpression("min(attribute(tensor_field_1),attribute(double_field))", - "min(attribute(tensor_field_1),attribute(double_field))"); - assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)", - "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)"); - assertTransformedExpression("min(constant(test_constant_tensor),1.0)", - "min(test_constant_tensor,1.0)"); - assertTransformedExpression("max(constant(base_constant_tensor),1.0)", - "max(base_constant_tensor,1.0)"); - assertTransformedExpression("min(constant(file_constant_tensor),1.0)", - "min(constant(file_constant_tensor),1.0)"); - assertTransformedExpression("max(query(q),1.0)", - "max(query(q),1.0)"); - assertTransformedExpression("max(query(n),1.0)", - "max(query(n),1.0)"); + assertContainsExpression("max(1.0,2.0)", "max(1.0,2.0)"); + assertContainsExpression("min(attribute(double_field),x)", "min(attribute(double_field),x)"); + assertContainsExpression("max(attribute(double_field),attribute(double_array_field))", "max(attribute(double_field),attribute(double_array_field))"); + assertContainsExpression("min(attribute(tensor_field_1),attribute(double_field))", "min(attribute(tensor_field_1),attribute(double_field))"); + assertContainsExpression("max(attribute(tensor_field_1),attribute(tensor_field_2))", "max(attribute(tensor_field_1),attribute(tensor_field_2))"); + assertContainsExpression("min(test_constant_tensor,1.0)", "min(constant(test_constant_tensor),1.0)"); + assertContainsExpression("max(base_constant_tensor,1.0)", "max(constant(base_constant_tensor),1.0)"); + assertContainsExpression("min(constant(file_constant_tensor),1.0)", "min(constant(file_constant_tensor),1.0)"); + assertContainsExpression("max(query(q),1.0)", "max(query(q),1.0)"); + assertContainsExpression("max(query(n),1.0)", "max(query(n),1.0)"); } @Test public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { - assertTransformedExpression("reduce(attribute(tensor_field_1),max,x)", - "max(attribute(tensor_field_1),x)"); - assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)", - "1 + max(attribute(tensor_field_1),x)"); - assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)", - "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)"); - assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)", - "max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)"); - assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)", - "max(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),x)"); - assertTransformedExpression("max(reduce(attribute(tensor_field_1),max,x),x)", - "max(max(attribute(tensor_field_1),x),x)"); // will result in deploy error. - assertTransformedExpression("reduce(reduce(attribute(tensor_field_2),max,x),max,y)", - "max(max(attribute(tensor_field_2),x),y)"); + assertContainsExpression("max(attribute(tensor_field_1),x)", "reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("1 + max(attribute(tensor_field_1),x)", "1+reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)", "if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)"); + assertContainsExpression("max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(max(attribute(tensor_field_1),x),x)", "max(reduce(attribute(tensor_field_1),max,x),x)"); // will result in deploy error. + assertContainsExpression("max(max(attribute(tensor_field_2),x),y)", "reduce(reduce(attribute(tensor_field_2),max,x),max,y)"); } @Test public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { - assertTransformedExpression("reduce(constant(test_constant_tensor),max,x)", - "max(test_constant_tensor,x)"); - assertTransformedExpression("reduce(constant(base_constant_tensor),max,x)", - "max(base_constant_tensor,x)"); - assertTransformedExpression("reduce(constant(file_constant_tensor),min,x)", - "min(constant(file_constant_tensor),x)"); + assertContainsExpression("max(test_constant_tensor,x)", "reduce(constant(test_constant_tensor),max,x)"); + assertContainsExpression("max(base_constant_tensor,x)", "reduce(constant(base_constant_tensor),max,x)"); + assertContainsExpression("min(constant(file_constant_tensor),x)", "reduce(constant(file_constant_tensor),min,x)"); } @Test public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { - assertTransformedExpression("reduce(attribute(double_field)+attribute(tensor_field_1),min,x)", - "min(attribute(double_field) + attribute(tensor_field_1),x)"); - assertTransformedExpression("reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)", - "min(attribute(tensor_field_1) * attribute(tensor_field_2),x)"); - assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)", - "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)"); - assertTransformedExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", - "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) - assertTransformedExpression("reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)", - "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + assertContainsExpression("min(attribute(double_field) + attribute(tensor_field_1),x)", "reduce(attribute(double_field)+attribute(tensor_field_1),min,x)"); + assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)"); + assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"); + assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) + assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)"); } @Test public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { - assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)", - "max(tensorFromLabels(attribute(double_array_field)),double_array_field)"); - assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field),x),max,x)", - "max(tensorFromLabels(attribute(double_array_field),x),x)"); - assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)", - "max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)"); - assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)", - "max(tensorFromWeightedSet(attribute(weightedset_field),x),x)"); + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field)),double_array_field)", "reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)"); + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field),x),x)", "reduce(tensorFromLabels(attribute(double_array_field),x),max,x)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)", "reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field),x),x)", "reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)"); } @Test public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { - assertTransformedExpression("reduce(query(q),max,x)", "max(query(q),x)"); - assertTransformedExpression("max(query(n),x)", "max(query(n),x)"); + assertContainsExpression("max(query(q),x)", "reduce(query(q),max,x)"); + assertContainsExpression("max(query(n),x)", "max(query(n),x)"); } @Test public void requireThatMaxAndMinWithTensoresReturnedFromMacrosAreReplaced() throws ParseException { - assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", - "max(returns_tensor,x)"); - assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", - "max(wraps_returns_tensor,x)"); - assertTransformedExpression("reduce(rankingExpression(tensor_inheriting),max,x)", - "max(tensor_inheriting,x)"); - assertTransformedExpression("reduce(rankingExpression(returns_tensor_with_arg@),max,x)", - "max(returns_tensor_with_arg(attribute(tensor_field_1)),x)"); + assertContainsExpression("max(returns_tensor,x)", "reduce(rankingExpression(returns_tensor),max,x)"); + assertContainsExpression("max(wraps_returns_tensor,x)", "reduce(rankingExpression(wraps_returns_tensor),max,x)"); + assertContainsExpression("max(tensor_inheriting,x)", "reduce(rankingExpression(tensor_inheriting),max,x)"); + assertContainsExpression("max(returns_tensor_with_arg(attribute(tensor_field_1)),x)", "reduce(rankingExpression(returns_tensor_with_arg@),max,x)"); } - private void assertTransformedExpression(String expected, String original) throws ParseException { - for (Pair<String, String> rankPropertyExpression : buildSearch(original)) { + private void assertContainsExpression(String expr, String transformedExpression) throws ParseException { + assertTrue("Expected expression '" + transformedExpression + "' found", + containsExpression(expr, transformedExpression)); + } + + private boolean containsExpression(String expr, String transformedExpression) throws ParseException { + for (Pair<String, String> rankPropertyExpression : buildSearch(expr)) { String rankProperty = rankPropertyExpression.getFirst(); if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) { String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); - assertEquals(expected, rankExpression); - return; + return rankExpression.equals(transformedExpression); } } - fail("No 'rankingExpression(firstphase).rankingScript' property produced"); + return false; } private List<Pair<String, String>> buildSearch(String expression) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = setupQueryProfileTypes(); - SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( "search test {\n" + " document test { \n" + @@ -198,16 +167,16 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(new BaseDeployLogger()); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(queryProfiles); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); List<Pair<String, String>> testRankProperties = new RawRankProfile(test, - queryProfiles, + new QueryProfileRegistry(), new AttributeFields(s)).configProperties(); return testRankProperties; } - private static QueryProfileRegistry setupQueryProfileTypes() { + private static QueryProfiles setupQueryProfileTypes() { QueryProfileRegistry registry = new QueryProfileRegistry(); QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); @@ -216,7 +185,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { type.addField(new FieldDescription("ranking.features.query(n)", FieldType.fromString("integer", typeRegistry)), typeRegistry); typeRegistry.register(type); - return registry; + return new QueryProfiles(registry); } private String censorBindingHash(String s) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index aeddd05209f..a4ab5ebdb5e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; * Test validation of changes between a current and next document type used in a document database. * * @author toregge + * @since 2014-11-25 */ public class DocumentTypeChangeValidatorTest { |