diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-09-25 10:20:23 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-09-25 10:20:23 -0700 |
commit | 0246064bbfb9657515f516e2fea12d593cd13016 (patch) | |
tree | 5e2727aaf1eef5ef5043b399508cfca99b2e33c2 | |
parent | f627463a8100090ec109d27c3aeb439a3395a34f (diff) | |
parent | dfe56b565f6728a6c537867d28e388cd04718071 (diff) |
Merge pull request #7094 from vespa-engine/revert-7070-bratseth/rank-type-information-2
Revert "Bratseth/rank type information 2"
35 files changed, 241 insertions, 452 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 937151c0d3a..16e494c2db1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -449,7 +449,7 @@ public class RankProfile implements Serializable, Cloneable { addRankProperty(new RankProperty(name, parameter)); } - private void addRankProperty(RankProperty rankProperty) { + public void addRankProperty(RankProperty rankProperty) { // Just the usual multimap semantics here List<RankProperty> properties = rankProperties.get(rankProperty.getName()); if (properties == null) { @@ -541,14 +541,15 @@ public class RankProfile implements Serializable, Cloneable { /** Returns an unmodifiable view of the functions in this */ public Map<String, RankingExpressionFunction> getFunctions() { - if (functions.isEmpty() && getInherited() == null) return Collections.emptyMap(); - if (functions.isEmpty()) return getInherited().getFunctions(); + if (functions.size() == 0 && getInherited() == null) return Collections.emptyMap(); + if (functions.size() == 0) return getInherited().getFunctions(); if (getInherited() == null) return Collections.unmodifiableMap(functions); // Neither is null Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions()); allFunctions.putAll(functions); return Collections.unmodifiableMap(allFunctions); + } public int getKeepRankCount() { @@ -694,7 +695,7 @@ public class RankProfile implements Serializable, Cloneable { for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms); - compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withExpression(compiled)); + compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withBody(compiled)); } return compiledFunctions; } @@ -897,7 +898,7 @@ public class RankProfile implements Serializable, Cloneable { /** A function in a rank profile */ public static class RankingExpressionFunction { - private ExpressionFunction function; + private final ExpressionFunction function; /** True if this should be inlined into calling expressions. Useful for very cheap functions. */ private final boolean inline; @@ -907,17 +908,13 @@ public class RankProfile implements Serializable, Cloneable { this.inline = inline; } - public void setReturnType(TensorType type) { - this.function = function.withReturnType(type); - } - public ExpressionFunction function() { return function; } public boolean inline() { return inline && function.arguments().isEmpty(); // only inline no-arg functions; } - public RankingExpressionFunction withExpression(RankingExpression expression) { + public RankingExpressionFunction withBody(RankingExpression expression) { return new RankingExpressionFunction(function.withBody(expression), inline); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index a988da9664e..f42d5de21e8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -59,6 +59,9 @@ public class Search implements Serializable, ImmutableSearch { // Field sets private FieldSets fieldSets = new FieldSets(); + // Whether or not this object has been processed. + private boolean processed; + // The unique name of this search definition. private String name; @@ -582,6 +585,17 @@ public class Search implements Serializable, ImmutableSearch { return false; } + public void process() { + if (processed) { + throw new IllegalStateException("Search '" + getName() + "' already processed."); + } + processed = true; + } + + public boolean isProcessed() { + return processed; + } + /** * The field set settings for this search * diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 151ad02a3fa..3c2ebc058ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -188,6 +188,10 @@ public class SearchBuilder { throw new IllegalArgumentException("Search has no name."); } String rawName = rawSearch.getName(); + if (rawSearch.isProcessed()) { + throw new IllegalArgumentException("A search definition with a search section called '" + rawName + + "' has already been processed."); + } for (Search search : searchList) { if (rawName.equals(search.getName())) { throw new IllegalArgumentException("A search definition with a search section called '" + rawName + @@ -243,7 +247,8 @@ public class SearchBuilder { DocumentModelBuilder builder = new DocumentModelBuilder(model); for (Search search : new SearchOrderer().order(searchList)) { - new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list? + new FieldOperationApplierForSearch().process(search); + // These two needed for a couple of old unit tests, ideally these are just read from app process(search, deployLogger, new QueryProfiles(queryProfileRegistry), validate); built.add(search); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 7c2d9a3b0ad..9a00ee5bbd0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -74,11 +74,21 @@ public class DerivedConfiguration { QueryProfileRegistry queryProfiles, ImportedModels importedModels) { Validator.ensureNotNull("Search definition", search); + if ( ! search.isProcessed()) { + throw new IllegalArgumentException("Search '" + search.getName() + "' not processed."); + } this.search = search; if ( ! search.isDocumentsOnly()) { streamingFields = new VsmFields(search); streamingSummary = new VsmSummary(search); } + if (abstractSearchList != null) { + for (Search abstractSearch : abstractSearchList) { + if (!abstractSearch.isProcessed()) { + throw new IllegalArgumentException("Search '" + search.getName() + "' not processed."); + } + } + } if ( ! search.isDocumentsOnly()) { attributeFields = new AttributeFields(search); summaries = new Summaries(search, deployLogger); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 279b5334187..c041d5c6a89 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -13,7 +13,6 @@ import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; -import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; import java.nio.charset.Charset; @@ -24,7 +23,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; /** * A rank profile derived from a search definition, containing exactly the features available natively in the server @@ -182,39 +180,32 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { if (functions.isEmpty()) return; - - List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); + Map<String, ExpressionFunction> expressionFunctions = new LinkedHashMap<>(); + for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : functions.entrySet()) { + expressionFunctions.put(function.getKey(), function.getValue().function()); + } Map<String, String> functionProperties = new LinkedHashMap<>(); - functionProperties.putAll(deriveFunctionProperties(functions, functionExpressions)); - + functionProperties.putAll(deriveFunctionProperties(expressionFunctions)); if (firstPhaseRanking != null) { - functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); + functionProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } if (secondPhaseRanking != null) { - functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); + functionProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values()))); } for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + SerializationContext context = new SerializationContext(expressionFunctions.values(), null, functionProperties); replaceFunctionSummaryFeatures(context); } - private Map<String, String> deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, - List<ExpressionFunction> functionExpressions) { - SerializationContext context = new SerializationContext(functionExpressions); - for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { - String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); - context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); - - for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) - context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); - if (e.getValue().function().returnType().isPresent()) - context.addFunctionTypeSerialization(e.getKey(), e.getValue().function().returnType().get()); - else if (e.getValue().function().arguments().isEmpty()) - throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved"); + private Map<String, String> deriveFunctionProperties(Map<String, ExpressionFunction> functions) { + SerializationContext context = new SerializationContext(functions); + for (Map.Entry<String, ExpressionFunction> e : functions.entrySet()) { + String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression); } return context.serializedFunctions(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 15d295736c1..8c8c32389e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -7,8 +7,10 @@ import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor; import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; /** * Executor of processors. This defines the right order of processor execution. @@ -73,20 +75,12 @@ public class Processing { ReferenceFieldsProcessor::new, FastAccessValidator::new, ReservedFunctionNames::new, - RankingExpressionTypeResolver::new, + RankingExpressionTypeValidator::new, // These should be last: IndexingValidation::new, IndexingValues::new); } - /** Processors of rank profiles only (those who tolerate and so something useful when the search field is null) */ - private Collection<ProcessorFactory> rankProfileProcessors() { - return Arrays.asList( - RankProfileTypeSettingsProcessor::new, - ReservedFunctionNames::new, - RankingExpressionTypeResolver::new); - } - /** * Runs all search processors on the given {@link Search} object. These will modify the search object, <b>possibly * exchanging it with another</b>, as well as its document types. @@ -99,26 +93,12 @@ public class Processing { public void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { Collection<ProcessorFactory> factories = processors(); + search.process(); factories.stream() .map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles)) .forEach(processor -> processor.process(validate, documentsOnly)); } - /** - * Runs rank profiles processors only. - * - * @param deployLogger The log to log messages and warnings for application deployment to - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} - * @param queryProfiles The query profiles contained in the application this search is part of. - */ - public void processRankProfiles(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) { - Collection<ProcessorFactory> factories = rankProfileProcessors(); - factories.stream() - .map(factory -> factory.create(null, deployLogger, rankProfileRegistry, queryProfiles)) - .forEach(processor -> processor.process(validate, documentsOnly)); - } - @FunctionalInterface public interface ProcessorFactory { Processor create(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index 4c8b5910b78..102d1910360 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -7,44 +7,39 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.vespa.model.container.search.QueryProfiles; -import java.util.Map; - /** - * Resolves and assigns types to all functions in a ranking expression, and - * validates the types of all ranking expressions under a search instance: + * Validates the types of all ranking expressions under a search instance: * Some operators constrain the types of inputs, and first-and second-phase expressions - * must return scalar values. - * - * In addition, the existence of all referred attribute, query and constant + * must return scalar values. In addition, the existence of all referred attribute, query and constant * features is ensured. * * @author bratseth */ -public class RankingExpressionTypeResolver extends Processor { +public class RankingExpressionTypeValidator extends Processor { private final QueryProfileRegistry queryProfiles; - public RankingExpressionTypeResolver(Search search, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { + public RankingExpressionTypeValidator(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); this.queryProfiles = queryProfiles.getRegistry(); } @Override public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; if (documentsOnly) return; for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) { try { - resolveTypesIn(profile, validate); + validate(profile); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("In " + search + ", " + profile, e); @@ -52,34 +47,20 @@ public class RankingExpressionTypeResolver extends Processor { } } - /** - * Resolves the types of all functions in the given profile - * - * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types - */ - private void resolveTypesIn(RankProfile profile, boolean validate) { - TypeContext<Reference> context = profile.typeContext(queryProfiles); - for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) { - if ( ! function.getValue().function().arguments().isEmpty()) continue; - TensorType type = resolveType(function.getValue().function().getBody(), - "function '" + function.getKey() + "'", - context); - function.getValue().setReturnType(type); - } - - if (validate) { - profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context)); - ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); - ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - } + /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */ + private void validate(RankProfile profile) { + TypeContext context = profile.typeContext(queryProfiles); + profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context)); + ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); + ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); } - private TensorType resolveType(RankingExpression expression, String expressionDescription, TypeContext context) { + private TensorType ensureValid(RankingExpression expression, String expressionDescription, TypeContext context) { if (expression == null) return null; - return resolveType(expression.getRoot(), expressionDescription, context); + return ensureValid(expression.getRoot(), expressionDescription, context); } - private TensorType resolveType(ExpressionNode expression, String expressionDescription, TypeContext context) { + private TensorType ensureValid(ExpressionNode expression, String expressionDescription, TypeContext context) { TensorType type; try { type = expression.type(context); @@ -94,7 +75,7 @@ public class RankingExpressionTypeResolver extends Processor { private void ensureValidDouble(RankingExpression expression, String expressionDescription, TypeContext context) { if (expression == null) return; - TensorType type = resolveType(expression, expressionDescription, context); + TensorType type = ensureValid(expression, expressionDescription, context); if ( ! type.equals(TensorType.empty)) throw new IllegalArgumentException("The " + expressionDescription + " must produce a double " + "(a tensor with no dimensions), but produces " + type); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index 3bde76c1c79..ec4cbdfe58b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -44,7 +44,6 @@ public class RankProfileTypeSettingsProcessor extends Processor { } private void processAttributeFields() { - if (search == null) return; // we're processing global profiles for (SDField field : search.allConcreteFields()) { Attribute attribute = field.getAttributes().get(field.getName()); if (attribute != null && attribute.tensorType().isPresent()) { @@ -54,7 +53,6 @@ public class RankProfileTypeSettingsProcessor extends Processor { } private void processImportedFields() { - if (search == null) return; // we're processing global profiles Optional<ImportedFields> importedFields = search.importedFields(); if (importedFields.isPresent()) { importedFields.get().fields().forEach((fieldName, field) -> processImportedField(field)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java index 29ec26b06d2..620e44bc11a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Service.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java @@ -7,7 +7,7 @@ import java.util.HashMap; import java.util.Optional; /** - * Representation of a markProcessed which runs a service + * Representation of a process which runs a service * * @author gjoranv */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 13304ea10ee..4b70b1b5ae2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -32,9 +32,7 @@ import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RankProfileList; -import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel; @@ -170,7 +168,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri createGlobalRankProfiles(deployState.getImportedModels(), deployState.rankProfileRegistry(), - deployState.getQueryProfiles()); + deployState.getQueryProfiles().getRegistry()); this.rankProfileList = new RankProfileList(null, // null search -> global rankingConstants, AttributeFields.empty, @@ -221,23 +219,26 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Adds generic application specific clusters of services */ private void addServiceClusters(ApplicationPackage app, VespaModelBuilder builder) { - serviceClusters.addAll(builder.getClusters(app, this)); + for (ServiceCluster sc : builder.getClusters(app, this)) + serviceClusters.add(sc); } /** - * Creates a rank profile not attached to any search definition, for each imported model in the application package, - * and adds it to the given rank profile registry. + * Creates a rank profile not attached to any search definition, for each imported model in the application package */ - private void createGlobalRankProfiles(ImportedModels importedModels, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { + private ImmutableList<RankProfile> createGlobalRankProfiles(ImportedModels importedModels, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfiles) { + List<RankProfile> profiles = new ArrayList<>(); if ( ! importedModels.all().isEmpty()) { // models/ directory is available for (ImportedModel model : importedModels.all()) { RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()), - model.name(), profile, queryProfiles.getRegistry(), model); - convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); + model.name(), profile, queryProfiles, model); + for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); + } } } else { // generated and stored model information may be available instead @@ -247,12 +248,12 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry); rankProfileRegistry.add(profile); ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile); - convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); + for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) { + profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false); + } } } - new Processing().processRankProfiles(deployState.getDeployLogger(), - rankProfileRegistry, - queryProfiles, true, false); + return ImmutableList.copyOf(profiles); } /** Returns the global rank profiles as a rank profile list */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index fb0109ed32e..adf5c81283e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -48,7 +48,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -68,14 +67,14 @@ public class ConvertedModel { private final ModelName modelName; private final String modelDescription; - private final ImmutableMap<String, ExpressionFunction> expressions; + private final ImmutableMap<String, RankingExpression> expressions; /** The source importedModel, or empty if this was created from a stored converted model */ private final Optional<ImportedModel> sourceModel; private ConvertedModel(ModelName modelName, String modelDescription, - Map<String, ExpressionFunction> expressions, + Map<String, RankingExpression> expressions, Optional<ImportedModel> sourceModel) { this.modelName = modelName; this.modelDescription = modelDescription; @@ -133,23 +132,23 @@ public class ConvertedModel { * if signatures are used, or the expression name if signatures are not used and there are multiple * expressions, and the second is the output name if signature names are used. */ - public Map<String, ExpressionFunction> expressions() { return expressions; } + public Map<String, RankingExpression> expressions() { return expressions; } /** * Returns the expression matching the given arguments. */ public ExpressionNode expression(FeatureArguments arguments, RankProfileTransformContext context) { - ExpressionFunction expression = selectExpression(arguments); - if (sourceModel.isPresent()) // we should verify - verifyInputs(expression.getBody(), sourceModel.get(), context.rankProfile(), context.queryProfiles()); - return expression.getBody().getRoot(); + RankingExpression expression = selectExpression(arguments); + if (sourceModel.isPresent()) // we can verify + verifyRequiredFunctions(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles()); + return expression.getRoot(); } - private ExpressionFunction selectExpression(FeatureArguments arguments) { + private RankingExpression selectExpression(FeatureArguments arguments) { if (expressions.isEmpty()) throw new IllegalArgumentException("No expressions available in " + this); - ExpressionFunction expression = expressions.get(arguments.toName()); + RankingExpression expression = expressions.get(arguments.toName()); if (expression != null) return expression; if ( ! arguments.signature().isPresent()) { @@ -159,7 +158,7 @@ public class ConvertedModel { } if ( ! arguments.output().isPresent()) { - List<Map.Entry<String, ExpressionFunction>> entriesWithTheRightPrefix = + List<Map.Entry<String, RankingExpression>> entriesWithTheRightPrefix = expressions.entrySet().stream().filter(entry -> entry.getKey().startsWith(arguments.signature().get() + ".")).collect(Collectors.toList()); if (entriesWithTheRightPrefix.size() < 1) throw new IllegalArgumentException("No expressions named '" + arguments.signature().get() + @@ -180,10 +179,10 @@ public class ConvertedModel { // ----------------------- Static model conversion/storage below here - private static Map<String, ExpressionFunction> convertAndStore(ImportedModel model, - RankProfile profile, - QueryProfileRegistry queryProfiles, - ModelStore store) { + private static Map<String, RankingExpression> convertAndStore(ImportedModel model, + RankProfile profile, + QueryProfileRegistry queryProfiles, + ModelStore store) { // Add constants Set<String> constantsReplacedByFunctions = new HashSet<>(); model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v)); @@ -194,8 +193,8 @@ public class ConvertedModel { addGeneratedFunctions(model, profile); // Add expressions - Map<String, ExpressionFunction> expressions = new HashMap<>(); - for (Pair<String, ExpressionFunction> output : model.outputExpressions()) { + Map<String, RankingExpression> expressions = new HashMap<>(); + for (Pair<String, RankingExpression> output : model.outputExpressions()) { addExpression(output.getSecond(), output.getFirst(), constantsReplacedByFunctions, model, store, profile, queryProfiles, @@ -211,21 +210,21 @@ public class ConvertedModel { return expressions; } - private static void addExpression(ExpressionFunction expression, + private static void addExpression(RankingExpression expression, String expressionName, Set<String> constantsReplacedByFunctions, ImportedModel model, ModelStore store, RankProfile profile, QueryProfileRegistry queryProfiles, - Map<String, ExpressionFunction> expressions) { - expression = expression.withBody(replaceConstantsByFunctions(expression.getBody(), constantsReplacedByFunctions)); - reduceBatchDimensions(expression.getBody(), model, profile, queryProfiles); + Map<String, RankingExpression> expressions) { + expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions); + reduceBatchDimensions(expression, model, profile, queryProfiles); store.writeExpression(expressionName, expression); expressions.put(expressionName, expression); } - private static Map<String, ExpressionFunction> convertStored(ModelStore store, RankProfile profile) { + private static Map<String, RankingExpression> convertStored(ModelStore store, RankProfile profile) { for (Pair<String, Tensor> constant : store.readSmallConstants()) profile.addConstant(constant.getFirst(), asValue(constant.getSecond())); @@ -291,15 +290,15 @@ public class ConvertedModel { } /** - * Verify that the inputs declared in the given expression exists in the given rank profile as functions, - * and return tensors of the correct types. + * Verify that the functions referred in the given expression exists in the given rank profile, + * and return tensors of the types specified in requiredFunctions. */ - private static void verifyInputs(RankingExpression expression, ImportedModel model, - RankProfile profile, QueryProfileRegistry queryProfiles) { + private static void verifyRequiredFunctions(RankingExpression expression, ImportedModel model, + RankProfile profile, QueryProfileRegistry queryProfiles) { Set<String> functionNames = new HashSet<>(); addFunctionNamesIn(expression.getRoot(), functionNames, model); for (String functionName : functionNames) { - TensorType requiredType = model.inputs().get(functionName); + TensorType requiredType = model.requiredFunctions().get(functionName); if (requiredType == null) continue; // Not a required function RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName); @@ -376,7 +375,7 @@ public class ConvertedModel { List<ExpressionNode> children = ((TensorFunctionNode)node).children(); if (children.size() == 1 && children.get(0) instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) children.get(0); - if (model.inputs().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(tensorFunction, typeContext); } } @@ -384,7 +383,7 @@ public class ConvertedModel { } if (node instanceof ReferenceNode) { ReferenceNode referenceNode = (ReferenceNode) node; - if (model.inputs().containsKey(referenceNode.getName())) { + if (model.requiredFunctions().containsKey(referenceNode.getName())) { return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext); } } @@ -452,8 +451,7 @@ public class ConvertedModel { Set<String> constantsReplacedByFunctions) { if (constantsReplacedByFunctions.isEmpty()) return expression; return new RankingExpression(expression.getName(), - replaceConstantsByFunctions(expression.getRoot(), - constantsReplacedByFunctions)); + replaceConstantsByFunctions(expression.getRoot(), constantsReplacedByFunctions)); } private static ExpressionNode replaceConstantsByFunctions(ExpressionNode node, Set<String> constantsReplacedByFunctions) { @@ -526,21 +524,19 @@ public class ConvertedModel { * @param name the name of this ranking expression - may have 1-3 parts separated by dot where the first part * is always the model name */ - void writeExpression(String name, ExpressionFunction expression) { - StringBuilder b = new StringBuilder(expression.getBody().getRoot().toString()); - for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet()) - b.append('\n').append(input.getKey()).append('\t').append(input.getValue()); - application.getFile(modelFiles.expressionPath(name)).writeFile(new StringReader(b.toString())); + void writeExpression(String name, RankingExpression expression) { + application.getFile(modelFiles.expressionPath(name)) + .writeFile(new StringReader(expression.getRoot().toString())); } - Map<String, ExpressionFunction> readExpressions() { - Map<String, ExpressionFunction> expressions = new HashMap<>(); + Map<String, RankingExpression> readExpressions() { + Map<String, RankingExpression> expressions = new HashMap<>(); ApplicationFile expressionPath = application.getFile(modelFiles.expressionsPath()); if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return Collections.emptyMap(); for (ApplicationFile expressionFile : expressionPath.listFiles()) { - try (BufferedReader reader = new BufferedReader(expressionFile.createReader())){ + try (Reader reader = new BufferedReader(expressionFile.createReader())){ String name = expressionFile.getPath().getName(); - expressions.put(name, readExpression(name, reader)); + expressions.put(name, new RankingExpression(name, reader)); } catch (IOException e) { throw new UncheckedIOException("Failed reading " + expressionFile.getPath(), e); @@ -552,22 +548,8 @@ public class ConvertedModel { return expressions; } - private ExpressionFunction readExpression(String name, BufferedReader reader) - throws IOException, ParseException { - // First line is expression - RankingExpression expression = new RankingExpression(name, reader.readLine()); - // Next lines are inputs on the format name\ttensorTypeSpec - Map<String, TensorType> inputs = new LinkedHashMap<>(); - String line; - while (null != (line = reader.readLine())) { - String[] parts = line.split("\t"); - inputs.put(parts[0], TensorType.fromSpec(parts[1])); - } - return new ExpressionFunction(name, new ArrayList<>(inputs.keySet()), expression, inputs, Optional.empty()); - } - /** Adds this function expression to the application package so it can be read later. */ - public void writeFunction(String name, RankingExpression expression) { + void writeFunction(String name, RankingExpression expression) { application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" + expression.getRoot().toString() + "\n"); } @@ -579,20 +561,20 @@ public class ConvertedModel { if ( ! file.exists()) return Collections.emptyList(); List<Pair<String, RankingExpression>> functions = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader(file.createReader())) { - String line; - while (null != (line = reader.readLine())) { - String[] parts = line.split("\t"); - String name = parts[0]; - try { - RankingExpression expression = new RankingExpression(parts[0], parts[1]); - functions.add(new Pair<>(name, expression)); - } catch (ParseException e) { - throw new IllegalStateException("Could not parse " + name, e); - } + BufferedReader reader = new BufferedReader(file.createReader()); + String line; + while (null != (line = reader.readLine())) { + String[] parts = line.split("\t"); + String name = parts[0]; + try { + RankingExpression expression = new RankingExpression(parts[0], parts[1]); + functions.add(new Pair<>(name, expression)); + } + catch (ParseException e) { + throw new IllegalStateException("Could not parse " + name, e); } - return functions; } + return functions; } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/config-model/src/test/derived/gemini2/gemini.sd b/config-model/src/test/derived/gemini2/gemini.sd index 8a570e58fa8..01e20c1b30a 100644 --- a/config-model/src/test/derived/gemini2/gemini.sd +++ b/config-model/src/test/derived/gemini2/gemini.sd @@ -2,12 +2,6 @@ search gemini { document gemini { - field right type string { - indexing: attribute - } - field wrong type string { - indexing: attribute - } } rank-profile test { diff --git a/config-model/src/test/derived/gemini2/rank-profiles.cfg b/config-model/src/test/derived/gemini2/rank-profiles.cfg index 2b73e923c88..aa4f963320d 100644 --- a/config-model/src/test/derived/gemini2/rank-profiles.cfg +++ b/config-model/src/test/derived/gemini2/rank-profiles.cfg @@ -21,13 +21,9 @@ rankprofile[].fef.property[].name "rankingExpression(wrapper1@2d437c13405e61d6). rankprofile[].fef.property[].value "rankingExpression(wrapper2@2d437c13405e61d6)" rankprofile[].fef.property[].name "rankingExpression(toplevel).rankingScript" rankprofile[].fef.property[].value "rankingExpression(wrapper1@2d437c13405e61d6)" -rankprofile[].fef.property[].name "rankingExpression(toplevel).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(wrapper2@8fc8470e911f253f).rankingScript" rankprofile[].fef.property[].value "attribute(wrong)" rankprofile[].fef.property[].name "rankingExpression(wrapper1@8fc8470e911f253f).rankingScript" rankprofile[].fef.property[].value "rankingExpression(wrapper2@8fc8470e911f253f)" rankprofile[].fef.property[].name "rankingExpression(interfering).rankingScript" rankprofile[].fef.property[].value "rankingExpression(wrapper1@8fc8470e911f253f)" -rankprofile[].fef.property[].name "rankingExpression(interfering).type" -rankprofile[].fef.property[].value "tensor()" diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg index d109ca4f0ec..9629ad863d4 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg @@ -128,8 +128,6 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + rankBoost)" rankprofile[].fef.property[].name "vespa.rank.firstphase" @@ -147,16 +145,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + match)" rankprofile[].fef.property[].name "vespa.rank.firstphase" @@ -172,15 +164,11 @@ rankprofile[].fef.property[].value "rankingExpression(mysummaryfeature)" rankprofile[].name "macros3" rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).rankingScript" rankprofile[].fef.property[].value "5" -rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "vespa.summary.feature" rankprofile[].fef.property[].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))" rankprofile[].name "macros3-inherited" rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).rankingScript" rankprofile[].fef.property[].value "5" -rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "vespa.summary.feature" rankprofile[].fef.property[].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))" rankprofile[].name "macros-inherited" @@ -190,16 +178,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + match)" rankprofile[].fef.property[].name "vespa.rank.firstphase" @@ -221,16 +203,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)" -rankprofile[].fef.property[].name "rankingExpression(myfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript" rankprofile[].fef.property[].value "4 * (match + match)" rankprofile[].fef.property[].name "vespa.rank.firstphase" @@ -252,16 +228,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript rankprofile[].fef.property[].value "4 * (var1 + var2)" rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(myfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript" rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript" rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" @@ -279,14 +249,8 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" -rankprofile[].fef.property[].name "rankingExpression(m2).type" -rankprofile[].fef.property[].value "tensor()" -rankprofile[].fef.property[].name "rankingExpression(m1).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" -rankprofile[].fef.property[].name "rankingExpression(m4).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" @@ -296,18 +260,10 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" -rankprofile[].fef.property[].name "rankingExpression(m2).type" -rankprofile[].fef.property[].value "tensor()" -rankprofile[].fef.property[].name "rankingExpression(m1).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(m4).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" -rankprofile[].fef.property[].name "rankingExpression(m3).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" @@ -317,14 +273,8 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" -rankprofile[].fef.property[].name "rankingExpression(m2).type" -rankprofile[].fef.property[].value "tensor()" -rankprofile[].fef.property[].name "rankingExpression(m1).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness" -rankprofile[].fef.property[].name "rankingExpression(m4).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" @@ -334,22 +284,12 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" -rankprofile[].fef.property[].name "rankingExpression(m2).type" -rankprofile[].fef.property[].value "tensor()" -rankprofile[].fef.property[].name "rankingExpression(m1).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness" -rankprofile[].fef.property[].name "rankingExpression(m4).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" -rankprofile[].fef.property[].name "rankingExpression(m3).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "rankingExpression(m5).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(m4))" -rankprofile[].fef.property[].name "rankingExpression(m5).type" -rankprofile[].fef.property[].value "tensor()" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index 471343da63c..cb496c06367 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -43,14 +43,10 @@ rankprofile[].fef.property[].value "tensor(x{})" rankprofile[].fef.property[].name "vespa.type.attribute.f4" rankprofile[].fef.property[].value "tensor(x[10],y[20])" rankprofile[].name "profile3" -rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript" -rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)" -rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type" -rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" -rankprofile[].fef.property[].value "reduce(rankingExpression(joinedtensors), sum)" +rankprofile[].fef.property[].value "reduce(tensor(i[10])(i) * attribute(f4), sum)" rankprofile[].fef.property[].name "vespa.type.attribute.f2" rankprofile[].fef.property[].value "tensor(x[2],y[])" rankprofile[].fef.property[].name "vespa.type.attribute.f3" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index 54463448e28..5792a7997f8 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -36,11 +36,7 @@ search tensor { rank-profile profile3 { first-phase { - expression: sum(joinedtensors) - } - - function joinedtensors() { - expression: tensor(i[10])(i) * attribute(f4) + expression: sum(tensor(i[10])(i) * attribute(f4)) } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 2c1f4c8ecb6..150469cc928 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -82,7 +82,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase new ImportedModels(), new AttributeFields(s)).configProperties(); assertEquals("(rankingExpression(foo).rankingScript,14.0)", rankProperties.get(0).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript,16.6)", rankProperties.get(3).toString()); + assertEquals("(rankingExpression(firstphase).rankingScript,16.6)", rankProperties.get(2).toString()); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java index 0ff8a5cc7ca..17bebcba70e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java @@ -40,7 +40,7 @@ public class RankingExpressionLoopDetectionTestCase { fail("Excepted exception"); } catch (IllegalArgumentException e) { - assertEquals("In search definition 'test', rank profile 'test': The function 'foo' is invalid: Invocation loop: foo -> foo", + assertEquals("In search definition 'test', rank profile 'test': The first-phase expression is invalid: Invocation loop: foo -> foo", Exceptions.toMessageString(e)); } } @@ -75,7 +75,7 @@ public class RankingExpressionLoopDetectionTestCase { fail("Excepted exception"); } catch (IllegalArgumentException e) { - assertEquals("In search definition 'test', rank profile 'test': The function 'foo' is invalid: Invocation loop: arg(5) -> foo -> arg(5)", + assertEquals("In search definition 'test', rank profile 'test': The first-phase expression is invalid: Invocation loop: foo -> arg(5) -> foo", Exceptions.toMessageString(e)); } } @@ -110,7 +110,7 @@ public class RankingExpressionLoopDetectionTestCase { fail("Excepted exception"); } catch (IllegalArgumentException e) { - assertEquals("In search definition 'test', rank profile 'test': The function 'foo' is invalid: Invocation loop: arg(foo) -> foo -> arg(foo)", + assertEquals("In search definition 'test', rank profile 'test': The first-phase expression is invalid: Invocation loop: foo -> arg(foo) -> foo", Exceptions.toMessageString(e)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 4f99922a422..e15d4075b19 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -211,16 +211,12 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase censorBindingHash(testRankProperties.get(1).toString())); assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(hidden_layer).type,tensor(x[]))", - censorBindingHash(testRankProperties.get(3).toString())); assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", - testRankProperties.get(4).toString()); - assertEquals("(rankingExpression(final_layer).type,tensor(x[]))", - testRankProperties.get(5).toString()); + testRankProperties.get(3).toString()); assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", - testRankProperties.get(6).toString()); + testRankProperties.get(4).toString()); assertEquals("(rankingExpression(secondphase).rankingScript,reduce(rankingExpression(final_layer), sum))", - testRankProperties.get(7).toString()); + testRankProperties.get(5).toString()); } private QueryProfileRegistry queryProfileWith(String field, String type) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java index 1b917b6f3a3..0d8cbbf2e6a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java @@ -19,7 +19,7 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -public class RankingExpressionTypeResolverTestCase { +public class RankingExpressionTypeValidatorTestCase { @Test public void tensorFirstPhaseMustProduceDouble() throws Exception { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 7d62bc5089d..fd048737b43 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -41,7 +41,7 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { new QueryProfileRegistry(), new ImportedModels(), new AttributeFields(search)).configProperties(); - assertEquals(7, rankProperties.size()); + assertEquals(6, rankProperties.size()); assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(0).getFirst()); assertEquals("var1 * var2 + 890", rankProperties.get(0).getSecond()); @@ -49,17 +49,14 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(1).getFirst()); assertEquals("78 + closeness(distance)", rankProperties.get(1).getSecond()); - assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(6).getFirst()); - assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(6).getSecond()); + assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); + assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); - assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(4).getFirst()); - assertEquals("7 * 8 + 890", rankProperties.get(4).getSecond()); + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(3).getSecond()); - assertEquals("rankingExpression(artistmatch).type", rankProperties.get(2).getFirst()); - assertEquals("tensor()", rankProperties.get(2).getSecond()); - - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(3).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(3).getSecond()); + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(2).getSecond()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 6e3a227e2a9..8e721dbe503 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -58,8 +58,8 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { "max(attribute(tensor_field_1),x)"); assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)", "1 + max(attribute(tensor_field_1),x)"); - assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),attribute(tensor_field_1))", - "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),attribute(tensor_field_1))"); + assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)", + "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)"); assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)", "max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)"); assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)", diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java index 4dbec304bde..d74f29a3b77 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java @@ -147,7 +147,7 @@ public class AsyncExecution { } private static <T> Future<T> getFuture(Callable<T> callable) { - FutureTask<T> future = new FutureTask<>(callable); + final FutureTask<T> future = new FutureTask<>(callable); getExecutor().execute(future); return future; } diff --git a/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java b/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java index 2c40165f8e5..eac96e9b408 100644 --- a/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java +++ b/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java @@ -59,14 +59,14 @@ public class AsyncExecution { /** * Create an async execution of a single processor */ - public AsyncExecution(Processor processor, Execution parent) { + public AsyncExecution(final Processor processor, Execution parent) { this(new Execution(processor, parent)); } /** * Create an async execution of a chain */ - public AsyncExecution(Chain<? extends Processor> chain, Execution parent) { + public AsyncExecution(final Chain<? extends Processor> chain, Execution parent) { this(new Execution(chain, parent)); } @@ -81,7 +81,7 @@ public class AsyncExecution { * * @param execution the execution from which the state of this is created */ - public AsyncExecution(Execution execution) { + public AsyncExecution(final Execution execution) { this.execution = new Execution(execution); } @@ -89,7 +89,7 @@ public class AsyncExecution { * Performs an async processing. Note that the given request cannot be simultaneously * used in multiple such processings - a clone must be created for each. */ - public FutureResponse process(Request request) { + public FutureResponse process(final Request request) { return getFutureResponse(new Callable<Response>() { @Override public Response call() { @@ -99,13 +99,13 @@ public class AsyncExecution { } private static <T> Future<T> getFuture(final Callable<T> callable) { - FutureTask<T> future = new FutureTask<>(callable); + final FutureTask<T> future = new FutureTask<>(callable); executorMain.execute(future); return future; } - private FutureResponse getFutureResponse(Callable<Response> callable, Request request) { - FutureResponse future = new FutureResponse(callable, execution, request); + private FutureResponse getFutureResponse(final Callable<Response> callable, final Request request) { + final FutureResponse future = new FutureResponse(callable, execution, request); executorMain.execute(future.delegate()); return future; } @@ -118,15 +118,15 @@ public class AsyncExecution { * @return the list of responses in the same order as returned from the task collection */ // Note that this may also be achieved using guava Futures. Not sure if this should be deprecated because of it. - public static List<Response> waitForAll(Collection<FutureResponse> tasks, long timeout) { + public static List<Response> waitForAll(final Collection<FutureResponse> tasks, final long timeout) { // Copy the list in case it is modified while we are waiting - List<FutureResponse> workingTasks = new ArrayList<>(tasks); + final List<FutureResponse> workingTasks = new ArrayList<>(tasks); @SuppressWarnings({"rawtypes", "unchecked"}) - Future task = getFuture(new Callable() { + final Future task = getFuture(new Callable() { @Override public List<Future> call() { - for (FutureResponse task : workingTasks) { + for (final FutureResponse task : workingTasks) { task.get(); } return null; @@ -135,12 +135,12 @@ public class AsyncExecution { try { task.get(timeout, TimeUnit.MILLISECONDS); - } catch (TimeoutException | InterruptedException | ExecutionException e) { + } catch (final TimeoutException | InterruptedException | ExecutionException e) { // Handle timeouts below } - List<Response> responses = new ArrayList<>(tasks.size()); - for (FutureResponse future : workingTasks) + final List<Response> responses = new ArrayList<>(tasks.size()); + for (final FutureResponse future : workingTasks) responses.add(getTaskResponse(future)); return responses; } @@ -153,4 +153,5 @@ public class AsyncExecution { } } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index f6502a9801d..da34ab8822d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -2,11 +2,8 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; -import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import java.security.MessageDigest; @@ -16,14 +13,9 @@ import java.util.Deque; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Optional; /** - * A function defined by a ranking expression, optionally containing type information - * for inputs and outputs. - * - * Immutable, but note that ranking expressions are *not* immutable. + * A function defined by a ranking expression * * @author Simon Thoresen Hult * @author bratseth @@ -32,13 +24,8 @@ public class ExpressionFunction { private final String name; private final ImmutableList<String> arguments; - - /** Types of the inputs, if known. The keys here is any subset (including empty and identity) of the argument list */ - private final ImmutableMap<String, TensorType> argumentTypes; private final RankingExpression body; - private final Optional<TensorType> returnType; - /** * Constructs a new function with no arguments * @@ -57,18 +44,9 @@ public class ExpressionFunction { * @param body the ranking expression that defines this function */ public ExpressionFunction(String name, List<String> arguments, RankingExpression body) { - this(name, arguments, body, ImmutableMap.of(), Optional.empty()); - } - - public ExpressionFunction(String name, List<String> arguments, RankingExpression body, - Map<String, TensorType> argumentTypes, Optional<TensorType> returnType) { - this.name = Objects.requireNonNull(name, "name cannot be null"); + this.name = name; this.arguments = arguments==null ? ImmutableList.of() : ImmutableList.copyOf(arguments); - this.body = Objects.requireNonNull(body, "body cannot be null"); - if ( ! this.arguments.containsAll(argumentTypes.keySet())) - throw new IllegalArgumentException("Argument type keys must be a subset of the argument keys"); - this.argumentTypes = ImmutableMap.copyOf(argumentTypes); - this.returnType = Objects.requireNonNull(returnType, "returnType cannot be null"); + this.body = body; } public String getName() { return name; } @@ -78,27 +56,9 @@ public class ExpressionFunction { public RankingExpression getBody() { return body; } - /** Returns the types of the arguments of this, if specified. The keys of this may be any subset of the arguments */ - public Map<String, TensorType> argumentTypes() { return argumentTypes; } - - /** Returns the return type of this, or empty if not specified */ - public Optional<TensorType> returnType() { return returnType; } - - public ExpressionFunction withName(String name) { - return new ExpressionFunction(name, arguments, body, argumentTypes, returnType); - } - /** Returns a copy of this with the body changed to the given value */ public ExpressionFunction withBody(RankingExpression body) { - return new ExpressionFunction(name, arguments, body, argumentTypes, returnType); - } - - public ExpressionFunction withReturnType(TensorType returnType) { - return new ExpressionFunction(name, arguments, body, argumentTypes, Optional.of(returnType)); - } - - public ExpressionFunction withArgumentTypes(Map<String, TensorType> argumentTypes) { - return new ExpressionFunction(name, arguments, body, argumentTypes, returnType); + return new ExpressionFunction(name, arguments, body); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java index 9ff391a5cfe..282a4c5e0a9 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java @@ -1,22 +1,15 @@ package com.yahoo.searchlib.rankingexpression.integration.ml; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.yahoo.collections.Pair; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.regex.Pattern; /** @@ -33,11 +26,12 @@ public class ImportedModel { private final String source; private final Map<String, Signature> signatures = new HashMap<>(); - private final Map<String, TensorType> inputs = new HashMap<>(); + private final Map<String, TensorType> arguments = new HashMap<>(); private final Map<String, Tensor> smallConstants = new HashMap<>(); private final Map<String, Tensor> largeConstants = new HashMap<>(); private final Map<String, RankingExpression> expressions = new HashMap<>(); private final Map<String, RankingExpression> functions = new HashMap<>(); + private final Map<String, TensorType> requiredFunctions = new HashMap<>(); /** * Creates a new imported model. @@ -55,11 +49,11 @@ public class ImportedModel { /** Returns the name of this model, which can only contain the characters in [A-Za-z0-9_] */ public String name() { return name; } - /** Returns the source path (directory or file) of this model */ + /** Returns the source path (directiry or file) of this model */ public String source() { return source; } - /** Returns an immutable map of the inputs of this */ - public Map<String, TensorType> inputs() { return Collections.unmodifiableMap(inputs); } + /** Returns an immutable map of the arguments ("Placeholders") of this */ + public Map<String, TensorType> arguments() { return Collections.unmodifiableMap(arguments); } /** * Returns an immutable map of the small constants of this. @@ -77,7 +71,7 @@ public class ImportedModel { /** * Returns an immutable map of the expressions of this - corresponding to graph nodes - * which are not Inputs/Placeholders or Variables (which instead become respectively inputs and constants). + * which are not Inputs/Placeholders or Variables (which instead become respectively arguments and constants). * Note that only nodes recursively referenced by a placeholder/input are added. */ public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); } @@ -88,6 +82,9 @@ public class ImportedModel { */ public Map<String, RankingExpression> functions() { return Collections.unmodifiableMap(functions); } + /** Returns an immutable map of the functions that must be provided by the environment running this model */ + public Map<String, TensorType> requiredFunctions() { return Collections.unmodifiableMap(requiredFunctions); } + /** Returns an immutable map of the signatures of this */ public Map<String, Signature> signatures() { return Collections.unmodifiableMap(signatures); } @@ -99,11 +96,12 @@ public class ImportedModel { /** Convenience method for returning a default signature */ Signature defaultSignature() { return signature(defaultSignatureName); } - void input(String name, TensorType argumentType) { inputs.put(name, argumentType); } + void argument(String name, TensorType argumentType) { arguments.put(name, argumentType); } void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); } void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); } void expression(String name, RankingExpression expression) { expressions.put(name, expression); } void function(String name, RankingExpression expression) { functions.put(name, expression); } + void requiredFunction(String name, TensorType type) { requiredFunctions.put(name, type); } /** * Returns all the output expressions of this indexed by name. The names consist of one or two parts @@ -111,39 +109,24 @@ public class ImportedModel { * if signatures are used, or the expression name if signatures are not used and there are multiple * expressions, and the second is the output name if signature names are used. */ - public List<Pair<String, ExpressionFunction>> outputExpressions() { - List<Pair<String, ExpressionFunction>> expressions = new ArrayList<>(); + public List<Pair<String, RankingExpression>> outputExpressions() { + List<Pair<String, RankingExpression>> expressions = new ArrayList<>(); for (Map.Entry<String, Signature> signatureEntry : signatures().entrySet()) { for (Map.Entry<String, String> outputEntry : signatureEntry.getValue().outputs().entrySet()) expressions.add(new Pair<>(signatureEntry.getKey() + "." + outputEntry.getKey(), - signatureEntry.getValue().outputExpression(outputEntry.getKey()) - .withName(signatureEntry.getKey() + "." + outputEntry.getKey()))); + expressions().get(outputEntry.getValue()))); if (signatureEntry.getValue().outputs().isEmpty()) // fallback: Signature without outputs expressions.add(new Pair<>(signatureEntry.getKey(), - new ExpressionFunction(signatureEntry.getKey(), - new ArrayList<>(signatureEntry.getValue().inputs().keySet()), - expressions().get(signatureEntry.getKey()), - signatureEntry.getValue().inputMap(), - Optional.empty()))); + expressions().get(signatureEntry.getKey()))); } if (signatures().isEmpty()) { // fallback for models without signatures if (expressions().size() == 1) { Map.Entry<String, RankingExpression> singleEntry = this.expressions.entrySet().iterator().next(); - expressions.add(new Pair<>(singleEntry.getKey(), - new ExpressionFunction(singleEntry.getKey(), - new ArrayList<>(inputs.keySet()), - singleEntry.getValue(), - inputs, - Optional.empty()))); + expressions.add(new Pair<>(singleEntry.getKey(), singleEntry.getValue())); } else { for (Map.Entry<String, RankingExpression> expressionEntry : expressions().entrySet()) { - expressions.add(new Pair<>(expressionEntry.getKey(), - new ExpressionFunction(expressionEntry.getKey(), - new ArrayList<>(inputs.keySet()), - expressionEntry.getValue(), - inputs, - Optional.empty()))); + expressions.add(new Pair<>(expressionEntry.getKey(), expressionEntry.getValue())); } } } @@ -151,7 +134,7 @@ public class ImportedModel { } /** - * A signature is a set of named inputs and outputs, where the inputs maps to input + * A signature is a set of named inputs and outputs, where the inputs maps to argument * ("placeholder") names+types, and outputs maps to expressions nodes. * Note that TensorFlow supports multiple signatures in their format, but ONNX has no explicit * concept of signatures. For now, we handle ONNX models as having a single signature. @@ -159,8 +142,8 @@ public class ImportedModel { public class Signature { private final String name; - private final Map<String, String> inputs = new LinkedHashMap<>(); - private final Map<String, String> outputs = new LinkedHashMap<>(); + private final Map<String, String> inputs = new HashMap<>(); + private final Map<String, String> outputs = new HashMap<>(); private final Map<String, String> skippedOutputs = new HashMap<>(); private final List<String> importWarnings = new ArrayList<>(); @@ -175,20 +158,12 @@ public class ImportedModel { /** * Returns an immutable map of the inputs (evaluation context) of this. This is a map from input name - * in this signature to input name in the owning model + * to argument (Placeholder) name in the owner of this */ public Map<String, String> inputs() { return Collections.unmodifiableMap(inputs); } - /** Returns the name and type of all inputs in this signature as an immutable map */ - public Map<String, TensorType> inputMap() { - ImmutableMap.Builder<String, TensorType> inputs = new ImmutableMap.Builder<>(); - for (Map.Entry<String, String> inputEntry : inputs().entrySet()) - inputs.put(inputEntry.getKey(), owner().inputs().get(inputEntry.getValue())); - return inputs.build(); - } - - /** Returns the type of the input this input references */ - public TensorType inputArgument(String inputName) { return owner().inputs().get(inputs.get(inputName)); } + /** Returns the type of the argument this input references */ + public TensorType inputArgument(String inputName) { return owner().arguments().get(inputs.get(inputName)); } /** Returns an immutable list of the expression names of this */ public Map<String, String> outputs() { return Collections.unmodifiableMap(outputs); } @@ -205,13 +180,7 @@ public class ImportedModel { public List<String> importWarnings() { return Collections.unmodifiableList(importWarnings); } /** Returns the expression this output references */ - public ExpressionFunction outputExpression(String outputName) { - return new ExpressionFunction(outputName, - new ArrayList<>(inputs.keySet()), - owner().expressions().get(outputs.get(outputName)), - inputMap(), - Optional.empty()); - } + public RankingExpression outputExpression(String outputName) { return owner().expressions().get(outputs.get(outputName)); } @Override public String toString() { return "signature '" + name + "'"; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java index b7138ad87e3..d25502fd149 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java @@ -187,7 +187,8 @@ public abstract class ModelImporter { if (operation.isInput()) { // All inputs must have dimensions with standard naming convention: d0, d1, ... OrderedTensorType standardNamingConvention = OrderedTensorType.standardType(operation.type().get()); - model.input(operation.vespaName(), standardNamingConvention.type()); + model.argument(operation.vespaName(), standardNamingConvention.type()); + model.requiredFunction(operation.vespaName(), standardNamingConvention.type()); } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java index e6bb5f40b3f..917b0d6a389 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java @@ -2,6 +2,7 @@ package com.yahoo.searchlib.rankingexpression.integration.ml; +import com.yahoo.io.IOUtils; import com.yahoo.searchlib.rankingexpression.integration.ml.importer.IntermediateGraph; import com.yahoo.searchlib.rankingexpression.integration.ml.importer.onnx.GraphImporter; import onnx.Onnx; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index 94d663b4954..796c13a8669 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -3,8 +3,6 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.tensor.TensorType; import java.util.Collection; import java.util.Collections; @@ -82,14 +80,9 @@ public class SerializationContext extends FunctionReferenceContext { serializedFunctions.put(name, expressionString); } - /** Adds the serialization of the an argument type to a function */ - public void addArgumentTypeSerialization(String functionName, String argumentName, TensorType type) { - serializedFunctions.put("rankingExpression(" + functionName + ")." + argumentName + ".type", type.toString()); - } - - /** Adds the serialization of the return type of a function */ - public void addFunctionTypeSerialization(String functionName, TensorType type) { - serializedFunctions.put("rankingExpression(" + functionName + ").type", type.toString()); + /** Returns the existing serialization of a function, or null if none */ + public String getFunctionSerialization(String name) { + return serializedFunctions.get(name); } @Override diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java index 969bc318391..118eba2cd96 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java @@ -15,7 +15,6 @@ import org.junit.Test; import java.io.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import static org.junit.Assert.fail; @@ -41,8 +40,7 @@ public class GroupingSerializationTest { t.assertMatch(new FloatResultNode(7.3)); t.assertMatch(new StringResultNode("7.3")); t.assertMatch(new StringResultNode( - new String(new byte[]{(byte)0xe5, (byte)0xa6, (byte)0x82, (byte)0xe6, (byte)0x9e, (byte)0x9c}, - StandardCharsets.UTF_8))); + new String(new byte[]{(byte)0xe5, (byte)0xa6, (byte)0x82, (byte)0xe6, (byte)0x9e, (byte)0x9c}))); t.assertMatch(new RawResultNode(new byte[]{'7', '.', '4'})); t.assertMatch(new IntegerBucketResultNode()); t.assertMatch(new FloatBucketResultNode()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java index 593e7b54c10..bf9684082f4 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.integration.ml; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import org.junit.Test; @@ -21,11 +20,10 @@ public class BatchNormImportTestCase { assertEquals("Has skipped outputs", 0, model.get().signature("serving_default").skippedOutputs().size()); - ExpressionFunction output = signature.outputExpression("y"); + RankingExpression output = signature.outputExpression("y"); assertNotNull(output); - assertEquals("dnn/batch_normalization_3/batchnorm/add_1", output.getBody().getName()); - model.assertEqualResult("X", output.getBody().getName()); - assertEquals("{x=tensor(d0[],d1[784])}", output.argumentTypes().toString()); + assertEquals("dnn/batch_normalization_3/batchnorm/add_1", output.getName()); + model.assertEqualResult("X", output.getName()); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java index 59712c0152f..a8f7542f3a4 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.integration.ml; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.tensor.TensorType; import org.junit.Test; @@ -20,23 +19,22 @@ public class DropoutImportTestCase { TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/files/integration/tensorflow/dropout/saved"); // Check required functions - assertEquals(1, model.get().inputs().size()); - assertTrue(model.get().inputs().containsKey("X")); + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("X")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().inputs().get("X")); + model.get().requiredFunctions().get("X")); ImportedModel.Signature signature = model.get().signature("serving_default"); assertEquals("Has skipped outputs", 0, model.get().signature("serving_default").skippedOutputs().size()); - ExpressionFunction output = signature.outputExpression("y"); + RankingExpression output = signature.outputExpression("y"); assertNotNull(output); - assertEquals("outputs/Maximum", output.getBody().getName()); + assertEquals("outputs/Maximum", output.getName()); assertEquals("join(join(imported_ml_function_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_function_test_outputs_BiasAdd, f(a,b)(max(a,b)))", - output.getBody().getRoot().toString()); - model.assertEqualResult("X", output.getBody().getName()); - assertEquals("{x=tensor(d0[],d1[784])}", output.argumentTypes().toString()); + output.getRoot().toString()); + model.assertEqualResult("X", output.getName()); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java index 3d8d5d5a570..add66eece1a 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.integration.ml; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import org.junit.Test; @@ -21,10 +20,11 @@ public class MnistImportTestCase { assertEquals("Has skipped outputs", 0, model.get().signature("serving_default").skippedOutputs().size()); - ExpressionFunction output = signature.outputExpression("y"); + RankingExpression output = signature.outputExpression("y"); assertNotNull(output); - assertEquals("dnn/outputs/add", output.getBody().getName()); - model.assertEqualResultSum("input", output.getBody().getName(), 0.00001); + assertEquals("dnn/outputs/add", output.getName()); + model.assertEqualResultSum("input", output.getName(), 0.00001); } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java index b6e83404ab1..e20ac16a691 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java @@ -1,6 +1,5 @@ package com.yahoo.searchlib.rankingexpression.integration.ml; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; @@ -28,28 +27,27 @@ public class OnnxMnistSoftmaxImportTestCase { Tensor constant0 = model.largeConstants().get("test_Variable"); assertNotNull(constant0); assertEquals(new TensorType.Builder().indexed("d2", 784).indexed("d1", 10).build(), - constant0.type()); + constant0.type()); assertEquals(7840, constant0.size()); Tensor constant1 = model.largeConstants().get("test_Variable_1"); assertNotNull(constant1); - assertEquals(new TensorType.Builder().indexed("d1", 10).build(), constant1.type()); + assertEquals(new TensorType.Builder().indexed("d1", 10).build(), + constant1.type()); assertEquals(10, constant1.size()); - // Check inputs - assertEquals(1, model.inputs().size()); - assertTrue(model.inputs().containsKey("Placeholder")); - assertEquals(TensorType.fromSpec("tensor(d0[],d1[784])"), model.inputs().get("Placeholder")); + // Check required functions (inputs) + assertEquals(1, model.requiredFunctions().size()); + assertTrue(model.requiredFunctions().containsKey("Placeholder")); + assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), + model.requiredFunctions().get("Placeholder")); - // Check signature - ExpressionFunction output = model.defaultSignature().outputExpression("add"); + // Check outputs + RankingExpression output = model.defaultSignature().outputExpression("add"); assertNotNull(output); - assertEquals("add", output.getBody().getName()); + assertEquals("add", output.getName()); assertEquals("join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(test_Variable), f(a,b)(a * b)), sum, d2), constant(test_Variable_1), f(a,b)(a + b))", - output.getBody().getRoot().toString()); - assertEquals(TensorType.fromSpec("tensor(d0[],d1[784])"), - model.inputs().get(model.defaultSignature().inputs().get("Placeholder"))); - assertEquals("{Placeholder=tensor(d0[],d1[784])}", output.argumentTypes().toString()); + output.getRoot().toString()); } @Test diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java index 0a48ecfce21..ef28eb4678f 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.integration.ml; -import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -39,10 +38,10 @@ public class TensorFlowMnistSoftmaxImportTestCase { assertEquals(0, model.get().functions().size()); // Check required functions - assertEquals(1, model.get().inputs().size()); - assertTrue(model.get().inputs().containsKey("Placeholder")); + assertEquals(1, model.get().requiredFunctions().size()); + assertTrue(model.get().requiredFunctions().containsKey("Placeholder")); assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(), - model.get().inputs().get("Placeholder")); + model.get().requiredFunctions().get("Placeholder")); // Check signatures assertEquals(1, model.get().signatures().size()); @@ -57,12 +56,11 @@ public class TensorFlowMnistSoftmaxImportTestCase { // ... signature outputs assertEquals(1, signature.outputs().size()); - ExpressionFunction output = signature.outputExpression("y"); + RankingExpression output = signature.outputExpression("y"); assertNotNull(output); - assertEquals("add", output.getBody().getName()); + assertEquals("add", output.getName()); assertEquals("join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(test_Variable_read), f(a,b)(a * b)), sum, d2), constant(test_Variable_1_read), f(a,b)(a + b))", - output.getBody().getRoot().toString()); - assertEquals("{x=tensor(d0[],d1[784])}", output.argumentTypes().toString()); + output.getRoot().toString()); // Test execution model.assertEqualResult("Placeholder", "MatMul"); |