aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-09-25 10:20:23 -0700
committerGitHub <noreply@github.com>2018-09-25 10:20:23 -0700
commit0246064bbfb9657515f516e2fea12d593cd13016 (patch)
tree5e2727aaf1eef5ef5043b399508cfca99b2e33c2
parentf627463a8100090ec109d27c3aeb439a3395a34f (diff)
parentdfe56b565f6728a6c537867d28e388cd04718071 (diff)
Merge pull request #7094 from vespa-engine/revert-7070-bratseth/rank-type-information-2
Revert "Bratseth/rank type information 2"
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java17
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/Search.java14
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java7
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java10
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java35
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java28
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java (renamed from config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java)57
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/Service.java2
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java31
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java116
-rw-r--r--config-model/src/test/derived/gemini2/gemini.sd6
-rw-r--r--config-model/src/test/derived/gemini2/rank-profiles.cfg4
-rw-r--r--config-model/src/test/derived/rankexpression/rank-profiles.cfg60
-rw-r--r--config-model/src/test/derived/tensor/rank-profiles.cfg6
-rw-r--r--config-model/src/test/derived/tensor/tensor.sd6
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java6
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java10
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java (renamed from config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java)2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java17
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java2
-rw-r--r--processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java29
-rwxr-xr-xsearchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java48
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java79
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java3
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java1
-rw-r--r--searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java13
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java4
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java8
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java16
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java8
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java26
-rw-r--r--searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java14
35 files changed, 241 insertions, 452 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index 937151c0d3a..16e494c2db1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -449,7 +449,7 @@ public class RankProfile implements Serializable, Cloneable {
addRankProperty(new RankProperty(name, parameter));
}
- private void addRankProperty(RankProperty rankProperty) {
+ public void addRankProperty(RankProperty rankProperty) {
// Just the usual multimap semantics here
List<RankProperty> properties = rankProperties.get(rankProperty.getName());
if (properties == null) {
@@ -541,14 +541,15 @@ public class RankProfile implements Serializable, Cloneable {
/** Returns an unmodifiable view of the functions in this */
public Map<String, RankingExpressionFunction> getFunctions() {
- if (functions.isEmpty() && getInherited() == null) return Collections.emptyMap();
- if (functions.isEmpty()) return getInherited().getFunctions();
+ if (functions.size() == 0 && getInherited() == null) return Collections.emptyMap();
+ if (functions.size() == 0) return getInherited().getFunctions();
if (getInherited() == null) return Collections.unmodifiableMap(functions);
// Neither is null
Map<String, RankingExpressionFunction> allFunctions = new LinkedHashMap<>(getInherited().getFunctions());
allFunctions.putAll(functions);
return Collections.unmodifiableMap(allFunctions);
+
}
public int getKeepRankCount() {
@@ -694,7 +695,7 @@ public class RankProfile implements Serializable, Cloneable {
for (Map.Entry<String, RankingExpressionFunction> entry : functions.entrySet()) {
RankingExpressionFunction rankingExpressionFunction = entry.getValue();
RankingExpression compiled = compile(rankingExpressionFunction.function().getBody(), queryProfiles, importedModels, getConstants(), inlineFunctions, expressionTransforms);
- compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withExpression(compiled));
+ compiledFunctions.put(entry.getKey(), rankingExpressionFunction.withBody(compiled));
}
return compiledFunctions;
}
@@ -897,7 +898,7 @@ public class RankProfile implements Serializable, Cloneable {
/** A function in a rank profile */
public static class RankingExpressionFunction {
- private ExpressionFunction function;
+ private final ExpressionFunction function;
/** True if this should be inlined into calling expressions. Useful for very cheap functions. */
private final boolean inline;
@@ -907,17 +908,13 @@ public class RankProfile implements Serializable, Cloneable {
this.inline = inline;
}
- public void setReturnType(TensorType type) {
- this.function = function.withReturnType(type);
- }
-
public ExpressionFunction function() { return function; }
public boolean inline() {
return inline && function.arguments().isEmpty(); // only inline no-arg functions;
}
- public RankingExpressionFunction withExpression(RankingExpression expression) {
+ public RankingExpressionFunction withBody(RankingExpression expression) {
return new RankingExpressionFunction(function.withBody(expression), inline);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index a988da9664e..f42d5de21e8 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -59,6 +59,9 @@ public class Search implements Serializable, ImmutableSearch {
// Field sets
private FieldSets fieldSets = new FieldSets();
+ // Whether or not this object has been processed.
+ private boolean processed;
+
// The unique name of this search definition.
private String name;
@@ -582,6 +585,17 @@ public class Search implements Serializable, ImmutableSearch {
return false;
}
+ public void process() {
+ if (processed) {
+ throw new IllegalStateException("Search '" + getName() + "' already processed.");
+ }
+ processed = true;
+ }
+
+ public boolean isProcessed() {
+ return processed;
+ }
+
/**
* The field set settings for this search
*
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 151ad02a3fa..3c2ebc058ac 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -188,6 +188,10 @@ public class SearchBuilder {
throw new IllegalArgumentException("Search has no name.");
}
String rawName = rawSearch.getName();
+ if (rawSearch.isProcessed()) {
+ throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
+ "' has already been processed.");
+ }
for (Search search : searchList) {
if (rawName.equals(search.getName())) {
throw new IllegalArgumentException("A search definition with a search section called '" + rawName +
@@ -243,7 +247,8 @@ public class SearchBuilder {
DocumentModelBuilder builder = new DocumentModelBuilder(model);
for (Search search : new SearchOrderer().order(searchList)) {
- new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list?
+ new FieldOperationApplierForSearch().process(search);
+ // These two needed for a couple of old unit tests, ideally these are just read from app
process(search, deployLogger, new QueryProfiles(queryProfileRegistry), validate);
built.add(search);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index 7c2d9a3b0ad..9a00ee5bbd0 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -74,11 +74,21 @@ public class DerivedConfiguration {
QueryProfileRegistry queryProfiles,
ImportedModels importedModels) {
Validator.ensureNotNull("Search definition", search);
+ if ( ! search.isProcessed()) {
+ throw new IllegalArgumentException("Search '" + search.getName() + "' not processed.");
+ }
this.search = search;
if ( ! search.isDocumentsOnly()) {
streamingFields = new VsmFields(search);
streamingSummary = new VsmSummary(search);
}
+ if (abstractSearchList != null) {
+ for (Search abstractSearch : abstractSearchList) {
+ if (!abstractSearch.isProcessed()) {
+ throw new IllegalArgumentException("Search '" + search.getName() + "' not processed.");
+ }
+ }
+ }
if ( ! search.isDocumentsOnly()) {
attributeFields = new AttributeFields(search);
summaries = new Summaries(search, deployLogger);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index 279b5334187..c041d5c6a89 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -13,7 +13,6 @@ import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels;
import com.yahoo.searchlib.rankingexpression.parser.ParseException;
import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
-import com.yahoo.tensor.TensorType;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import java.nio.charset.Charset;
@@ -24,7 +23,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* A rank profile derived from a search definition, containing exactly the features available natively in the server
@@ -182,39 +180,32 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) {
if (functions.isEmpty()) return;
-
- List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList());
+ Map<String, ExpressionFunction> expressionFunctions = new LinkedHashMap<>();
+ for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : functions.entrySet()) {
+ expressionFunctions.put(function.getKey(), function.getValue().function());
+ }
Map<String, String> functionProperties = new LinkedHashMap<>();
- functionProperties.putAll(deriveFunctionProperties(functions, functionExpressions));
-
+ functionProperties.putAll(deriveFunctionProperties(expressionFunctions));
if (firstPhaseRanking != null) {
- functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions));
+ functionProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values())));
}
if (secondPhaseRanking != null) {
- functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions));
+ functionProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionFunctions.values())));
}
for (Map.Entry<String, String> e : functionProperties.entrySet()) {
rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue()));
}
- SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties);
+ SerializationContext context = new SerializationContext(expressionFunctions.values(), null, functionProperties);
replaceFunctionSummaryFeatures(context);
}
- private Map<String, String> deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions,
- List<ExpressionFunction> functionExpressions) {
- SerializationContext context = new SerializationContext(functionExpressions);
- for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) {
- String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString();
- context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString);
-
- for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet())
- context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue());
- if (e.getValue().function().returnType().isPresent())
- context.addFunctionTypeSerialization(e.getKey(), e.getValue().function().returnType().get());
- else if (e.getValue().function().arguments().isEmpty())
- throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved");
+ private Map<String, String> deriveFunctionProperties(Map<String, ExpressionFunction> functions) {
+ SerializationContext context = new SerializationContext(functions);
+ for (Map.Entry<String, ExpressionFunction> e : functions.entrySet()) {
+ String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString();
+ context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression);
}
return context.serializedFunctions();
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 15d295736c1..8c8c32389e2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -7,8 +7,10 @@ import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor;
import com.yahoo.vespa.model.container.search.QueryProfiles;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.List;
/**
* Executor of processors. This defines the right order of processor execution.
@@ -73,20 +75,12 @@ public class Processing {
ReferenceFieldsProcessor::new,
FastAccessValidator::new,
ReservedFunctionNames::new,
- RankingExpressionTypeResolver::new,
+ RankingExpressionTypeValidator::new,
// These should be last:
IndexingValidation::new,
IndexingValues::new);
}
- /** Processors of rank profiles only (those who tolerate and so something useful when the search field is null) */
- private Collection<ProcessorFactory> rankProfileProcessors() {
- return Arrays.asList(
- RankProfileTypeSettingsProcessor::new,
- ReservedFunctionNames::new,
- RankingExpressionTypeResolver::new);
- }
-
/**
* Runs all search processors on the given {@link Search} object. These will modify the search object, <b>possibly
* exchanging it with another</b>, as well as its document types.
@@ -99,26 +93,12 @@ public class Processing {
public void process(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) {
Collection<ProcessorFactory> factories = processors();
+ search.process();
factories.stream()
.map(factory -> factory.create(search, deployLogger, rankProfileRegistry, queryProfiles))
.forEach(processor -> processor.process(validate, documentsOnly));
}
- /**
- * Runs rank profiles processors only.
- *
- * @param deployLogger The log to log messages and warnings for application deployment to
- * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
- * @param queryProfiles The query profiles contained in the application this search is part of.
- */
- public void processRankProfiles(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry,
- QueryProfiles queryProfiles, boolean validate, boolean documentsOnly) {
- Collection<ProcessorFactory> factories = rankProfileProcessors();
- factories.stream()
- .map(factory -> factory.create(null, deployLogger, rankProfileRegistry, queryProfiles))
- .forEach(processor -> processor.process(validate, documentsOnly));
- }
-
@FunctionalInterface
public interface ProcessorFactory {
Processor create(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java
index 4c8b5910b78..102d1910360 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java
@@ -7,44 +7,39 @@ import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.searchlib.rankingexpression.Reference;
import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
import com.yahoo.tensor.TensorType;
import com.yahoo.tensor.evaluation.TypeContext;
import com.yahoo.vespa.model.container.search.QueryProfiles;
-import java.util.Map;
-
/**
- * Resolves and assigns types to all functions in a ranking expression, and
- * validates the types of all ranking expressions under a search instance:
+ * Validates the types of all ranking expressions under a search instance:
* Some operators constrain the types of inputs, and first-and second-phase expressions
- * must return scalar values.
- *
- * In addition, the existence of all referred attribute, query and constant
+ * must return scalar values. In addition, the existence of all referred attribute, query and constant
* features is ensured.
*
* @author bratseth
*/
-public class RankingExpressionTypeResolver extends Processor {
+public class RankingExpressionTypeValidator extends Processor {
private final QueryProfileRegistry queryProfiles;
- public RankingExpressionTypeResolver(Search search,
- DeployLogger deployLogger,
- RankProfileRegistry rankProfileRegistry,
- QueryProfiles queryProfiles) {
+ public RankingExpressionTypeValidator(Search search,
+ DeployLogger deployLogger,
+ RankProfileRegistry rankProfileRegistry,
+ QueryProfiles queryProfiles) {
super(search, deployLogger, rankProfileRegistry, queryProfiles);
this.queryProfiles = queryProfiles.getRegistry();
}
@Override
public void process(boolean validate, boolean documentsOnly) {
+ if ( ! validate) return;
if (documentsOnly) return;
for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) {
try {
- resolveTypesIn(profile, validate);
+ validate(profile);
}
catch (IllegalArgumentException e) {
throw new IllegalArgumentException("In " + search + ", " + profile, e);
@@ -52,34 +47,20 @@ public class RankingExpressionTypeResolver extends Processor {
}
}
- /**
- * Resolves the types of all functions in the given profile
- *
- * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types
- */
- private void resolveTypesIn(RankProfile profile, boolean validate) {
- TypeContext<Reference> context = profile.typeContext(queryProfiles);
- for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) {
- if ( ! function.getValue().function().arguments().isEmpty()) continue;
- TensorType type = resolveType(function.getValue().function().getBody(),
- "function '" + function.getKey() + "'",
- context);
- function.getValue().setReturnType(type);
- }
-
- if (validate) {
- profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context));
- ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context);
- ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context);
- }
+ /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */
+ private void validate(RankProfile profile) {
+ TypeContext context = profile.typeContext(queryProfiles);
+ profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context));
+ ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context);
+ ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context);
}
- private TensorType resolveType(RankingExpression expression, String expressionDescription, TypeContext context) {
+ private TensorType ensureValid(RankingExpression expression, String expressionDescription, TypeContext context) {
if (expression == null) return null;
- return resolveType(expression.getRoot(), expressionDescription, context);
+ return ensureValid(expression.getRoot(), expressionDescription, context);
}
- private TensorType resolveType(ExpressionNode expression, String expressionDescription, TypeContext context) {
+ private TensorType ensureValid(ExpressionNode expression, String expressionDescription, TypeContext context) {
TensorType type;
try {
type = expression.type(context);
@@ -94,7 +75,7 @@ public class RankingExpressionTypeResolver extends Processor {
private void ensureValidDouble(RankingExpression expression, String expressionDescription, TypeContext context) {
if (expression == null) return;
- TensorType type = resolveType(expression, expressionDescription, context);
+ TensorType type = ensureValid(expression, expressionDescription, context);
if ( ! type.equals(TensorType.empty))
throw new IllegalArgumentException("The " + expressionDescription + " must produce a double " +
"(a tensor with no dimensions), but produces " + type);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
index 3bde76c1c79..ec4cbdfe58b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -44,7 +44,6 @@ public class RankProfileTypeSettingsProcessor extends Processor {
}
private void processAttributeFields() {
- if (search == null) return; // we're processing global profiles
for (SDField field : search.allConcreteFields()) {
Attribute attribute = field.getAttributes().get(field.getName());
if (attribute != null && attribute.tensorType().isPresent()) {
@@ -54,7 +53,6 @@ public class RankProfileTypeSettingsProcessor extends Processor {
}
private void processImportedFields() {
- if (search == null) return; // we're processing global profiles
Optional<ImportedFields> importedFields = search.importedFields();
if (importedFields.isPresent()) {
importedFields.get().fields().forEach((fieldName, field) -> processImportedField(field));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java
index 29ec26b06d2..620e44bc11a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/Service.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java
@@ -7,7 +7,7 @@ import java.util.HashMap;
import java.util.Optional;
/**
- * Representation of a markProcessed which runs a service
+ * Representation of a process which runs a service
*
* @author gjoranv
*/
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index 13304ea10ee..4b70b1b5ae2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -32,9 +32,7 @@ import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.RankingConstants;
import com.yahoo.searchdefinition.derived.AttributeFields;
import com.yahoo.searchdefinition.derived.RankProfileList;
-import com.yahoo.searchdefinition.processing.Processing;
import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
-import com.yahoo.vespa.model.container.search.QueryProfiles;
import com.yahoo.vespa.model.ml.ConvertedModel;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel;
@@ -170,7 +168,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
createGlobalRankProfiles(deployState.getImportedModels(),
deployState.rankProfileRegistry(),
- deployState.getQueryProfiles());
+ deployState.getQueryProfiles().getRegistry());
this.rankProfileList = new RankProfileList(null, // null search -> global
rankingConstants,
AttributeFields.empty,
@@ -221,23 +219,26 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
/** Adds generic application specific clusters of services */
private void addServiceClusters(ApplicationPackage app, VespaModelBuilder builder) {
- serviceClusters.addAll(builder.getClusters(app, this));
+ for (ServiceCluster sc : builder.getClusters(app, this))
+ serviceClusters.add(sc);
}
/**
- * Creates a rank profile not attached to any search definition, for each imported model in the application package,
- * and adds it to the given rank profile registry.
+ * Creates a rank profile not attached to any search definition, for each imported model in the application package
*/
- private void createGlobalRankProfiles(ImportedModels importedModels,
- RankProfileRegistry rankProfileRegistry,
- QueryProfiles queryProfiles) {
+ private ImmutableList<RankProfile> createGlobalRankProfiles(ImportedModels importedModels,
+ RankProfileRegistry rankProfileRegistry,
+ QueryProfileRegistry queryProfiles) {
+ List<RankProfile> profiles = new ArrayList<>();
if ( ! importedModels.all().isEmpty()) { // models/ directory is available
for (ImportedModel model : importedModels.all()) {
RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry);
rankProfileRegistry.add(profile);
ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()),
- model.name(), profile, queryProfiles.getRegistry(), model);
- convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false));
+ model.name(), profile, queryProfiles, model);
+ for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) {
+ profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false);
+ }
}
}
else { // generated and stored model information may be available instead
@@ -247,12 +248,12 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry);
rankProfileRegistry.add(profile);
ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile);
- convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false));
+ for (Map.Entry<String, RankingExpression> entry : convertedModel.expressions().entrySet()) {
+ profile.addFunction(new ExpressionFunction(entry.getKey(), entry.getValue()), false);
+ }
}
}
- new Processing().processRankProfiles(deployState.getDeployLogger(),
- rankProfileRegistry,
- queryProfiles, true, false);
+ return ImmutableList.copyOf(profiles);
}
/** Returns the global rank profiles as a rank profile list */
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
index fb0109ed32e..adf5c81283e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java
@@ -48,7 +48,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -68,14 +67,14 @@ public class ConvertedModel {
private final ModelName modelName;
private final String modelDescription;
- private final ImmutableMap<String, ExpressionFunction> expressions;
+ private final ImmutableMap<String, RankingExpression> expressions;
/** The source importedModel, or empty if this was created from a stored converted model */
private final Optional<ImportedModel> sourceModel;
private ConvertedModel(ModelName modelName,
String modelDescription,
- Map<String, ExpressionFunction> expressions,
+ Map<String, RankingExpression> expressions,
Optional<ImportedModel> sourceModel) {
this.modelName = modelName;
this.modelDescription = modelDescription;
@@ -133,23 +132,23 @@ public class ConvertedModel {
* if signatures are used, or the expression name if signatures are not used and there are multiple
* expressions, and the second is the output name if signature names are used.
*/
- public Map<String, ExpressionFunction> expressions() { return expressions; }
+ public Map<String, RankingExpression> expressions() { return expressions; }
/**
* Returns the expression matching the given arguments.
*/
public ExpressionNode expression(FeatureArguments arguments, RankProfileTransformContext context) {
- ExpressionFunction expression = selectExpression(arguments);
- if (sourceModel.isPresent()) // we should verify
- verifyInputs(expression.getBody(), sourceModel.get(), context.rankProfile(), context.queryProfiles());
- return expression.getBody().getRoot();
+ RankingExpression expression = selectExpression(arguments);
+ if (sourceModel.isPresent()) // we can verify
+ verifyRequiredFunctions(expression, sourceModel.get(), context.rankProfile(), context.queryProfiles());
+ return expression.getRoot();
}
- private ExpressionFunction selectExpression(FeatureArguments arguments) {
+ private RankingExpression selectExpression(FeatureArguments arguments) {
if (expressions.isEmpty())
throw new IllegalArgumentException("No expressions available in " + this);
- ExpressionFunction expression = expressions.get(arguments.toName());
+ RankingExpression expression = expressions.get(arguments.toName());
if (expression != null) return expression;
if ( ! arguments.signature().isPresent()) {
@@ -159,7 +158,7 @@ public class ConvertedModel {
}
if ( ! arguments.output().isPresent()) {
- List<Map.Entry<String, ExpressionFunction>> entriesWithTheRightPrefix =
+ List<Map.Entry<String, RankingExpression>> entriesWithTheRightPrefix =
expressions.entrySet().stream().filter(entry -> entry.getKey().startsWith(arguments.signature().get() + ".")).collect(Collectors.toList());
if (entriesWithTheRightPrefix.size() < 1)
throw new IllegalArgumentException("No expressions named '" + arguments.signature().get() +
@@ -180,10 +179,10 @@ public class ConvertedModel {
// ----------------------- Static model conversion/storage below here
- private static Map<String, ExpressionFunction> convertAndStore(ImportedModel model,
- RankProfile profile,
- QueryProfileRegistry queryProfiles,
- ModelStore store) {
+ private static Map<String, RankingExpression> convertAndStore(ImportedModel model,
+ RankProfile profile,
+ QueryProfileRegistry queryProfiles,
+ ModelStore store) {
// Add constants
Set<String> constantsReplacedByFunctions = new HashSet<>();
model.smallConstants().forEach((k, v) -> transformSmallConstant(store, profile, k, v));
@@ -194,8 +193,8 @@ public class ConvertedModel {
addGeneratedFunctions(model, profile);
// Add expressions
- Map<String, ExpressionFunction> expressions = new HashMap<>();
- for (Pair<String, ExpressionFunction> output : model.outputExpressions()) {
+ Map<String, RankingExpression> expressions = new HashMap<>();
+ for (Pair<String, RankingExpression> output : model.outputExpressions()) {
addExpression(output.getSecond(), output.getFirst(),
constantsReplacedByFunctions,
model, store, profile, queryProfiles,
@@ -211,21 +210,21 @@ public class ConvertedModel {
return expressions;
}
- private static void addExpression(ExpressionFunction expression,
+ private static void addExpression(RankingExpression expression,
String expressionName,
Set<String> constantsReplacedByFunctions,
ImportedModel model,
ModelStore store,
RankProfile profile,
QueryProfileRegistry queryProfiles,
- Map<String, ExpressionFunction> expressions) {
- expression = expression.withBody(replaceConstantsByFunctions(expression.getBody(), constantsReplacedByFunctions));
- reduceBatchDimensions(expression.getBody(), model, profile, queryProfiles);
+ Map<String, RankingExpression> expressions) {
+ expression = replaceConstantsByFunctions(expression, constantsReplacedByFunctions);
+ reduceBatchDimensions(expression, model, profile, queryProfiles);
store.writeExpression(expressionName, expression);
expressions.put(expressionName, expression);
}
- private static Map<String, ExpressionFunction> convertStored(ModelStore store, RankProfile profile) {
+ private static Map<String, RankingExpression> convertStored(ModelStore store, RankProfile profile) {
for (Pair<String, Tensor> constant : store.readSmallConstants())
profile.addConstant(constant.getFirst(), asValue(constant.getSecond()));
@@ -291,15 +290,15 @@ public class ConvertedModel {
}
/**
- * Verify that the inputs declared in the given expression exists in the given rank profile as functions,
- * and return tensors of the correct types.
+ * Verify that the functions referred in the given expression exists in the given rank profile,
+ * and return tensors of the types specified in requiredFunctions.
*/
- private static void verifyInputs(RankingExpression expression, ImportedModel model,
- RankProfile profile, QueryProfileRegistry queryProfiles) {
+ private static void verifyRequiredFunctions(RankingExpression expression, ImportedModel model,
+ RankProfile profile, QueryProfileRegistry queryProfiles) {
Set<String> functionNames = new HashSet<>();
addFunctionNamesIn(expression.getRoot(), functionNames, model);
for (String functionName : functionNames) {
- TensorType requiredType = model.inputs().get(functionName);
+ TensorType requiredType = model.requiredFunctions().get(functionName);
if (requiredType == null) continue; // Not a required function
RankProfile.RankingExpressionFunction rankingExpressionFunction = profile.getFunctions().get(functionName);
@@ -376,7 +375,7 @@ public class ConvertedModel {
List<ExpressionNode> children = ((TensorFunctionNode)node).children();
if (children.size() == 1 && children.get(0) instanceof ReferenceNode) {
ReferenceNode referenceNode = (ReferenceNode) children.get(0);
- if (model.inputs().containsKey(referenceNode.getName())) {
+ if (model.requiredFunctions().containsKey(referenceNode.getName())) {
return reduceBatchDimensionExpression(tensorFunction, typeContext);
}
}
@@ -384,7 +383,7 @@ public class ConvertedModel {
}
if (node instanceof ReferenceNode) {
ReferenceNode referenceNode = (ReferenceNode) node;
- if (model.inputs().containsKey(referenceNode.getName())) {
+ if (model.requiredFunctions().containsKey(referenceNode.getName())) {
return reduceBatchDimensionExpression(TensorFunctionNode.wrapArgument(node), typeContext);
}
}
@@ -452,8 +451,7 @@ public class ConvertedModel {
Set<String> constantsReplacedByFunctions) {
if (constantsReplacedByFunctions.isEmpty()) return expression;
return new RankingExpression(expression.getName(),
- replaceConstantsByFunctions(expression.getRoot(),
- constantsReplacedByFunctions));
+ replaceConstantsByFunctions(expression.getRoot(), constantsReplacedByFunctions));
}
private static ExpressionNode replaceConstantsByFunctions(ExpressionNode node, Set<String> constantsReplacedByFunctions) {
@@ -526,21 +524,19 @@ public class ConvertedModel {
* @param name the name of this ranking expression - may have 1-3 parts separated by dot where the first part
* is always the model name
*/
- void writeExpression(String name, ExpressionFunction expression) {
- StringBuilder b = new StringBuilder(expression.getBody().getRoot().toString());
- for (Map.Entry<String, TensorType> input : expression.argumentTypes().entrySet())
- b.append('\n').append(input.getKey()).append('\t').append(input.getValue());
- application.getFile(modelFiles.expressionPath(name)).writeFile(new StringReader(b.toString()));
+ void writeExpression(String name, RankingExpression expression) {
+ application.getFile(modelFiles.expressionPath(name))
+ .writeFile(new StringReader(expression.getRoot().toString()));
}
- Map<String, ExpressionFunction> readExpressions() {
- Map<String, ExpressionFunction> expressions = new HashMap<>();
+ Map<String, RankingExpression> readExpressions() {
+ Map<String, RankingExpression> expressions = new HashMap<>();
ApplicationFile expressionPath = application.getFile(modelFiles.expressionsPath());
if ( ! expressionPath.exists() || ! expressionPath.isDirectory()) return Collections.emptyMap();
for (ApplicationFile expressionFile : expressionPath.listFiles()) {
- try (BufferedReader reader = new BufferedReader(expressionFile.createReader())){
+ try (Reader reader = new BufferedReader(expressionFile.createReader())){
String name = expressionFile.getPath().getName();
- expressions.put(name, readExpression(name, reader));
+ expressions.put(name, new RankingExpression(name, reader));
}
catch (IOException e) {
throw new UncheckedIOException("Failed reading " + expressionFile.getPath(), e);
@@ -552,22 +548,8 @@ public class ConvertedModel {
return expressions;
}
- private ExpressionFunction readExpression(String name, BufferedReader reader)
- throws IOException, ParseException {
- // First line is expression
- RankingExpression expression = new RankingExpression(name, reader.readLine());
- // Next lines are inputs on the format name\ttensorTypeSpec
- Map<String, TensorType> inputs = new LinkedHashMap<>();
- String line;
- while (null != (line = reader.readLine())) {
- String[] parts = line.split("\t");
- inputs.put(parts[0], TensorType.fromSpec(parts[1]));
- }
- return new ExpressionFunction(name, new ArrayList<>(inputs.keySet()), expression, inputs, Optional.empty());
- }
-
/** Adds this function expression to the application package so it can be read later. */
- public void writeFunction(String name, RankingExpression expression) {
+ void writeFunction(String name, RankingExpression expression) {
application.getFile(modelFiles.functionsPath()).appendFile(name + "\t" +
expression.getRoot().toString() + "\n");
}
@@ -579,20 +561,20 @@ public class ConvertedModel {
if ( ! file.exists()) return Collections.emptyList();
List<Pair<String, RankingExpression>> functions = new ArrayList<>();
- try (BufferedReader reader = new BufferedReader(file.createReader())) {
- String line;
- while (null != (line = reader.readLine())) {
- String[] parts = line.split("\t");
- String name = parts[0];
- try {
- RankingExpression expression = new RankingExpression(parts[0], parts[1]);
- functions.add(new Pair<>(name, expression));
- } catch (ParseException e) {
- throw new IllegalStateException("Could not parse " + name, e);
- }
+ BufferedReader reader = new BufferedReader(file.createReader());
+ String line;
+ while (null != (line = reader.readLine())) {
+ String[] parts = line.split("\t");
+ String name = parts[0];
+ try {
+ RankingExpression expression = new RankingExpression(parts[0], parts[1]);
+ functions.add(new Pair<>(name, expression));
+ }
+ catch (ParseException e) {
+ throw new IllegalStateException("Could not parse " + name, e);
}
- return functions;
}
+ return functions;
}
catch (IOException e) {
throw new UncheckedIOException(e);
diff --git a/config-model/src/test/derived/gemini2/gemini.sd b/config-model/src/test/derived/gemini2/gemini.sd
index 8a570e58fa8..01e20c1b30a 100644
--- a/config-model/src/test/derived/gemini2/gemini.sd
+++ b/config-model/src/test/derived/gemini2/gemini.sd
@@ -2,12 +2,6 @@
search gemini {
document gemini {
- field right type string {
- indexing: attribute
- }
- field wrong type string {
- indexing: attribute
- }
}
rank-profile test {
diff --git a/config-model/src/test/derived/gemini2/rank-profiles.cfg b/config-model/src/test/derived/gemini2/rank-profiles.cfg
index 2b73e923c88..aa4f963320d 100644
--- a/config-model/src/test/derived/gemini2/rank-profiles.cfg
+++ b/config-model/src/test/derived/gemini2/rank-profiles.cfg
@@ -21,13 +21,9 @@ rankprofile[].fef.property[].name "rankingExpression(wrapper1@2d437c13405e61d6).
rankprofile[].fef.property[].value "rankingExpression(wrapper2@2d437c13405e61d6)"
rankprofile[].fef.property[].name "rankingExpression(toplevel).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(wrapper1@2d437c13405e61d6)"
-rankprofile[].fef.property[].name "rankingExpression(toplevel).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(wrapper2@8fc8470e911f253f).rankingScript"
rankprofile[].fef.property[].value "attribute(wrong)"
rankprofile[].fef.property[].name "rankingExpression(wrapper1@8fc8470e911f253f).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(wrapper2@8fc8470e911f253f)"
rankprofile[].fef.property[].name "rankingExpression(interfering).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(wrapper1@8fc8470e911f253f)"
-rankprofile[].fef.property[].name "rankingExpression(interfering).type"
-rankprofile[].fef.property[].value "tensor()"
diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg
index d109ca4f0ec..9629ad863d4 100644
--- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg
+++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg
@@ -128,8 +128,6 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript
rankprofile[].fef.property[].value "4 * (var1 + var2)"
rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript"
rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
-rankprofile[].fef.property[].name "rankingExpression(myfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86).rankingScript"
rankprofile[].fef.property[].value "4 * (match + rankBoost)"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
@@ -147,16 +145,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript
rankprofile[].fef.property[].value "4 * (var1 + var2)"
rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript"
rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
-rankprofile[].fef.property[].name "rankingExpression(myfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript"
rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript"
rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript"
rankprofile[].fef.property[].value "4 * (match + match)"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
@@ -172,15 +164,11 @@ rankprofile[].fef.property[].value "rankingExpression(mysummaryfeature)"
rankprofile[].name "macros3"
rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).rankingScript"
rankprofile[].fef.property[].value "5"
-rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "vespa.summary.feature"
rankprofile[].fef.property[].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))"
rankprofile[].name "macros3-inherited"
rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).rankingScript"
rankprofile[].fef.property[].value "5"
-rankprofile[].fef.property[].name "rankingExpression(onlyusedinsummaryfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "vespa.summary.feature"
rankprofile[].fef.property[].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))"
rankprofile[].name "macros-inherited"
@@ -190,16 +178,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript
rankprofile[].fef.property[].value "4 * (var1 + var2)"
rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript"
rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
-rankprofile[].fef.property[].name "rankingExpression(myfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript"
rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript"
rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript"
rankprofile[].fef.property[].value "4 * (match + match)"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
@@ -221,16 +203,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript
rankprofile[].fef.property[].value "4 * (var1 + var2)"
rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript"
rankprofile[].fef.property[].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
-rankprofile[].fef.property[].name "rankingExpression(myfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript"
rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript"
rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript"
rankprofile[].fef.property[].value "4 * (match + match)"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
@@ -252,16 +228,10 @@ rankprofile[].fef.property[].name "rankingExpression(fourtimessum).rankingScript
rankprofile[].fef.property[].value "4 * (var1 + var2)"
rankprofile[].fef.property[].name "rankingExpression(myfeature).rankingScript"
rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(myfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).rankingScript"
rankprofile[].fef.property[].value "80 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).rankingScript"
rankprofile[].fef.property[].value "71 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(mysummaryfeature2).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
rankprofile[].fef.property[].value "rankingExpression(firstphase)"
rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
@@ -279,14 +249,8 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
-rankprofile[].fef.property[].name "rankingExpression(m2).type"
-rankprofile[].fef.property[].value "tensor()"
-rankprofile[].fef.property[].name "rankingExpression(m1).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness"
-rankprofile[].fef.property[].name "rankingExpression(m4).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "vespa.rank.secondphase"
rankprofile[].fef.property[].value "rankingExpression(secondphase)"
rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
@@ -296,18 +260,10 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
-rankprofile[].fef.property[].name "rankingExpression(m2).type"
-rankprofile[].fef.property[].value "tensor()"
-rankprofile[].fef.property[].name "rankingExpression(m1).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(m4).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript"
rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))"
-rankprofile[].fef.property[].name "rankingExpression(m3).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "vespa.rank.secondphase"
rankprofile[].fef.property[].value "rankingExpression(secondphase)"
rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
@@ -317,14 +273,8 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
-rankprofile[].fef.property[].name "rankingExpression(m2).type"
-rankprofile[].fef.property[].value "tensor()"
-rankprofile[].fef.property[].name "rankingExpression(m1).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
rankprofile[].fef.property[].value "703 * fieldMatch(fromfile).completeness"
-rankprofile[].fef.property[].name "rankingExpression(m4).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "vespa.rank.secondphase"
rankprofile[].fef.property[].value "rankingExpression(secondphase)"
rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
@@ -334,22 +284,12 @@ rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript"
rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness"
rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(m1) * 67"
-rankprofile[].fef.property[].name "rankingExpression(m2).type"
-rankprofile[].fef.property[].value "tensor()"
-rankprofile[].fef.property[].name "rankingExpression(m1).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript"
rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness"
-rankprofile[].fef.property[].name "rankingExpression(m4).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript"
rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))"
-rankprofile[].fef.property[].name "rankingExpression(m3).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "rankingExpression(m5).rankingScript"
rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(m4))"
-rankprofile[].fef.property[].name "rankingExpression(m5).type"
-rankprofile[].fef.property[].value "tensor()"
rankprofile[].fef.property[].name "vespa.rank.secondphase"
rankprofile[].fef.property[].value "rankingExpression(secondphase)"
rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript"
diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg
index 471343da63c..cb496c06367 100644
--- a/config-model/src/test/derived/tensor/rank-profiles.cfg
+++ b/config-model/src/test/derived/tensor/rank-profiles.cfg
@@ -43,14 +43,10 @@ rankprofile[].fef.property[].value "tensor(x{})"
rankprofile[].fef.property[].name "vespa.type.attribute.f4"
rankprofile[].fef.property[].value "tensor(x[10],y[20])"
rankprofile[].name "profile3"
-rankprofile[].fef.property[].name "rankingExpression(joinedtensors).rankingScript"
-rankprofile[].fef.property[].value "tensor(i[10])(i) * attribute(f4)"
-rankprofile[].fef.property[].name "rankingExpression(joinedtensors).type"
-rankprofile[].fef.property[].value "tensor(i[10],x[10],y[20])"
rankprofile[].fef.property[].name "vespa.rank.firstphase"
rankprofile[].fef.property[].value "rankingExpression(firstphase)"
rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
-rankprofile[].fef.property[].value "reduce(rankingExpression(joinedtensors), sum)"
+rankprofile[].fef.property[].value "reduce(tensor(i[10])(i) * attribute(f4), sum)"
rankprofile[].fef.property[].name "vespa.type.attribute.f2"
rankprofile[].fef.property[].value "tensor(x[2],y[])"
rankprofile[].fef.property[].name "vespa.type.attribute.f3"
diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd
index 54463448e28..5792a7997f8 100644
--- a/config-model/src/test/derived/tensor/tensor.sd
+++ b/config-model/src/test/derived/tensor/tensor.sd
@@ -36,11 +36,7 @@ search tensor {
rank-profile profile3 {
first-phase {
- expression: sum(joinedtensors)
- }
-
- function joinedtensors() {
- expression: tensor(i[10])(i) * attribute(f4)
+ expression: sum(tensor(i[10])(i) * attribute(f4))
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
index 2c1f4c8ecb6..150469cc928 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
@@ -82,7 +82,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
new ImportedModels(),
new AttributeFields(s)).configProperties();
assertEquals("(rankingExpression(foo).rankingScript,14.0)", rankProperties.get(0).toString());
- assertEquals("(rankingExpression(firstphase).rankingScript,16.6)", rankProperties.get(3).toString());
+ assertEquals("(rankingExpression(firstphase).rankingScript,16.6)", rankProperties.get(2).toString());
}
@Test
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java
index 0ff8a5cc7ca..17bebcba70e 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java
@@ -40,7 +40,7 @@ public class RankingExpressionLoopDetectionTestCase {
fail("Excepted exception");
}
catch (IllegalArgumentException e) {
- assertEquals("In search definition 'test', rank profile 'test': The function 'foo' is invalid: Invocation loop: foo -> foo",
+ assertEquals("In search definition 'test', rank profile 'test': The first-phase expression is invalid: Invocation loop: foo -> foo",
Exceptions.toMessageString(e));
}
}
@@ -75,7 +75,7 @@ public class RankingExpressionLoopDetectionTestCase {
fail("Excepted exception");
}
catch (IllegalArgumentException e) {
- assertEquals("In search definition 'test', rank profile 'test': The function 'foo' is invalid: Invocation loop: arg(5) -> foo -> arg(5)",
+ assertEquals("In search definition 'test', rank profile 'test': The first-phase expression is invalid: Invocation loop: foo -> arg(5) -> foo",
Exceptions.toMessageString(e));
}
}
@@ -110,7 +110,7 @@ public class RankingExpressionLoopDetectionTestCase {
fail("Excepted exception");
}
catch (IllegalArgumentException e) {
- assertEquals("In search definition 'test', rank profile 'test': The function 'foo' is invalid: Invocation loop: arg(foo) -> foo -> arg(foo)",
+ assertEquals("In search definition 'test', rank profile 'test': The first-phase expression is invalid: Invocation loop: foo -> arg(foo) -> foo",
Exceptions.toMessageString(e));
}
}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
index 4f99922a422..e15d4075b19 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
@@ -211,16 +211,12 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
censorBindingHash(testRankProperties.get(1).toString()));
assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))",
censorBindingHash(testRankProperties.get(2).toString()));
- assertEquals("(rankingExpression(hidden_layer).type,tensor(x[]))",
- censorBindingHash(testRankProperties.get(3).toString()));
assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))",
- testRankProperties.get(4).toString());
- assertEquals("(rankingExpression(final_layer).type,tensor(x[]))",
- testRankProperties.get(5).toString());
+ testRankProperties.get(3).toString());
assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))",
- testRankProperties.get(6).toString());
+ testRankProperties.get(4).toString());
assertEquals("(rankingExpression(secondphase).rankingScript,reduce(rankingExpression(final_layer), sum))",
- testRankProperties.get(7).toString());
+ testRankProperties.get(5).toString());
}
private QueryProfileRegistry queryProfileWith(String field, String type) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java
index 1b917b6f3a3..0d8cbbf2e6a 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java
@@ -19,7 +19,7 @@ import static org.junit.Assert.fail;
/**
* @author bratseth
*/
-public class RankingExpressionTypeResolverTestCase {
+public class RankingExpressionTypeValidatorTestCase {
@Test
public void tensorFirstPhaseMustProduceDouble() throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
index 7d62bc5089d..fd048737b43 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
@@ -41,7 +41,7 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase {
new QueryProfileRegistry(),
new ImportedModels(),
new AttributeFields(search)).configProperties();
- assertEquals(7, rankProperties.size());
+ assertEquals(6, rankProperties.size());
assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(0).getFirst());
assertEquals("var1 * var2 + 890", rankProperties.get(0).getSecond());
@@ -49,17 +49,14 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase {
assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(1).getFirst());
assertEquals("78 + closeness(distance)", rankProperties.get(1).getSecond());
- assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(6).getFirst());
- assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(6).getSecond());
+ assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst());
+ assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond());
- assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(4).getFirst());
- assertEquals("7 * 8 + 890", rankProperties.get(4).getSecond());
+ assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(3).getFirst());
+ assertEquals("7 * 8 + 890", rankProperties.get(3).getSecond());
- assertEquals("rankingExpression(artistmatch).type", rankProperties.get(2).getFirst());
- assertEquals("tensor()", rankProperties.get(2).getSecond());
-
- assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(3).getFirst());
- assertEquals("4 * 5 + 890", rankProperties.get(3).getSecond());
+ assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(2).getFirst());
+ assertEquals("4 * 5 + 890", rankProperties.get(2).getSecond());
}
@Test(expected = IllegalArgumentException.class)
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index 6e3a227e2a9..8e721dbe503 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -58,8 +58,8 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
"max(attribute(tensor_field_1),x)");
assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)",
"1 + max(attribute(tensor_field_1),x)");
- assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),attribute(tensor_field_1))",
- "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),attribute(tensor_field_1))");
+ assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)",
+ "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)");
assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)",
"max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)");
assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)",
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java
index 4dbec304bde..d74f29a3b77 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java
@@ -147,7 +147,7 @@ public class AsyncExecution {
}
private static <T> Future<T> getFuture(Callable<T> callable) {
- FutureTask<T> future = new FutureTask<>(callable);
+ final FutureTask<T> future = new FutureTask<>(callable);
getExecutor().execute(future);
return future;
}
diff --git a/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java b/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java
index 2c40165f8e5..eac96e9b408 100644
--- a/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java
+++ b/processing/src/main/java/com/yahoo/processing/execution/AsyncExecution.java
@@ -59,14 +59,14 @@ public class AsyncExecution {
/**
* Create an async execution of a single processor
*/
- public AsyncExecution(Processor processor, Execution parent) {
+ public AsyncExecution(final Processor processor, Execution parent) {
this(new Execution(processor, parent));
}
/**
* Create an async execution of a chain
*/
- public AsyncExecution(Chain<? extends Processor> chain, Execution parent) {
+ public AsyncExecution(final Chain<? extends Processor> chain, Execution parent) {
this(new Execution(chain, parent));
}
@@ -81,7 +81,7 @@ public class AsyncExecution {
*
* @param execution the execution from which the state of this is created
*/
- public AsyncExecution(Execution execution) {
+ public AsyncExecution(final Execution execution) {
this.execution = new Execution(execution);
}
@@ -89,7 +89,7 @@ public class AsyncExecution {
* Performs an async processing. Note that the given request cannot be simultaneously
* used in multiple such processings - a clone must be created for each.
*/
- public FutureResponse process(Request request) {
+ public FutureResponse process(final Request request) {
return getFutureResponse(new Callable<Response>() {
@Override
public Response call() {
@@ -99,13 +99,13 @@ public class AsyncExecution {
}
private static <T> Future<T> getFuture(final Callable<T> callable) {
- FutureTask<T> future = new FutureTask<>(callable);
+ final FutureTask<T> future = new FutureTask<>(callable);
executorMain.execute(future);
return future;
}
- private FutureResponse getFutureResponse(Callable<Response> callable, Request request) {
- FutureResponse future = new FutureResponse(callable, execution, request);
+ private FutureResponse getFutureResponse(final Callable<Response> callable, final Request request) {
+ final FutureResponse future = new FutureResponse(callable, execution, request);
executorMain.execute(future.delegate());
return future;
}
@@ -118,15 +118,15 @@ public class AsyncExecution {
* @return the list of responses in the same order as returned from the task collection
*/
// Note that this may also be achieved using guava Futures. Not sure if this should be deprecated because of it.
- public static List<Response> waitForAll(Collection<FutureResponse> tasks, long timeout) {
+ public static List<Response> waitForAll(final Collection<FutureResponse> tasks, final long timeout) {
// Copy the list in case it is modified while we are waiting
- List<FutureResponse> workingTasks = new ArrayList<>(tasks);
+ final List<FutureResponse> workingTasks = new ArrayList<>(tasks);
@SuppressWarnings({"rawtypes", "unchecked"})
- Future task = getFuture(new Callable() {
+ final Future task = getFuture(new Callable() {
@Override
public List<Future> call() {
- for (FutureResponse task : workingTasks) {
+ for (final FutureResponse task : workingTasks) {
task.get();
}
return null;
@@ -135,12 +135,12 @@ public class AsyncExecution {
try {
task.get(timeout, TimeUnit.MILLISECONDS);
- } catch (TimeoutException | InterruptedException | ExecutionException e) {
+ } catch (final TimeoutException | InterruptedException | ExecutionException e) {
// Handle timeouts below
}
- List<Response> responses = new ArrayList<>(tasks.size());
- for (FutureResponse future : workingTasks)
+ final List<Response> responses = new ArrayList<>(tasks.size());
+ for (final FutureResponse future : workingTasks)
responses.add(getTaskResponse(future));
return responses;
}
@@ -153,4 +153,5 @@ public class AsyncExecution {
}
}
+
}
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
index f6502a9801d..da34ab8822d 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
@@ -2,11 +2,8 @@
package com.yahoo.searchlib.rankingexpression;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.yahoo.log.event.Collection;
import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
-import com.yahoo.tensor.TensorType;
import com.yahoo.text.Utf8;
import java.security.MessageDigest;
@@ -16,14 +13,9 @@ import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
/**
- * A function defined by a ranking expression, optionally containing type information
- * for inputs and outputs.
- *
- * Immutable, but note that ranking expressions are *not* immutable.
+ * A function defined by a ranking expression
*
* @author Simon Thoresen Hult
* @author bratseth
@@ -32,13 +24,8 @@ public class ExpressionFunction {
private final String name;
private final ImmutableList<String> arguments;
-
- /** Types of the inputs, if known. The keys here is any subset (including empty and identity) of the argument list */
- private final ImmutableMap<String, TensorType> argumentTypes;
private final RankingExpression body;
- private final Optional<TensorType> returnType;
-
/**
* Constructs a new function with no arguments
*
@@ -57,18 +44,9 @@ public class ExpressionFunction {
* @param body the ranking expression that defines this function
*/
public ExpressionFunction(String name, List<String> arguments, RankingExpression body) {
- this(name, arguments, body, ImmutableMap.of(), Optional.empty());
- }
-
- public ExpressionFunction(String name, List<String> arguments, RankingExpression body,
- Map<String, TensorType> argumentTypes, Optional<TensorType> returnType) {
- this.name = Objects.requireNonNull(name, "name cannot be null");
+ this.name = name;
this.arguments = arguments==null ? ImmutableList.of() : ImmutableList.copyOf(arguments);
- this.body = Objects.requireNonNull(body, "body cannot be null");
- if ( ! this.arguments.containsAll(argumentTypes.keySet()))
- throw new IllegalArgumentException("Argument type keys must be a subset of the argument keys");
- this.argumentTypes = ImmutableMap.copyOf(argumentTypes);
- this.returnType = Objects.requireNonNull(returnType, "returnType cannot be null");
+ this.body = body;
}
public String getName() { return name; }
@@ -78,27 +56,9 @@ public class ExpressionFunction {
public RankingExpression getBody() { return body; }
- /** Returns the types of the arguments of this, if specified. The keys of this may be any subset of the arguments */
- public Map<String, TensorType> argumentTypes() { return argumentTypes; }
-
- /** Returns the return type of this, or empty if not specified */
- public Optional<TensorType> returnType() { return returnType; }
-
- public ExpressionFunction withName(String name) {
- return new ExpressionFunction(name, arguments, body, argumentTypes, returnType);
- }
-
/** Returns a copy of this with the body changed to the given value */
public ExpressionFunction withBody(RankingExpression body) {
- return new ExpressionFunction(name, arguments, body, argumentTypes, returnType);
- }
-
- public ExpressionFunction withReturnType(TensorType returnType) {
- return new ExpressionFunction(name, arguments, body, argumentTypes, Optional.of(returnType));
- }
-
- public ExpressionFunction withArgumentTypes(Map<String, TensorType> argumentTypes) {
- return new ExpressionFunction(name, arguments, body, argumentTypes, returnType);
+ return new ExpressionFunction(name, arguments, body);
}
/**
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java
index 9ff391a5cfe..282a4c5e0a9 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ImportedModel.java
@@ -1,22 +1,15 @@
package com.yahoo.searchlib.rankingexpression.integration.ml;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import com.yahoo.collections.Pair;
-import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
import java.util.regex.Pattern;
/**
@@ -33,11 +26,12 @@ public class ImportedModel {
private final String source;
private final Map<String, Signature> signatures = new HashMap<>();
- private final Map<String, TensorType> inputs = new HashMap<>();
+ private final Map<String, TensorType> arguments = new HashMap<>();
private final Map<String, Tensor> smallConstants = new HashMap<>();
private final Map<String, Tensor> largeConstants = new HashMap<>();
private final Map<String, RankingExpression> expressions = new HashMap<>();
private final Map<String, RankingExpression> functions = new HashMap<>();
+ private final Map<String, TensorType> requiredFunctions = new HashMap<>();
/**
* Creates a new imported model.
@@ -55,11 +49,11 @@ public class ImportedModel {
/** Returns the name of this model, which can only contain the characters in [A-Za-z0-9_] */
public String name() { return name; }
- /** Returns the source path (directory or file) of this model */
+ /** Returns the source path (directiry or file) of this model */
public String source() { return source; }
- /** Returns an immutable map of the inputs of this */
- public Map<String, TensorType> inputs() { return Collections.unmodifiableMap(inputs); }
+ /** Returns an immutable map of the arguments ("Placeholders") of this */
+ public Map<String, TensorType> arguments() { return Collections.unmodifiableMap(arguments); }
/**
* Returns an immutable map of the small constants of this.
@@ -77,7 +71,7 @@ public class ImportedModel {
/**
* Returns an immutable map of the expressions of this - corresponding to graph nodes
- * which are not Inputs/Placeholders or Variables (which instead become respectively inputs and constants).
+ * which are not Inputs/Placeholders or Variables (which instead become respectively arguments and constants).
* Note that only nodes recursively referenced by a placeholder/input are added.
*/
public Map<String, RankingExpression> expressions() { return Collections.unmodifiableMap(expressions); }
@@ -88,6 +82,9 @@ public class ImportedModel {
*/
public Map<String, RankingExpression> functions() { return Collections.unmodifiableMap(functions); }
+ /** Returns an immutable map of the functions that must be provided by the environment running this model */
+ public Map<String, TensorType> requiredFunctions() { return Collections.unmodifiableMap(requiredFunctions); }
+
/** Returns an immutable map of the signatures of this */
public Map<String, Signature> signatures() { return Collections.unmodifiableMap(signatures); }
@@ -99,11 +96,12 @@ public class ImportedModel {
/** Convenience method for returning a default signature */
Signature defaultSignature() { return signature(defaultSignatureName); }
- void input(String name, TensorType argumentType) { inputs.put(name, argumentType); }
+ void argument(String name, TensorType argumentType) { arguments.put(name, argumentType); }
void smallConstant(String name, Tensor constant) { smallConstants.put(name, constant); }
void largeConstant(String name, Tensor constant) { largeConstants.put(name, constant); }
void expression(String name, RankingExpression expression) { expressions.put(name, expression); }
void function(String name, RankingExpression expression) { functions.put(name, expression); }
+ void requiredFunction(String name, TensorType type) { requiredFunctions.put(name, type); }
/**
* Returns all the output expressions of this indexed by name. The names consist of one or two parts
@@ -111,39 +109,24 @@ public class ImportedModel {
* if signatures are used, or the expression name if signatures are not used and there are multiple
* expressions, and the second is the output name if signature names are used.
*/
- public List<Pair<String, ExpressionFunction>> outputExpressions() {
- List<Pair<String, ExpressionFunction>> expressions = new ArrayList<>();
+ public List<Pair<String, RankingExpression>> outputExpressions() {
+ List<Pair<String, RankingExpression>> expressions = new ArrayList<>();
for (Map.Entry<String, Signature> signatureEntry : signatures().entrySet()) {
for (Map.Entry<String, String> outputEntry : signatureEntry.getValue().outputs().entrySet())
expressions.add(new Pair<>(signatureEntry.getKey() + "." + outputEntry.getKey(),
- signatureEntry.getValue().outputExpression(outputEntry.getKey())
- .withName(signatureEntry.getKey() + "." + outputEntry.getKey())));
+ expressions().get(outputEntry.getValue())));
if (signatureEntry.getValue().outputs().isEmpty()) // fallback: Signature without outputs
expressions.add(new Pair<>(signatureEntry.getKey(),
- new ExpressionFunction(signatureEntry.getKey(),
- new ArrayList<>(signatureEntry.getValue().inputs().keySet()),
- expressions().get(signatureEntry.getKey()),
- signatureEntry.getValue().inputMap(),
- Optional.empty())));
+ expressions().get(signatureEntry.getKey())));
}
if (signatures().isEmpty()) { // fallback for models without signatures
if (expressions().size() == 1) {
Map.Entry<String, RankingExpression> singleEntry = this.expressions.entrySet().iterator().next();
- expressions.add(new Pair<>(singleEntry.getKey(),
- new ExpressionFunction(singleEntry.getKey(),
- new ArrayList<>(inputs.keySet()),
- singleEntry.getValue(),
- inputs,
- Optional.empty())));
+ expressions.add(new Pair<>(singleEntry.getKey(), singleEntry.getValue()));
}
else {
for (Map.Entry<String, RankingExpression> expressionEntry : expressions().entrySet()) {
- expressions.add(new Pair<>(expressionEntry.getKey(),
- new ExpressionFunction(expressionEntry.getKey(),
- new ArrayList<>(inputs.keySet()),
- expressionEntry.getValue(),
- inputs,
- Optional.empty())));
+ expressions.add(new Pair<>(expressionEntry.getKey(), expressionEntry.getValue()));
}
}
}
@@ -151,7 +134,7 @@ public class ImportedModel {
}
/**
- * A signature is a set of named inputs and outputs, where the inputs maps to input
+ * A signature is a set of named inputs and outputs, where the inputs maps to argument
* ("placeholder") names+types, and outputs maps to expressions nodes.
* Note that TensorFlow supports multiple signatures in their format, but ONNX has no explicit
* concept of signatures. For now, we handle ONNX models as having a single signature.
@@ -159,8 +142,8 @@ public class ImportedModel {
public class Signature {
private final String name;
- private final Map<String, String> inputs = new LinkedHashMap<>();
- private final Map<String, String> outputs = new LinkedHashMap<>();
+ private final Map<String, String> inputs = new HashMap<>();
+ private final Map<String, String> outputs = new HashMap<>();
private final Map<String, String> skippedOutputs = new HashMap<>();
private final List<String> importWarnings = new ArrayList<>();
@@ -175,20 +158,12 @@ public class ImportedModel {
/**
* Returns an immutable map of the inputs (evaluation context) of this. This is a map from input name
- * in this signature to input name in the owning model
+ * to argument (Placeholder) name in the owner of this
*/
public Map<String, String> inputs() { return Collections.unmodifiableMap(inputs); }
- /** Returns the name and type of all inputs in this signature as an immutable map */
- public Map<String, TensorType> inputMap() {
- ImmutableMap.Builder<String, TensorType> inputs = new ImmutableMap.Builder<>();
- for (Map.Entry<String, String> inputEntry : inputs().entrySet())
- inputs.put(inputEntry.getKey(), owner().inputs().get(inputEntry.getValue()));
- return inputs.build();
- }
-
- /** Returns the type of the input this input references */
- public TensorType inputArgument(String inputName) { return owner().inputs().get(inputs.get(inputName)); }
+ /** Returns the type of the argument this input references */
+ public TensorType inputArgument(String inputName) { return owner().arguments().get(inputs.get(inputName)); }
/** Returns an immutable list of the expression names of this */
public Map<String, String> outputs() { return Collections.unmodifiableMap(outputs); }
@@ -205,13 +180,7 @@ public class ImportedModel {
public List<String> importWarnings() { return Collections.unmodifiableList(importWarnings); }
/** Returns the expression this output references */
- public ExpressionFunction outputExpression(String outputName) {
- return new ExpressionFunction(outputName,
- new ArrayList<>(inputs.keySet()),
- owner().expressions().get(outputs.get(outputName)),
- inputMap(),
- Optional.empty());
- }
+ public RankingExpression outputExpression(String outputName) { return owner().expressions().get(outputs.get(outputName)); }
@Override
public String toString() { return "signature '" + name + "'"; }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java
index b7138ad87e3..d25502fd149 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/ModelImporter.java
@@ -187,7 +187,8 @@ public abstract class ModelImporter {
if (operation.isInput()) {
// All inputs must have dimensions with standard naming convention: d0, d1, ...
OrderedTensorType standardNamingConvention = OrderedTensorType.standardType(operation.type().get());
- model.input(operation.vespaName(), standardNamingConvention.type());
+ model.argument(operation.vespaName(), standardNamingConvention.type());
+ model.requiredFunction(operation.vespaName(), standardNamingConvention.type());
}
}
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java
index e6bb5f40b3f..917b0d6a389 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxImporter.java
@@ -2,6 +2,7 @@
package com.yahoo.searchlib.rankingexpression.integration.ml;
+import com.yahoo.io.IOUtils;
import com.yahoo.searchlib.rankingexpression.integration.ml.importer.IntermediateGraph;
import com.yahoo.searchlib.rankingexpression.integration.ml.importer.onnx.GraphImporter;
import onnx.Onnx;
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java
index 94d663b4954..796c13a8669 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java
@@ -3,8 +3,6 @@ package com.yahoo.searchlib.rankingexpression.rule;
import com.google.common.collect.ImmutableMap;
import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
-import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.tensor.TensorType;
import java.util.Collection;
import java.util.Collections;
@@ -82,14 +80,9 @@ public class SerializationContext extends FunctionReferenceContext {
serializedFunctions.put(name, expressionString);
}
- /** Adds the serialization of the an argument type to a function */
- public void addArgumentTypeSerialization(String functionName, String argumentName, TensorType type) {
- serializedFunctions.put("rankingExpression(" + functionName + ")." + argumentName + ".type", type.toString());
- }
-
- /** Adds the serialization of the return type of a function */
- public void addFunctionTypeSerialization(String functionName, TensorType type) {
- serializedFunctions.put("rankingExpression(" + functionName + ").type", type.toString());
+ /** Returns the existing serialization of a function, or null if none */
+ public String getFunctionSerialization(String name) {
+ return serializedFunctions.get(name);
}
@Override
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java
index 969bc318391..118eba2cd96 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java
@@ -15,7 +15,6 @@ import org.junit.Test;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import static org.junit.Assert.fail;
@@ -41,8 +40,7 @@ public class GroupingSerializationTest {
t.assertMatch(new FloatResultNode(7.3));
t.assertMatch(new StringResultNode("7.3"));
t.assertMatch(new StringResultNode(
- new String(new byte[]{(byte)0xe5, (byte)0xa6, (byte)0x82, (byte)0xe6, (byte)0x9e, (byte)0x9c},
- StandardCharsets.UTF_8)));
+ new String(new byte[]{(byte)0xe5, (byte)0xa6, (byte)0x82, (byte)0xe6, (byte)0x9e, (byte)0x9c})));
t.assertMatch(new RawResultNode(new byte[]{'7', '.', '4'}));
t.assertMatch(new IntegerBucketResultNode());
t.assertMatch(new FloatBucketResultNode());
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java
index 593e7b54c10..bf9684082f4 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/BatchNormImportTestCase.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchlib.rankingexpression.integration.ml;
-import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import org.junit.Test;
@@ -21,11 +20,10 @@ public class BatchNormImportTestCase {
assertEquals("Has skipped outputs",
0, model.get().signature("serving_default").skippedOutputs().size());
- ExpressionFunction output = signature.outputExpression("y");
+ RankingExpression output = signature.outputExpression("y");
assertNotNull(output);
- assertEquals("dnn/batch_normalization_3/batchnorm/add_1", output.getBody().getName());
- model.assertEqualResult("X", output.getBody().getName());
- assertEquals("{x=tensor(d0[],d1[784])}", output.argumentTypes().toString());
+ assertEquals("dnn/batch_normalization_3/batchnorm/add_1", output.getName());
+ model.assertEqualResult("X", output.getName());
}
}
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java
index 59712c0152f..a8f7542f3a4 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/DropoutImportTestCase.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchlib.rankingexpression.integration.ml;
-import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.tensor.TensorType;
import org.junit.Test;
@@ -20,23 +19,22 @@ public class DropoutImportTestCase {
TestableTensorFlowModel model = new TestableTensorFlowModel("test", "src/test/files/integration/tensorflow/dropout/saved");
// Check required functions
- assertEquals(1, model.get().inputs().size());
- assertTrue(model.get().inputs().containsKey("X"));
+ assertEquals(1, model.get().requiredFunctions().size());
+ assertTrue(model.get().requiredFunctions().containsKey("X"));
assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(),
- model.get().inputs().get("X"));
+ model.get().requiredFunctions().get("X"));
ImportedModel.Signature signature = model.get().signature("serving_default");
assertEquals("Has skipped outputs",
0, model.get().signature("serving_default").skippedOutputs().size());
- ExpressionFunction output = signature.outputExpression("y");
+ RankingExpression output = signature.outputExpression("y");
assertNotNull(output);
- assertEquals("outputs/Maximum", output.getBody().getName());
+ assertEquals("outputs/Maximum", output.getName());
assertEquals("join(join(imported_ml_function_test_outputs_BiasAdd, reduce(constant(test_outputs_Const), sum, d1), f(a,b)(a * b)), imported_ml_function_test_outputs_BiasAdd, f(a,b)(max(a,b)))",
- output.getBody().getRoot().toString());
- model.assertEqualResult("X", output.getBody().getName());
- assertEquals("{x=tensor(d0[],d1[784])}", output.argumentTypes().toString());
+ output.getRoot().toString());
+ model.assertEqualResult("X", output.getName());
}
}
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java
index 3d8d5d5a570..add66eece1a 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/MnistImportTestCase.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchlib.rankingexpression.integration.ml;
-import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import org.junit.Test;
@@ -21,10 +20,11 @@ public class MnistImportTestCase {
assertEquals("Has skipped outputs",
0, model.get().signature("serving_default").skippedOutputs().size());
- ExpressionFunction output = signature.outputExpression("y");
+ RankingExpression output = signature.outputExpression("y");
assertNotNull(output);
- assertEquals("dnn/outputs/add", output.getBody().getName());
- model.assertEqualResultSum("input", output.getBody().getName(), 0.00001);
+ assertEquals("dnn/outputs/add", output.getName());
+ model.assertEqualResultSum("input", output.getName(), 0.00001);
}
+
}
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java
index b6e83404ab1..e20ac16a691 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/OnnxMnistSoftmaxImportTestCase.java
@@ -1,6 +1,5 @@
package com.yahoo.searchlib.rankingexpression.integration.ml;
-import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.evaluation.Context;
import com.yahoo.searchlib.rankingexpression.evaluation.MapContext;
@@ -28,28 +27,27 @@ public class OnnxMnistSoftmaxImportTestCase {
Tensor constant0 = model.largeConstants().get("test_Variable");
assertNotNull(constant0);
assertEquals(new TensorType.Builder().indexed("d2", 784).indexed("d1", 10).build(),
- constant0.type());
+ constant0.type());
assertEquals(7840, constant0.size());
Tensor constant1 = model.largeConstants().get("test_Variable_1");
assertNotNull(constant1);
- assertEquals(new TensorType.Builder().indexed("d1", 10).build(), constant1.type());
+ assertEquals(new TensorType.Builder().indexed("d1", 10).build(),
+ constant1.type());
assertEquals(10, constant1.size());
- // Check inputs
- assertEquals(1, model.inputs().size());
- assertTrue(model.inputs().containsKey("Placeholder"));
- assertEquals(TensorType.fromSpec("tensor(d0[],d1[784])"), model.inputs().get("Placeholder"));
+ // Check required functions (inputs)
+ assertEquals(1, model.requiredFunctions().size());
+ assertTrue(model.requiredFunctions().containsKey("Placeholder"));
+ assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(),
+ model.requiredFunctions().get("Placeholder"));
- // Check signature
- ExpressionFunction output = model.defaultSignature().outputExpression("add");
+ // Check outputs
+ RankingExpression output = model.defaultSignature().outputExpression("add");
assertNotNull(output);
- assertEquals("add", output.getBody().getName());
+ assertEquals("add", output.getName());
assertEquals("join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(test_Variable), f(a,b)(a * b)), sum, d2), constant(test_Variable_1), f(a,b)(a + b))",
- output.getBody().getRoot().toString());
- assertEquals(TensorType.fromSpec("tensor(d0[],d1[784])"),
- model.inputs().get(model.defaultSignature().inputs().get("Placeholder")));
- assertEquals("{Placeholder=tensor(d0[],d1[784])}", output.argumentTypes().toString());
+ output.getRoot().toString());
}
@Test
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java
index 0a48ecfce21..ef28eb4678f 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/ml/TensorFlowMnistSoftmaxImportTestCase.java
@@ -1,7 +1,6 @@
// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchlib.rankingexpression.integration.ml;
-import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.tensor.Tensor;
import com.yahoo.tensor.TensorType;
@@ -39,10 +38,10 @@ public class TensorFlowMnistSoftmaxImportTestCase {
assertEquals(0, model.get().functions().size());
// Check required functions
- assertEquals(1, model.get().inputs().size());
- assertTrue(model.get().inputs().containsKey("Placeholder"));
+ assertEquals(1, model.get().requiredFunctions().size());
+ assertTrue(model.get().requiredFunctions().containsKey("Placeholder"));
assertEquals(new TensorType.Builder().indexed("d0").indexed("d1", 784).build(),
- model.get().inputs().get("Placeholder"));
+ model.get().requiredFunctions().get("Placeholder"));
// Check signatures
assertEquals(1, model.get().signatures().size());
@@ -57,12 +56,11 @@ public class TensorFlowMnistSoftmaxImportTestCase {
// ... signature outputs
assertEquals(1, signature.outputs().size());
- ExpressionFunction output = signature.outputExpression("y");
+ RankingExpression output = signature.outputExpression("y");
assertNotNull(output);
- assertEquals("add", output.getBody().getName());
+ assertEquals("add", output.getName());
assertEquals("join(reduce(join(rename(Placeholder, (d0, d1), (d0, d2)), constant(test_Variable_read), f(a,b)(a * b)), sum, d2), constant(test_Variable_1_read), f(a,b)(a + b))",
- output.getBody().getRoot().toString());
- assertEquals("{x=tensor(d0[],d1[784])}", output.argumentTypes().toString());
+ output.getRoot().toString());
// Test execution
model.assertEqualResult("Placeholder", "MatMul");