aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2018-02-06 22:20:01 +0100
committerGitHub <noreply@github.com>2018-02-06 22:20:01 +0100
commitfb124132798c81a7ccc1aa432a5effae3c5869b3 (patch)
tree368cada4a98a52f78abf2ff1f23b48f7f85b93a2
parentc2797cb1f2745a1fee89610e6eb7a4c1d3215c18 (diff)
parent4062af167c5f1415d7f3f2eb0792447dafbf5096 (diff)
Merge pull request #4937 from vespa-engine/revert-4935-bratseth/typecheck-all
Revert "Bratseth/typecheck all"
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java12
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java32
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java23
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java (renamed from config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java)13
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java2
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java3
-rw-r--r--config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java72
-rw-r--r--config-model/src/test/derived/tensor/rank-profiles.cfg2
-rw-r--r--config-model/src/test/derived/tensor/tensor.sd2
-rw-r--r--config-model/src/test/examples/rankpropvars.sd8
-rw-r--r--config-model/src/test/examples/simple.sd2
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java8
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java3
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java40
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java4
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java104
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java18
-rw-r--r--config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java133
-rwxr-xr-xsearchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java17
-rw-r--r--vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java1
21 files changed, 119 insertions, 382 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
index 67724058e64..eb0c6067fca 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
@@ -333,7 +333,7 @@ public class DeployState implements ConfigDefinitionStore {
closeIgnoreException(reader.getReader());
}
}
- builder.build(logger);
+ builder.build(logger, queryProfiles);
return SearchDocumentModel.fromBuilderAndNames(builder, names);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
index c01b009e93b..dd03cb8b2a7 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
@@ -101,18 +101,6 @@ public class FeatureNames {
return canonicalize("query(\"" + propertyName + "\")");
}
- public static boolean isConstantFeature(String feature) {
- return feature.startsWith("constant(");
- }
-
- public static boolean isAttributeFeature(String feature) {
- return feature.startsWith("attribute(");
- }
-
- public static boolean isQueryFeature(String feature) {
- return feature.startsWith("query(");
- }
-
/**
* Returns the single argument of the given feature name, without any quotes,
* or empty if it is not a valid query, attribute or constant feature name
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index d1a29271014..bcbc7cc99e2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -2,9 +2,15 @@
package com.yahoo.searchdefinition;
import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.QueryProfile;
import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.config.QueryProfileXMLReader;
import com.yahoo.search.query.profile.types.FieldDescription;
import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.TensorFieldType;
import com.yahoo.search.query.ranking.Diversity;
import com.yahoo.searchdefinition.document.SDField;
import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext;
@@ -13,6 +19,7 @@ import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
import com.yahoo.searchlib.rankingexpression.FeatureList;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.TypeMapContext;
import com.yahoo.searchlib.rankingexpression.evaluation.Value;
import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
import com.yahoo.tensor.TensorType;
@@ -578,11 +585,8 @@ public class RankProfile implements Serializable, Cloneable {
}
/**
- * Will take the parser-set textual ranking expressions and turn into ranking expression objects,
- * if not already done
+ * Will take the parser-set textual ranking expressions and turn into objects
*/
- // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions
- // until this is called. Simplify by parsing them right away.
public void parseExpressions() {
try {
parseRankingExpressions();
@@ -600,23 +604,20 @@ public class RankProfile implements Serializable, Cloneable {
for (Map.Entry<String, Macro> e : getMacros().entrySet()) {
String macroName = e.getKey();
Macro macro = e.getValue();
- if (macro.getRankingExpression() == null) {
- RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression());
- macro.setRankingExpression(expr);
- macro.setTextualExpression(expr.getRoot().toString());
- }
+ RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression());
+ macro.setRankingExpression(expr);
+ macro.setTextualExpression(expr.getRoot().toString());
}
}
/**
* Passes ranking expressions on to parser
- *
* @throws ParseException if either of the ranking expressions could not be parsed
*/
private void parseRankingExpressions() throws ParseException {
- if (getFirstPhaseRankingString() != null && firstPhaseRanking == null)
+ if (getFirstPhaseRankingString() != null)
setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString()));
- if (getSecondPhaseRankingString() != null && secondPhaseRanking == null)
+ if (getSecondPhaseRankingString() != null)
setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString()));
}
@@ -747,7 +748,7 @@ public class RankProfile implements Serializable, Cloneable {
* referable from this rank profile.
*/
public TypeContext typeContext(QueryProfileRegistry queryProfiles) {
- MapTypeContext context = new MapTypeContext();
+ TypeMapContext context = new TypeMapContext();
// Add small constants
getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type()));
@@ -763,8 +764,7 @@ public class RankProfile implements Serializable, Cloneable {
for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) {
for (FieldDescription field : queryProfileType.declaredFields().values()) {
TensorType type = field.getType().asTensorType();
- if ( ! FeatureNames.isQueryFeature(field.getName())) continue;
- String feature = FeatureNames.canonicalize(field.getName());
+ String feature = FeatureNames.asQueryFeature(field.getName());
TensorType existingType = context.getType(feature);
if (existingType != null)
type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () ->
@@ -910,7 +910,7 @@ public class RankProfile implements Serializable, Cloneable {
*/
public static class Macro implements Serializable, Cloneable {
- private final String name;
+ private String name=null;
private String textualExpression=null;
private RankingExpression expression=null;
private List<String> formalParams = new ArrayList<>();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 469f29098ad..762c0fec838 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -34,6 +34,7 @@ import java.util.List;
* expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built
* search objects using the {@link #getSearch(String)} method.
*/
+// TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage
public class SearchBuilder {
private final DocumentTypeManager docTypeMgr = new DocumentTypeManager();
@@ -195,7 +196,11 @@ public class SearchBuilder {
* @throws IllegalStateException Thrown if this method has already been called.
*/
public void build() {
- build(new BaseDeployLogger());
+ build(new BaseDeployLogger(), new QueryProfiles());
+ }
+
+ public void build(DeployLogger logger) {
+ build(logger, new QueryProfiles());
}
/**
@@ -204,10 +209,12 @@ public class SearchBuilder {
*
* @throws IllegalStateException Thrown if this method has already been called.
* @param deployLogger The logger to use during build
+ * @param queryProfiles The query profiles contained in the application this search is part of.
*/
- public void build(DeployLogger deployLogger) {
- if (isBuilt) throw new IllegalStateException("Model already built");
-
+ public void build(DeployLogger deployLogger, QueryProfiles queryProfiles) {
+ if (isBuilt) {
+ throw new IllegalStateException("Searches already built.");
+ }
List<Search> built = new ArrayList<>();
List<SDDocumentType> sdocs = new ArrayList<>();
sdocs.add(SDDocumentType.VESPA_DOCUMENT);
@@ -233,7 +240,7 @@ public class SearchBuilder {
for (Search search : new SearchOrderer().order(searchList)) {
new FieldOperationApplierForSearch().process(search);
// These two needed for a couple of old unit tests, ideally these are just read from app
- process(search, deployLogger, new QueryProfiles(queryProfileRegistry));
+ process(search, deployLogger, queryProfiles);
built.add(search);
}
builder.addToModel(searchList);
@@ -247,6 +254,8 @@ public class SearchBuilder {
/**
* Processes and returns the given {@link Search} object. This method has been factored out of the {@link
* #build()} method so that subclasses can choose not to build anything.
+ *
+ * @param search The object to build.
*/
protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) {
Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles);
@@ -343,7 +352,7 @@ public class SearchBuilder {
rankProfileRegistry,
queryprofileRegistry);
builder.importFile(fileName);
- builder.build(deployLogger);
+ builder.build(deployLogger, new QueryProfiles());
return builder;
}
@@ -359,7 +368,7 @@ public class SearchBuilder {
for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) {
builder.importFile(i.next());
}
- builder.build(new BaseDeployLogger());
+ builder.build(new BaseDeployLogger(), new QueryProfiles());
return builder;
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java
index e0dc7a2f33c..40e9db1413f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java
@@ -10,13 +10,10 @@ import java.util.Map;
/**
* A context which only contains type information.
- * This returns empty tensor types (double) for unknown features which are not
- * query, attribute or constant features, as we do not have information about which such
- * features exist (but we know those that exist are doubles).
*
* @author bratseth
*/
-public class MapTypeContext implements TypeContext {
+public class TypeMapContext implements TypeContext {
private final Map<String, TensorType> featureTypes = new HashMap<>();
@@ -26,14 +23,10 @@ public class MapTypeContext implements TypeContext {
@Override
public TensorType getType(String name) {
- if (FeatureNames.isConstantFeature(name) ||
- FeatureNames.isAttributeFeature(name) ||
- FeatureNames.isQueryFeature(name))
- return featureTypes.get(FeatureNames.canonicalize(name));
- else
- return TensorType.empty; // we do not have type information for these. Correct would be either empty or null
+ return featureTypes.get(FeatureNames.canonicalize(name));
}
+ /** Returns an unmodifiable map of the bindings in this */
public Map<String, TensorType> bindings() { return Collections.unmodifiableMap(featureTypes); }
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
index cc634abef01..ee65c9bec02 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java
@@ -13,7 +13,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression;
import com.yahoo.vespa.model.container.search.QueryProfiles;
/**
- * @author Simon Thoresen Hult
+ * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a>
*/
public class IndexingValues extends Processor {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
index 061a803cb48..90183848094 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java
@@ -76,9 +76,8 @@ public class Processing {
ImportedFieldsInSummayValidator::new,
FastAccessValidator::new,
ReservedMacroNames::new,
- RankingExpressionTypeValidator::new,
- // These should be last.
+ // These two should be last.
IndexingValidation::new,
IndexingValues::new);
}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java
deleted file mode 100644
index a7a5ad58430..00000000000
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchdefinition.processing;
-
-import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.search.query.profile.QueryProfileRegistry;
-import com.yahoo.searchdefinition.MapTypeContext;
-import com.yahoo.searchdefinition.RankProfile;
-import com.yahoo.searchdefinition.RankProfileRegistry;
-import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.tensor.TensorType;
-import com.yahoo.tensor.evaluation.TypeContext;
-import com.yahoo.vespa.model.container.search.QueryProfiles;
-
-public class RankingExpressionTypeValidator extends Processor {
-
- private final QueryProfileRegistry queryProfiles;
-
- public RankingExpressionTypeValidator(Search search,
- DeployLogger deployLogger,
- RankProfileRegistry rankProfileRegistry,
- QueryProfiles queryProfiles) {
- super(search, deployLogger, rankProfileRegistry, queryProfiles);
- this.queryProfiles = queryProfiles.getRegistry();
- }
-
- @Override
- public void process() {
- for (RankProfile profile : rankProfileRegistry.allRankProfiles()) {
- try {
- validate(profile);
- }
- catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("In " + search + ", " + profile, e);
- }
- }
- }
-
- /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */
- private void validate(RankProfile profile) {
- profile.parseExpressions();
- TypeContext context = profile.typeContext(queryProfiles);
- for (RankProfile.Macro macro : profile.getMacros().values())
- ensureValid(macro.getRankingExpression(), "macro '" + macro.getName() + "'", context);
- ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context);
- ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context);
- }
-
- private TensorType ensureValid(RankingExpression expression, String expressionDescription, TypeContext context) {
- if (expression == null) return null;
-
- TensorType type;
- try {
- type = expression.type(context);
- }
- catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("The " + expressionDescription + " is invalid", e);
- }
- if (type == null) // Not expected to happen
- throw new IllegalStateException("Could not determine the type produced by " + expressionDescription);
- return type;
- }
-
- private void ensureValidDouble(RankingExpression expression, String expressionDescription, TypeContext context) {
- if (expression == null) return;
- TensorType type = ensureValid(expression, expressionDescription, context);
- if ( ! type.equals(TensorType.empty))
- throw new IllegalArgumentException("The " + expressionDescription + " must produce a double " +
- "(a tensor with no dimensions), but produces " + type);
- }
-
-}
diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg
index b6ad5372c05..2b231e0cda2 100644
--- a/config-model/src/test/derived/tensor/rank-profiles.cfg
+++ b/config-model/src/test/derived/tensor/rank-profiles.cfg
@@ -35,7 +35,7 @@ rankprofile[3].name "profile2"
rankprofile[3].fef.property[0].name "vespa.rank.firstphase"
rankprofile[3].fef.property[0].value "rankingExpression(firstphase)"
rankprofile[3].fef.property[1].name "rankingExpression(firstphase).rankingScript"
-rankprofile[3].fef.property[1].value "reduce(reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)"
+rankprofile[3].fef.property[1].value "reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x)"
rankprofile[3].fef.property[2].name "vespa.type.attribute.f2"
rankprofile[3].fef.property[2].value "tensor(x[2],y[])"
rankprofile[3].fef.property[3].name "vespa.type.attribute.f3"
diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd
index 3d64f6b807e..a6a9a98db3a 100644
--- a/config-model/src/test/derived/tensor/tensor.sd
+++ b/config-model/src/test/derived/tensor/tensor.sd
@@ -28,7 +28,7 @@ search tensor {
rank-profile profile2 {
first-phase {
- expression: sum(matmul(attribute(f4), diag(x[2],y[2],z[3]), x))
+ expression: matmul(attribute(f4), diag(x[2],y[2],z[3]), x)
}
}
diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd
index 28959edbc09..40f9e73f35a 100644
--- a/config-model/src/test/examples/rankpropvars.sd
+++ b/config-model/src/test/examples/rankpropvars.sd
@@ -18,8 +18,8 @@ first-phase {
second-phase {
expression {
if (attribute(artist) == query(testvar1),
- 0.0 * fieldMatch(title) + 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist),
- 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title))
+ 0.0 * fieldMatch(title) + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist),
+ 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title))
}
}
@@ -42,8 +42,8 @@ first-phase {
second-phase {
expression {
if (attribute(artist) == query(testvar1),
- 0.0 * fieldMatch(title) + 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist),
- 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title))
+ 0.0 * fieldMatch(title) + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist),
+ 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title))
}
}
}
diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd
index 96b0fa98098..4fda7f5039e 100644
--- a/config-model/src/test/examples/simple.sd
+++ b/config-model/src/test/examples/simple.sd
@@ -116,7 +116,7 @@ search simple {
first-phase {
keep-rank-count:200
rank-score-drop-limit: -13.0
- expression: attribute(popularity)
+ expression: attribute(year)
}
second-phase {
rerank-count: 99
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
index 11093d9f008..442c8bd41bd 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
@@ -135,13 +135,13 @@ public class RankProfileTestCase extends SearchDefinitionTestCase {
@Test
public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException {
RankProfileRegistry registry = new RankProfileRegistry();
- SearchBuilder builder = new SearchBuilder(registry, setupQueryProfileTypes());
+ SearchBuilder builder = new SearchBuilder(registry);
builder.importString("search test {\n" +
" document test { } \n" +
" rank-profile p1 {}\n" +
" rank-profile p2 {}\n" +
"}");
- builder.build(new BaseDeployLogger());
+ builder.build(new BaseDeployLogger(), setupQueryProfileTypes());
Search search = builder.getSearch();
assertEquals(4, registry.allRankProfiles().size());
@@ -151,7 +151,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase {
assertQueryFeatureTypeSettings(registry.getRankProfile(search, "p2"), search);
}
- private static QueryProfileRegistry setupQueryProfileTypes() {
+ private static QueryProfiles setupQueryProfileTypes() {
QueryProfileRegistry registry = new QueryProfileRegistry();
QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry();
QueryProfileType type = new QueryProfileType(new ComponentId("testtype"));
@@ -164,7 +164,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase {
type.addField(new FieldDescription("ranking.features.query(numeric)",
FieldType.fromString("integer", typeRegistry)), typeRegistry);
typeRegistry.register(type);
- return registry;
+ return new QueryProfiles(registry);
}
private static void assertQueryFeatureTypeSettings(RankProfile profile, Search search) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
index 82b9f5ac043..e94880e61c7 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
@@ -207,9 +207,6 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase
builder.importString(
"search test {\n" +
" document test { \n" +
- " field rating_yelp type int {" +
- " indexing: attribute" +
- " }" +
" }\n" +
" \n" +
" rank-profile test {\n" +
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
index ed1b00e2875..5100ac15c40 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
@@ -2,10 +2,7 @@
package com.yahoo.searchdefinition;
import com.yahoo.collections.Pair;
-import com.yahoo.search.query.profile.QueryProfile;
import com.yahoo.search.query.profile.QueryProfileRegistry;
-import com.yahoo.search.query.profile.types.FieldDescription;
-import com.yahoo.search.query.profile.types.QueryProfileType;
import com.yahoo.searchdefinition.derived.AttributeFields;
import com.yahoo.searchdefinition.derived.RawRankProfile;
import com.yahoo.searchdefinition.parser.ParseException;
@@ -152,12 +149,11 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
censorBindingHash(testRankProperties.get(4).toString()));
}
+
@Test
public void testNeuralNetworkSetup() throws ParseException {
- // Note: the type assigned to query profile and constant tensors here is not the correct type
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
- QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(x[])");
- SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles);
+ SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(
"search test {\n" +
" document test { \n" +
@@ -180,28 +176,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
" expression: sum(final_layer)\n" +
" }\n" +
" }\n" +
- " constant W_hidden {\n" +
- " type: tensor(x[])\n" +
- " file: ignored.json\n" +
- " }\n" +
- " constant b_input {\n" +
- " type: tensor(x[])\n" +
- " file: ignored.json\n" +
- " }\n" +
- " constant W_final {\n" +
- " type: tensor(x[])\n" +
- " file: ignored.json\n" +
- " }\n" +
- " constant b_final {\n" +
- " type: tensor(x[])\n" +
- " file: ignored.json\n" +
- " }\n" +
+ "\n" +
"}\n");
builder.build();
Search s = builder.getSearch();
- RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(queryProfiles);
+ RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry());
List<Pair<String, String>> testRankProperties = new RawRankProfile(test,
- queryProfiles,
+ new QueryProfileRegistry(),
new AttributeFields(s)).configProperties();
assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))",
testRankProperties.get(0).toString());
@@ -217,17 +198,6 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase
testRankProperties.get(5).toString());
}
- private QueryProfileRegistry queryProfileWith(String field, String type) {
- QueryProfileType queryProfileType = new QueryProfileType("root");
- queryProfileType.addField(new FieldDescription(field, type));
- QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry();
- queryProfileRegistry.getTypeRegistry().register(queryProfileType);
- QueryProfile profile = new QueryProfile("default");
- profile.setType(queryProfileType);
- queryProfileRegistry.register(profile);
- return queryProfileRegistry;
- }
-
private String censorBindingHash(String s) {
StringBuilder b = new StringBuilder();
boolean areInHash = false;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
index 0ce6129ef7f..800697b3430 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
@@ -38,8 +38,7 @@ class RankProfileSearchFixture {
RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry,
String rankProfiles, String constant, String field)
throws ParseException {
- this.queryProfileRegistry = queryProfileRegistry;
- SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, queryProfileRegistry);
+ SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, new QueryProfileRegistry());
String sdContent = "search test {\n" +
" " + (constant != null ? constant : "") + "\n" +
" document test {\n" +
@@ -51,6 +50,7 @@ class RankProfileSearchFixture {
builder.importString(sdContent);
builder.build();
search = builder.getSearch();
+ this.queryProfileRegistry = queryProfileRegistry;
}
public void assertFirstPhaseExpression(String expExpression, String rankProfile) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java
deleted file mode 100644
index db3b12db1bf..00000000000
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java
+++ /dev/null
@@ -1,104 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchdefinition.processing;
-
-import com.yahoo.searchdefinition.RankProfileRegistry;
-import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.yolean.Exceptions;
-import org.junit.Test;
-import static com.yahoo.config.model.test.TestUtil.joinLines;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class RankingExpressionTypeValidatorTestCase {
-
- @Test
- public void tensorFirstPhaseMustProduceDouble() throws Exception {
- try {
- RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
- SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
- searchBuilder.importString(joinLines(
- "search test {",
- " document test { ",
- " field a type tensor(x[],y[]) {",
- " indexing: attribute",
- " }",
- " }",
- " rank-profile my_rank_profile {",
- " first-phase {",
- " expression: attribute(a)",
- " }",
- " }",
- "}"
- ));
- searchBuilder.build();
- fail("Expected exception");
- }
- catch (IllegalArgumentException expected) {
- assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])",
- Exceptions.toMessageString(expected));
- }
- }
-
- @Test
- public void tensorSecondPhaseMustProduceDouble() throws Exception {
- try {
- RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
- SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
- searchBuilder.importString(joinLines(
- "search test {",
- " document test { ",
- " field a type tensor(x[],y[]) {",
- " indexing: attribute",
- " }",
- " }",
- " rank-profile my_rank_profile {",
- " first-phase {",
- " expression: sum(attribute(a))",
- " }",
- " second-phase {",
- " expression: attribute(a)",
- " }",
- " }",
- "}"
- ));
- searchBuilder.build();
- fail("Expected exception");
- }
- catch (IllegalArgumentException expected) {
- assertEquals("In search definition 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])",
- Exceptions.toMessageString(expected));
- }
- }
-
- @Test
- public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception {
- try {
- RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
- SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry);
- searchBuilder.importString(joinLines(
- "search test {",
- " document test { ",
- " field a type tensor(x[],y[]) {",
- " indexing: attribute",
- " }",
- " field b type tensor(z[10]) {",
- " indexing: attribute",
- " }",
- " }",
- " rank-profile my_rank_profile {",
- " first-phase {",
- " expression: sum(if(1>0, attribute(a), attribute(b)))",
- " }",
- " }",
- "}"
- ));
- searchBuilder.build();
- fail("Expected exception");
- }
- catch (IllegalArgumentException expected) {
- assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[],y[]) while the 'false' type is tensor(z[10])",
- Exceptions.toMessageString(expected));
- }
- }
-
-}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
index 58af8daf1b5..7246b22b0f8 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java
@@ -51,7 +51,7 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testTensorFlowReference() {
+ public void testTensorFlowReference() throws ParseException {
RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)",
"tensorflow('mnist_softmax/saved')");
search.assertFirstPhaseExpression(vespaExpression, "my_profile");
@@ -60,7 +60,7 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testTensorFlowReferenceWithConstantFeature() {
+ public void testTensorFlowReferenceWithConstantFeature() throws ParseException {
RankProfileSearchFixture search = fixtureWith("constant(mytensor)",
"tensorflow('mnist_softmax/saved')",
"constant mytensor { file: ignored\ntype: tensor(d0[7],d1[784]) }",
@@ -71,10 +71,10 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testTensorFlowReferenceWithQueryFeature() {
+ public void testTensorFlowReferenceWithQueryFeature() throws ParseException {
String queryProfile = "<query-profile id='default' type='root'/>";
String queryProfileType = "<query-profile-type id='root'>" +
- " <field name='query(mytensor)' type='tensor(d0[3],d1[784])'/>" +
+ " <field name='mytensor' type='tensor(d0[3],d1[784])'/>" +
"</query-profile-type>";
StoringApplicationPackage application = new StoringApplicationPackage(applicationDir,
queryProfile,
@@ -90,7 +90,7 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testTensorFlowReferenceWithDocumentFeature() {
+ public void testTensorFlowReferenceWithDocumentFeature() throws ParseException {
StoringApplicationPackage application = new StoringApplicationPackage(applicationDir);
RankProfileSearchFixture search = fixtureWith("attribute(mytensor)",
"tensorflow('mnist_softmax/saved')",
@@ -103,10 +103,10 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testTensorFlowReferenceWithFeatureCombination() {
+ public void testTensorFlowReferenceWithFeatureCombination() throws ParseException {
String queryProfile = "<query-profile id='default' type='root'/>";
String queryProfileType = "<query-profile-type id='root'>" +
- " <field name='query(mytensor)' type='tensor(d0[3],d1[784],d2[10])'/>" +
+ " <field name='mytensor' type='tensor(d0[3],d1[784],d2[10])'/>" +
"</query-profile-type>";
StoringApplicationPackage application = new StoringApplicationPackage(applicationDir,
queryProfile,
@@ -122,7 +122,7 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testNestedTensorFlowReference() {
+ public void testNestedTensorFlowReference() throws ParseException {
RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)",
"5 + sum(tensorflow('mnist_softmax/saved'))");
search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile");
@@ -131,7 +131,7 @@ public class RankingExpressionWithTensorFlowTestCase {
}
@Test
- public void testTensorFlowReferenceSpecifyingSignature() {
+ public void testTensorFlowReferenceSpecifyingSignature() throws ParseException {
RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)",
"tensorflow('mnist_softmax/saved', 'serving_default')");
search.assertFirstPhaseExpression(vespaExpression, "my_profile");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index d2211b86c9e..c18cfcfe1aa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -17,129 +17,98 @@ import com.yahoo.searchdefinition.SearchDefinitionTestCase;
import com.yahoo.searchdefinition.derived.AttributeFields;
import com.yahoo.searchdefinition.derived.RawRankProfile;
import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
import org.junit.Test;
import java.util.List;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
public class TensorTransformTestCase extends SearchDefinitionTestCase {
@Test
public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException {
- assertTransformedExpression("max(1.0,2.0)",
- "max(1.0,2.0)");
- assertTransformedExpression("min(attribute(double_field),x)",
- "min(attribute(double_field),x)");
- assertTransformedExpression("max(attribute(double_field),attribute(double_array_field))",
- "max(attribute(double_field),attribute(double_array_field))");
- assertTransformedExpression("min(attribute(tensor_field_1),attribute(double_field))",
- "min(attribute(tensor_field_1),attribute(double_field))");
- assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)",
- "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)");
- assertTransformedExpression("min(constant(test_constant_tensor),1.0)",
- "min(test_constant_tensor,1.0)");
- assertTransformedExpression("max(constant(base_constant_tensor),1.0)",
- "max(base_constant_tensor,1.0)");
- assertTransformedExpression("min(constant(file_constant_tensor),1.0)",
- "min(constant(file_constant_tensor),1.0)");
- assertTransformedExpression("max(query(q),1.0)",
- "max(query(q),1.0)");
- assertTransformedExpression("max(query(n),1.0)",
- "max(query(n),1.0)");
+ assertContainsExpression("max(1.0,2.0)", "max(1.0,2.0)");
+ assertContainsExpression("min(attribute(double_field),x)", "min(attribute(double_field),x)");
+ assertContainsExpression("max(attribute(double_field),attribute(double_array_field))", "max(attribute(double_field),attribute(double_array_field))");
+ assertContainsExpression("min(attribute(tensor_field_1),attribute(double_field))", "min(attribute(tensor_field_1),attribute(double_field))");
+ assertContainsExpression("max(attribute(tensor_field_1),attribute(tensor_field_2))", "max(attribute(tensor_field_1),attribute(tensor_field_2))");
+ assertContainsExpression("min(test_constant_tensor,1.0)", "min(constant(test_constant_tensor),1.0)");
+ assertContainsExpression("max(base_constant_tensor,1.0)", "max(constant(base_constant_tensor),1.0)");
+ assertContainsExpression("min(constant(file_constant_tensor),1.0)", "min(constant(file_constant_tensor),1.0)");
+ assertContainsExpression("max(query(q),1.0)", "max(query(q),1.0)");
+ assertContainsExpression("max(query(n),1.0)", "max(query(n),1.0)");
}
@Test
public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException {
- assertTransformedExpression("reduce(attribute(tensor_field_1),max,x)",
- "max(attribute(tensor_field_1),x)");
- assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)",
- "1 + max(attribute(tensor_field_1),x)");
- assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)",
- "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)");
- assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)",
- "max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)");
- assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)",
- "max(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),x)");
- assertTransformedExpression("max(reduce(attribute(tensor_field_1),max,x),x)",
- "max(max(attribute(tensor_field_1),x),x)"); // will result in deploy error.
- assertTransformedExpression("reduce(reduce(attribute(tensor_field_2),max,x),max,y)",
- "max(max(attribute(tensor_field_2),x),y)");
+ assertContainsExpression("max(attribute(tensor_field_1),x)", "reduce(attribute(tensor_field_1),max,x)");
+ assertContainsExpression("1 + max(attribute(tensor_field_1),x)", "1+reduce(attribute(tensor_field_1),max,x)");
+ assertContainsExpression("if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)", "if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)");
+ assertContainsExpression("max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)");
+ assertContainsExpression("max(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),max,x)");
+ assertContainsExpression("max(max(attribute(tensor_field_1),x),x)", "max(reduce(attribute(tensor_field_1),max,x),x)"); // will result in deploy error.
+ assertContainsExpression("max(max(attribute(tensor_field_2),x),y)", "reduce(reduce(attribute(tensor_field_2),max,x),max,y)");
}
@Test
public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException {
- assertTransformedExpression("reduce(constant(test_constant_tensor),max,x)",
- "max(test_constant_tensor,x)");
- assertTransformedExpression("reduce(constant(base_constant_tensor),max,x)",
- "max(base_constant_tensor,x)");
- assertTransformedExpression("reduce(constant(file_constant_tensor),min,x)",
- "min(constant(file_constant_tensor),x)");
+ assertContainsExpression("max(test_constant_tensor,x)", "reduce(constant(test_constant_tensor),max,x)");
+ assertContainsExpression("max(base_constant_tensor,x)", "reduce(constant(base_constant_tensor),max,x)");
+ assertContainsExpression("min(constant(file_constant_tensor),x)", "reduce(constant(file_constant_tensor),min,x)");
}
@Test
public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException {
- assertTransformedExpression("reduce(attribute(double_field)+attribute(tensor_field_1),min,x)",
- "min(attribute(double_field) + attribute(tensor_field_1),x)");
- assertTransformedExpression("reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)",
- "min(attribute(tensor_field_1) * attribute(tensor_field_2),x)");
- assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"
- , "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)");
- assertTransformedExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)",
- "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...)
- assertTransformedExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)",
- "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)");
+ assertContainsExpression("min(attribute(double_field) + attribute(tensor_field_1),x)", "reduce(attribute(double_field)+attribute(tensor_field_1),min,x)");
+ assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)");
+ assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)");
+ assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...)
+ assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)");
}
@Test
public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException {
- assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)",
- "max(tensorFromLabels(attribute(double_array_field)),double_array_field)");
- assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field),x),max,x)",
- "max(tensorFromLabels(attribute(double_array_field),x),x)");
- assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)",
- "max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)");
- assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)",
- "max(tensorFromWeightedSet(attribute(weightedset_field),x),x)");
+ assertContainsExpression("max(tensorFromLabels(attribute(double_array_field)),double_array_field)", "reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)");
+ assertContainsExpression("max(tensorFromLabels(attribute(double_array_field),x),x)", "reduce(tensorFromLabels(attribute(double_array_field),x),max,x)");
+ assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)", "reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)");
+ assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field),x),x)", "reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)");
}
@Test
public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException {
- assertTransformedExpression("reduce(query(q),max,x)", "max(query(q),x)");
- assertTransformedExpression("max(query(n),x)", "max(query(n),x)");
+ assertContainsExpression("max(query(q),x)", "reduce(query(q),max,x)");
+ assertContainsExpression("max(query(n),x)", "max(query(n),x)");
}
@Test
public void requireThatMaxAndMinWithTensoresReturnedFromMacrosAreReplaced() throws ParseException {
- assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)",
- "max(returns_tensor,x)");
- assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)",
- "max(wraps_returns_tensor,x)");
- assertTransformedExpression("reduce(rankingExpression(tensor_inheriting),max,x)",
- "max(tensor_inheriting,x)");
- assertTransformedExpression("reduce(rankingExpression(returns_tensor_with_arg@),max,x)",
- "max(returns_tensor_with_arg(attribute(tensor_field_1)),x)");
+ assertContainsExpression("max(returns_tensor,x)", "reduce(rankingExpression(returns_tensor),max,x)");
+ assertContainsExpression("max(wraps_returns_tensor,x)", "reduce(rankingExpression(wraps_returns_tensor),max,x)");
+ assertContainsExpression("max(tensor_inheriting,x)", "reduce(rankingExpression(tensor_inheriting),max,x)");
+ assertContainsExpression("max(returns_tensor_with_arg(attribute(tensor_field_1)),x)", "reduce(rankingExpression(returns_tensor_with_arg@),max,x)");
}
- private void assertTransformedExpression(String expected, String original) throws ParseException {
- for (Pair<String, String> rankPropertyExpression : buildSearch(original)) {
+ private void assertContainsExpression(String expr, String transformedExpression) throws ParseException {
+ assertTrue("Expected expression '" + transformedExpression + "' found",
+ containsExpression(expr, transformedExpression));
+ }
+
+ private boolean containsExpression(String expr, String transformedExpression) throws ParseException {
+ for (Pair<String, String> rankPropertyExpression : buildSearch(expr)) {
String rankProperty = rankPropertyExpression.getFirst();
if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) {
String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ",""));
- assertEquals(expected, rankExpression);
- return;
+ return rankExpression.equals(transformedExpression);
}
}
- fail("No 'rankingExpression(firstphase).rankingScript' property produced");
+ return false;
}
private List<Pair<String, String>> buildSearch(String expression) throws ParseException {
RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
- QueryProfileRegistry queryProfiles = setupQueryProfileTypes();
- SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles);
+ SearchBuilder builder = new SearchBuilder(rankProfileRegistry);
builder.importString(
"search test {\n" +
" document test { \n" +
@@ -198,16 +167,16 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
" }\n" +
" }\n" +
"}\n");
- builder.build(new BaseDeployLogger());
+ builder.build(new BaseDeployLogger(), setupQueryProfileTypes());
Search s = builder.getSearch();
- RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(queryProfiles);
+ RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry());
List<Pair<String, String>> testRankProperties = new RawRankProfile(test,
- queryProfiles,
+ new QueryProfileRegistry(),
new AttributeFields(s)).configProperties();
return testRankProperties;
}
- private static QueryProfileRegistry setupQueryProfileTypes() {
+ private static QueryProfiles setupQueryProfileTypes() {
QueryProfileRegistry registry = new QueryProfileRegistry();
QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry();
QueryProfileType type = new QueryProfileType(new ComponentId("testtype"));
@@ -216,7 +185,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase {
type.addField(new FieldDescription("ranking.features.query(n)",
FieldType.fromString("integer", typeRegistry)), typeRegistry);
typeRegistry.register(type);
- return registry;
+ return new QueryProfiles(registry);
}
private String censorBindingHash(String s) {
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java
index b9b377dc0ec..05a6773c5cb 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java
@@ -60,10 +60,6 @@ public final class ReferenceNode extends CompositeNode {
@Override
public String toString(SerializationContext context, Deque<String> path, CompositeNode parent) {
- return toString(context, path, true);
- }
-
- private String toString(SerializationContext context, Deque<String> path, boolean includeOutput) {
if (path == null)
path = new ArrayDeque<>();
String myName = this.name;
@@ -105,21 +101,14 @@ public final class ReferenceNode extends CompositeNode {
}
ret.append(")");
}
- if (includeOutput)
- ret.append(myOutput != null ? "." + myOutput : "");
+ ret.append(myOutput != null ? "." + myOutput : "");
return ret.toString();
}
@Override
public TensorType type(TypeContext context) {
- // Ensure base name (excluding output exists,
- // but don't support outputs of different tensor types (not used, so no need)
- String name = toString(new SerializationContext(), null, false);
- TensorType type = context.getType(name);
-
- if (type == null)
- throw new IllegalArgumentException("Unknown feature '" + toString() + "'");
- return type;
+ // Don't support outputs of different type, for simplicity
+ return context.getType(toString());
}
@Override
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java
index 416b74e7f94..76a938b9fe2 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java
@@ -106,7 +106,6 @@ public class Reduce extends PrimitiveTensorFunction {
}
private TensorType type(TensorType argumentType) {
- if (dimensions.isEmpty()) return TensorType.empty; // means reduce all
TensorType.Builder builder = new TensorType.Builder();
for (TensorType.Dimension dimension : argumentType.dimensions())
if ( ! dimensions.contains(dimension.name())) // keep