From 4062af167c5f1415d7f3f2eb0792447dafbf5096 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 6 Feb 2018 22:18:12 +0100 Subject: Revert "Bratseth/typecheck all" --- .../com/yahoo/config/model/deploy/DeployState.java | 2 +- .../com/yahoo/searchdefinition/FeatureNames.java | 12 -- .../com/yahoo/searchdefinition/MapTypeContext.java | 39 ------ .../com/yahoo/searchdefinition/RankProfile.java | 32 ++--- .../com/yahoo/searchdefinition/SearchBuilder.java | 23 ++-- .../com/yahoo/searchdefinition/TypeMapContext.java | 32 +++++ .../processing/IndexingValues.java | 2 +- .../searchdefinition/processing/Processing.java | 3 +- .../processing/RankingExpressionTypeValidator.java | 72 ----------- .../src/test/derived/tensor/rank-profiles.cfg | 2 +- config-model/src/test/derived/tensor/tensor.sd | 2 +- config-model/src/test/examples/rankpropvars.sd | 8 +- config-model/src/test/examples/simple.sd | 2 +- .../searchdefinition/RankProfileTestCase.java | 8 +- .../RankingExpressionConstantsTestCase.java | 3 - .../RankingExpressionShadowingTestCase.java | 40 +------ .../processing/RankProfileSearchFixture.java | 4 +- .../RankingExpressionTypeValidatorTestCase.java | 104 ---------------- .../RankingExpressionWithTensorFlowTestCase.java | 18 +-- .../processing/TensorTransformTestCase.java | 133 ++++++++------------- .../rankingexpression/rule/ReferenceNode.java | 17 +-- .../java/com/yahoo/tensor/functions/Reduce.java | 1 - 22 files changed, 148 insertions(+), 411 deletions(-) delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java create mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java delete mode 100644 config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 67724058e64..eb0c6067fca 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -333,7 +333,7 @@ public class DeployState implements ConfigDefinitionStore { closeIgnoreException(reader.getReader()); } } - builder.build(logger); + builder.build(logger, queryProfiles); return SearchDocumentModel.fromBuilderAndNames(builder, names); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index c01b009e93b..dd03cb8b2a7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -101,18 +101,6 @@ public class FeatureNames { return canonicalize("query(\"" + propertyName + "\")"); } - public static boolean isConstantFeature(String feature) { - return feature.startsWith("constant("); - } - - public static boolean isAttributeFeature(String feature) { - return feature.startsWith("attribute("); - } - - public static boolean isQueryFeature(String feature) { - return feature.startsWith("query("); - } - /** * Returns the single argument of the given feature name, without any quotes, * or empty if it is not a valid query, attribute or constant feature name diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java deleted file mode 100644 index e0dc7a2f33c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * A context which only contains type information. - * This returns empty tensor types (double) for unknown features which are not - * query, attribute or constant features, as we do not have information about which such - * features exist (but we know those that exist are doubles). - * - * @author bratseth - */ -public class MapTypeContext implements TypeContext { - - private final Map featureTypes = new HashMap<>(); - - public void setType(String name, TensorType type) { - featureTypes.put(FeatureNames.canonicalize(name), type); - } - - @Override - public TensorType getType(String name) { - if (FeatureNames.isConstantFeature(name) || - FeatureNames.isAttributeFeature(name) || - FeatureNames.isQueryFeature(name)) - return featureTypes.get(FeatureNames.canonicalize(name)); - else - return TensorType.empty; // we do not have type information for these. Correct would be either empty or null - } - - public Map bindings() { return Collections.unmodifiableMap(featureTypes); } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index d1a29271014..bcbc7cc99e2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -2,9 +2,15 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.TensorFieldType; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; @@ -13,6 +19,7 @@ import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.TypeMapContext; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.TensorType; @@ -578,11 +585,8 @@ public class RankProfile implements Serializable, Cloneable { } /** - * Will take the parser-set textual ranking expressions and turn into ranking expression objects, - * if not already done + * Will take the parser-set textual ranking expressions and turn into objects */ - // TODO: There doesn't appear to be any good reason to defer parsing of ranking expressions - // until this is called. Simplify by parsing them right away. public void parseExpressions() { try { parseRankingExpressions(); @@ -600,23 +604,20 @@ public class RankProfile implements Serializable, Cloneable { for (Map.Entry e : getMacros().entrySet()) { String macroName = e.getKey(); Macro macro = e.getValue(); - if (macro.getRankingExpression() == null) { - RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); - macro.setRankingExpression(expr); - macro.setTextualExpression(expr.getRoot().toString()); - } + RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); + macro.setRankingExpression(expr); + macro.setTextualExpression(expr.getRoot().toString()); } } /** * Passes ranking expressions on to parser - * * @throws ParseException if either of the ranking expressions could not be parsed */ private void parseRankingExpressions() throws ParseException { - if (getFirstPhaseRankingString() != null && firstPhaseRanking == null) + if (getFirstPhaseRankingString() != null) setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString())); - if (getSecondPhaseRankingString() != null && secondPhaseRanking == null) + if (getSecondPhaseRankingString() != null) setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); } @@ -747,7 +748,7 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public TypeContext typeContext(QueryProfileRegistry queryProfiles) { - MapTypeContext context = new MapTypeContext(); + TypeMapContext context = new TypeMapContext(); // Add small constants getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); @@ -763,8 +764,7 @@ public class RankProfile implements Serializable, Cloneable { for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { for (FieldDescription field : queryProfileType.declaredFields().values()) { TensorType type = field.getType().asTensorType(); - if ( ! FeatureNames.isQueryFeature(field.getName())) continue; - String feature = FeatureNames.canonicalize(field.getName()); + String feature = FeatureNames.asQueryFeature(field.getName()); TensorType existingType = context.getType(feature); if (existingType != null) type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () -> @@ -910,7 +910,7 @@ public class RankProfile implements Serializable, Cloneable { */ public static class Macro implements Serializable, Cloneable { - private final String name; + private String name=null; private String textualExpression=null; private RankingExpression expression=null; private List formalParams = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 469f29098ad..762c0fec838 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -34,6 +34,7 @@ import java.util.List; * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built * search objects using the {@link #getSearch(String)} method. */ +// TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage public class SearchBuilder { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); @@ -195,7 +196,11 @@ public class SearchBuilder { * @throws IllegalStateException Thrown if this method has already been called. */ public void build() { - build(new BaseDeployLogger()); + build(new BaseDeployLogger(), new QueryProfiles()); + } + + public void build(DeployLogger logger) { + build(logger, new QueryProfiles()); } /** @@ -204,10 +209,12 @@ public class SearchBuilder { * * @throws IllegalStateException Thrown if this method has already been called. * @param deployLogger The logger to use during build + * @param queryProfiles The query profiles contained in the application this search is part of. */ - public void build(DeployLogger deployLogger) { - if (isBuilt) throw new IllegalStateException("Model already built"); - + public void build(DeployLogger deployLogger, QueryProfiles queryProfiles) { + if (isBuilt) { + throw new IllegalStateException("Searches already built."); + } List built = new ArrayList<>(); List sdocs = new ArrayList<>(); sdocs.add(SDDocumentType.VESPA_DOCUMENT); @@ -233,7 +240,7 @@ public class SearchBuilder { for (Search search : new SearchOrderer().order(searchList)) { new FieldOperationApplierForSearch().process(search); // These two needed for a couple of old unit tests, ideally these are just read from app - process(search, deployLogger, new QueryProfiles(queryProfileRegistry)); + process(search, deployLogger, queryProfiles); built.add(search); } builder.addToModel(searchList); @@ -247,6 +254,8 @@ public class SearchBuilder { /** * Processes and returns the given {@link Search} object. This method has been factored out of the {@link * #build()} method so that subclasses can choose not to build anything. + * + * @param search The object to build. */ protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) { Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles); @@ -343,7 +352,7 @@ public class SearchBuilder { rankProfileRegistry, queryprofileRegistry); builder.importFile(fileName); - builder.build(deployLogger); + builder.build(deployLogger, new QueryProfiles()); return builder; } @@ -359,7 +368,7 @@ public class SearchBuilder { for (Iterator i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } - builder.build(new BaseDeployLogger()); + builder.build(new BaseDeployLogger(), new QueryProfiles()); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java new file mode 100644 index 00000000000..40e9db1413f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/TypeMapContext.java @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * A context which only contains type information. + * + * @author bratseth + */ +public class TypeMapContext implements TypeContext { + + private final Map featureTypes = new HashMap<>(); + + public void setType(String name, TensorType type) { + featureTypes.put(FeatureNames.canonicalize(name), type); + } + + @Override + public TensorType getType(String name) { + return featureTypes.get(FeatureNames.canonicalize(name)); + } + + /** Returns an unmodifiable map of the bindings in this */ + public Map bindings() { return Collections.unmodifiableMap(featureTypes); } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java index cc634abef01..ee65c9bec02 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; /** - * @author Simon Thoresen Hult + * @author Simon Thoresen Hult */ public class IndexingValues extends Processor { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 061a803cb48..90183848094 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -76,9 +76,8 @@ public class Processing { ImportedFieldsInSummayValidator::new, FastAccessValidator::new, ReservedMacroNames::new, - RankingExpressionTypeValidator::new, - // These should be last. + // These two should be last. IndexingValidation::new, IndexingValues::new); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java deleted file mode 100644 index a7a5ad58430..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.MapTypeContext; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Search; -import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -public class RankingExpressionTypeValidator extends Processor { - - private final QueryProfileRegistry queryProfiles; - - public RankingExpressionTypeValidator(Search search, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - this.queryProfiles = queryProfiles.getRegistry(); - } - - @Override - public void process() { - for (RankProfile profile : rankProfileRegistry.allRankProfiles()) { - try { - validate(profile); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("In " + search + ", " + profile, e); - } - } - } - - /** Throws an IllegalArgumentException if the given rank profile does not produce valid type */ - private void validate(RankProfile profile) { - profile.parseExpressions(); - TypeContext context = profile.typeContext(queryProfiles); - for (RankProfile.Macro macro : profile.getMacros().values()) - ensureValid(macro.getRankingExpression(), "macro '" + macro.getName() + "'", context); - ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); - ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); - } - - private TensorType ensureValid(RankingExpression expression, String expressionDescription, TypeContext context) { - if (expression == null) return null; - - TensorType type; - try { - type = expression.type(context); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("The " + expressionDescription + " is invalid", e); - } - if (type == null) // Not expected to happen - throw new IllegalStateException("Could not determine the type produced by " + expressionDescription); - return type; - } - - private void ensureValidDouble(RankingExpression expression, String expressionDescription, TypeContext context) { - if (expression == null) return; - TensorType type = ensureValid(expression, expressionDescription, context); - if ( ! type.equals(TensorType.empty)) - throw new IllegalArgumentException("The " + expressionDescription + " must produce a double " + - "(a tensor with no dimensions), but produces " + type); - } - -} diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index b6ad5372c05..2b231e0cda2 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -35,7 +35,7 @@ rankprofile[3].name "profile2" rankprofile[3].fef.property[0].name "vespa.rank.firstphase" rankprofile[3].fef.property[0].value "rankingExpression(firstphase)" rankprofile[3].fef.property[1].name "rankingExpression(firstphase).rankingScript" -rankprofile[3].fef.property[1].value "reduce(reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" +rankprofile[3].fef.property[1].value "reduce(join(attribute(f4), tensor(x[2],y[2],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x)" rankprofile[3].fef.property[2].name "vespa.type.attribute.f2" rankprofile[3].fef.property[2].value "tensor(x[2],y[])" rankprofile[3].fef.property[3].name "vespa.type.attribute.f3" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index 3d64f6b807e..a6a9a98db3a 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -28,7 +28,7 @@ search tensor { rank-profile profile2 { first-phase { - expression: sum(matmul(attribute(f4), diag(x[2],y[2],z[3]), x)) + expression: matmul(attribute(f4), diag(x[2],y[2],z[3]), x) } } diff --git a/config-model/src/test/examples/rankpropvars.sd b/config-model/src/test/examples/rankpropvars.sd index 28959edbc09..40f9e73f35a 100644 --- a/config-model/src/test/examples/rankpropvars.sd +++ b/config-model/src/test/examples/rankpropvars.sd @@ -18,8 +18,8 @@ first-phase { second-phase { expression { if (attribute(artist) == query(testvar1), - 0.0 * fieldMatch(title) + 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist), - 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) + 0.0 * fieldMatch(title) + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist), + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) } } @@ -42,8 +42,8 @@ first-phase { second-phase { expression { if (attribute(artist) == query(testvar1), - 0.0 * fieldMatch(title) + 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist), - 0.0 * attribute(Popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) + 0.0 * fieldMatch(title) + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist), + 0.0 * attribute(popularity) + 0.0 * fieldMatch(artist) + 0.0 * fieldMatch(title)) } } } diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd index 96b0fa98098..4fda7f5039e 100644 --- a/config-model/src/test/examples/simple.sd +++ b/config-model/src/test/examples/simple.sd @@ -116,7 +116,7 @@ search simple { first-phase { keep-rank-count:200 rank-score-drop-limit: -13.0 - expression: attribute(popularity) + expression: attribute(year) } second-phase { rerank-count: 99 diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 11093d9f008..442c8bd41bd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -135,13 +135,13 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { @Test public void requireThatConfigIsDerivedForQueryFeatureTypeSettings() throws ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - SearchBuilder builder = new SearchBuilder(registry, setupQueryProfileTypes()); + SearchBuilder builder = new SearchBuilder(registry); builder.importString("search test {\n" + " document test { } \n" + " rank-profile p1 {}\n" + " rank-profile p2 {}\n" + "}"); - builder.build(new BaseDeployLogger()); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); Search search = builder.getSearch(); assertEquals(4, registry.allRankProfiles().size()); @@ -151,7 +151,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { assertQueryFeatureTypeSettings(registry.getRankProfile(search, "p2"), search); } - private static QueryProfileRegistry setupQueryProfileTypes() { + private static QueryProfiles setupQueryProfileTypes() { QueryProfileRegistry registry = new QueryProfileRegistry(); QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); @@ -164,7 +164,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { type.addField(new FieldDescription("ranking.features.query(numeric)", FieldType.fromString("integer", typeRegistry)), typeRegistry); typeRegistry.register(type); - return registry; + return new QueryProfiles(registry); } private static void assertQueryFeatureTypeSettings(RankProfile profile, Search search) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 82b9f5ac043..e94880e61c7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -207,9 +207,6 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.importString( "search test {\n" + " document test { \n" + - " field rating_yelp type int {" + - " indexing: attribute" + - " }" + " }\n" + " \n" + " rank-profile test {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index ed1b00e2875..5100ac15c40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -2,10 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; -import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.types.FieldDescription; -import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -152,12 +149,11 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase censorBindingHash(testRankProperties.get(4).toString())); } + @Test public void testNeuralNetworkSetup() throws ParseException { - // Note: the type assigned to query profile and constant tensors here is not the correct type RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = queryProfileWith("query(q)", "tensor(x[])"); - SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( "search test {\n" + " document test { \n" + @@ -180,28 +176,13 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase " expression: sum(final_layer)\n" + " }\n" + " }\n" + - " constant W_hidden {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + - " constant b_input {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + - " constant W_final {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + - " constant b_final {\n" + - " type: tensor(x[])\n" + - " file: ignored.json\n" + - " }\n" + + "\n" + "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(queryProfiles); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); List> testRankProperties = new RawRankProfile(test, - queryProfiles, + new QueryProfileRegistry(), new AttributeFields(s)).configProperties(); assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(0).toString()); @@ -217,17 +198,6 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase testRankProperties.get(5).toString()); } - private QueryProfileRegistry queryProfileWith(String field, String type) { - QueryProfileType queryProfileType = new QueryProfileType("root"); - queryProfileType.addField(new FieldDescription(field, type)); - QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); - queryProfileRegistry.getTypeRegistry().register(queryProfileType); - QueryProfile profile = new QueryProfile("default"); - profile.setType(queryProfileType); - queryProfileRegistry.register(profile); - return queryProfileRegistry; - } - private String censorBindingHash(String s) { StringBuilder b = new StringBuilder(); boolean areInHash = false; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 0ce6129ef7f..800697b3430 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -38,8 +38,7 @@ class RankProfileSearchFixture { RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, String rankProfiles, String constant, String field) throws ParseException { - this.queryProfileRegistry = queryProfileRegistry; - SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, queryProfileRegistry); + SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, new QueryProfileRegistry()); String sdContent = "search test {\n" + " " + (constant != null ? constant : "") + "\n" + " document test {\n" + @@ -51,6 +50,7 @@ class RankProfileSearchFixture { builder.importString(sdContent); builder.build(); search = builder.getSearch(); + this.queryProfileRegistry = queryProfileRegistry; } public void assertFirstPhaseExpression(String expExpression, String rankProfile) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java deleted file mode 100644 index db3b12db1bf..00000000000 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidatorTestCase.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.yolean.Exceptions; -import org.junit.Test; -import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class RankingExpressionTypeValidatorTestCase { - - @Test - public void tensorFirstPhaseMustProduceDouble() throws Exception { - try { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry); - searchBuilder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: attribute(a)", - " }", - " }", - "}" - )); - searchBuilder.build(); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void tensorSecondPhaseMustProduceDouble() throws Exception { - try { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry); - searchBuilder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(attribute(a))", - " }", - " second-phase {", - " expression: attribute(a)", - " }", - " }", - "}" - )); - searchBuilder.build(); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The second-phase expression must produce a double (a tensor with no dimensions), but produces tensor(x[],y[])", - Exceptions.toMessageString(expected)); - } - } - - @Test - public void tensorConditionsMustHaveTypeCompatibleBranches() throws Exception { - try { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry); - searchBuilder.importString(joinLines( - "search test {", - " document test { ", - " field a type tensor(x[],y[]) {", - " indexing: attribute", - " }", - " field b type tensor(z[10]) {", - " indexing: attribute", - " }", - " }", - " rank-profile my_rank_profile {", - " first-phase {", - " expression: sum(if(1>0, attribute(a), attribute(b)))", - " }", - " }", - "}" - )); - searchBuilder.build(); - fail("Expected exception"); - } - catch (IllegalArgumentException expected) { - assertEquals("In search definition 'test', rank profile 'my_rank_profile': The first-phase expression is invalid: An if expression must produce compatible types in both alternatives, but the 'true' type is tensor(x[],y[]) while the 'false' type is tensor(z[10])", - Exceptions.toMessageString(expected)); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index 58af8daf1b5..7246b22b0f8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -51,7 +51,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReference() { + public void testTensorFlowReference() throws ParseException { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", "tensorflow('mnist_softmax/saved')"); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); @@ -60,7 +60,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceWithConstantFeature() { + public void testTensorFlowReferenceWithConstantFeature() throws ParseException { RankProfileSearchFixture search = fixtureWith("constant(mytensor)", "tensorflow('mnist_softmax/saved')", "constant mytensor { file: ignored\ntype: tensor(d0[7],d1[784]) }", @@ -71,10 +71,10 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceWithQueryFeature() { + public void testTensorFlowReferenceWithQueryFeature() throws ParseException { String queryProfile = ""; String queryProfileType = "" + - " " + + " " + ""; StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, @@ -90,7 +90,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceWithDocumentFeature() { + public void testTensorFlowReferenceWithDocumentFeature() throws ParseException { StoringApplicationPackage application = new StoringApplicationPackage(applicationDir); RankProfileSearchFixture search = fixtureWith("attribute(mytensor)", "tensorflow('mnist_softmax/saved')", @@ -103,10 +103,10 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceWithFeatureCombination() { + public void testTensorFlowReferenceWithFeatureCombination() throws ParseException { String queryProfile = ""; String queryProfileType = "" + - " " + + " " + ""; StoringApplicationPackage application = new StoringApplicationPackage(applicationDir, queryProfile, @@ -122,7 +122,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testNestedTensorFlowReference() { + public void testNestedTensorFlowReference() throws ParseException { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", "5 + sum(tensorflow('mnist_softmax/saved'))"); search.assertFirstPhaseExpression("5 + reduce(" + vespaExpression + ", sum)", "my_profile"); @@ -131,7 +131,7 @@ public class RankingExpressionWithTensorFlowTestCase { } @Test - public void testTensorFlowReferenceSpecifyingSignature() { + public void testTensorFlowReferenceSpecifyingSignature() throws ParseException { RankProfileSearchFixture search = fixtureWith("tensor(d0[2],d1[784])(0.0)", "tensorflow('mnist_softmax/saved', 'serving_default')"); search.assertFirstPhaseExpression(vespaExpression, "my_profile"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index d2211b86c9e..c18cfcfe1aa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -17,129 +17,98 @@ import com.yahoo.searchdefinition.SearchDefinitionTestCase; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; import java.util.List; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class TensorTransformTestCase extends SearchDefinitionTestCase { @Test public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException { - assertTransformedExpression("max(1.0,2.0)", - "max(1.0,2.0)"); - assertTransformedExpression("min(attribute(double_field),x)", - "min(attribute(double_field),x)"); - assertTransformedExpression("max(attribute(double_field),attribute(double_array_field))", - "max(attribute(double_field),attribute(double_array_field))"); - assertTransformedExpression("min(attribute(tensor_field_1),attribute(double_field))", - "min(attribute(tensor_field_1),attribute(double_field))"); - assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)", - "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),sum)"); - assertTransformedExpression("min(constant(test_constant_tensor),1.0)", - "min(test_constant_tensor,1.0)"); - assertTransformedExpression("max(constant(base_constant_tensor),1.0)", - "max(base_constant_tensor,1.0)"); - assertTransformedExpression("min(constant(file_constant_tensor),1.0)", - "min(constant(file_constant_tensor),1.0)"); - assertTransformedExpression("max(query(q),1.0)", - "max(query(q),1.0)"); - assertTransformedExpression("max(query(n),1.0)", - "max(query(n),1.0)"); + assertContainsExpression("max(1.0,2.0)", "max(1.0,2.0)"); + assertContainsExpression("min(attribute(double_field),x)", "min(attribute(double_field),x)"); + assertContainsExpression("max(attribute(double_field),attribute(double_array_field))", "max(attribute(double_field),attribute(double_array_field))"); + assertContainsExpression("min(attribute(tensor_field_1),attribute(double_field))", "min(attribute(tensor_field_1),attribute(double_field))"); + assertContainsExpression("max(attribute(tensor_field_1),attribute(tensor_field_2))", "max(attribute(tensor_field_1),attribute(tensor_field_2))"); + assertContainsExpression("min(test_constant_tensor,1.0)", "min(constant(test_constant_tensor),1.0)"); + assertContainsExpression("max(base_constant_tensor,1.0)", "max(constant(base_constant_tensor),1.0)"); + assertContainsExpression("min(constant(file_constant_tensor),1.0)", "min(constant(file_constant_tensor),1.0)"); + assertContainsExpression("max(query(q),1.0)", "max(query(q),1.0)"); + assertContainsExpression("max(query(n),1.0)", "max(query(n),1.0)"); } @Test public void requireThatMaxAndMinWithTensorAttributesAreReplaced() throws ParseException { - assertTransformedExpression("reduce(attribute(tensor_field_1),max,x)", - "max(attribute(tensor_field_1),x)"); - assertTransformedExpression("1+reduce(attribute(tensor_field_1),max,x)", - "1 + max(attribute(tensor_field_1),x)"); - assertTransformedExpression("if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)", - "if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)"); - assertTransformedExpression("reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)", - "max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)"); - assertTransformedExpression("reduce(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),max,x)", - "max(if(attribute(double_field),attribute(tensor_field_2),attribute(tensor_field_2)),x)"); - assertTransformedExpression("max(reduce(attribute(tensor_field_1),max,x),x)", - "max(max(attribute(tensor_field_1),x),x)"); // will result in deploy error. - assertTransformedExpression("reduce(reduce(attribute(tensor_field_2),max,x),max,y)", - "max(max(attribute(tensor_field_2),x),y)"); + assertContainsExpression("max(attribute(tensor_field_1),x)", "reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("1 + max(attribute(tensor_field_1),x)", "1+reduce(attribute(tensor_field_1),max,x)"); + assertContainsExpression("if(attribute(double_field),1 + max(attribute(tensor_field_1),x),0)", "if(attribute(double_field),1+reduce(attribute(tensor_field_1),max,x),0)"); + assertContainsExpression("max(max(attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(max(attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),x)", "reduce(if(attribute(double_field),attribute(tensor_field_1),attribute(tensor_field_2)),max,x)"); + assertContainsExpression("max(max(attribute(tensor_field_1),x),x)", "max(reduce(attribute(tensor_field_1),max,x),x)"); // will result in deploy error. + assertContainsExpression("max(max(attribute(tensor_field_2),x),y)", "reduce(reduce(attribute(tensor_field_2),max,x),max,y)"); } @Test public void requireThatMaxAndMinWithConstantTensorsAreReplaced() throws ParseException { - assertTransformedExpression("reduce(constant(test_constant_tensor),max,x)", - "max(test_constant_tensor,x)"); - assertTransformedExpression("reduce(constant(base_constant_tensor),max,x)", - "max(base_constant_tensor,x)"); - assertTransformedExpression("reduce(constant(file_constant_tensor),min,x)", - "min(constant(file_constant_tensor),x)"); + assertContainsExpression("max(test_constant_tensor,x)", "reduce(constant(test_constant_tensor),max,x)"); + assertContainsExpression("max(base_constant_tensor,x)", "reduce(constant(base_constant_tensor),max,x)"); + assertContainsExpression("min(constant(file_constant_tensor),x)", "reduce(constant(file_constant_tensor),min,x)"); } @Test public void requireThatMaxAndMinWithTensorExpressionsAreReplaced() throws ParseException { - assertTransformedExpression("reduce(attribute(double_field)+attribute(tensor_field_1),min,x)", - "min(attribute(double_field) + attribute(tensor_field_1),x)"); - assertTransformedExpression("reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)", - "min(attribute(tensor_field_1) * attribute(tensor_field_2),x)"); - assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)" - , "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)"); - assertTransformedExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", - "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) - assertTransformedExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", - "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); + assertContainsExpression("min(attribute(double_field) + attribute(tensor_field_1),x)", "reduce(attribute(double_field)+attribute(tensor_field_1),min,x)"); + assertContainsExpression("min(attribute(tensor_field_1) * attribute(tensor_field_2),x)", "reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)"); + assertContainsExpression("min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)", "reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)"); + assertContainsExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) + assertContainsExpression("min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)", "min(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),x)"); } @Test public void requireThatMaxAndMinWithTensorFromIsReplaced() throws ParseException { - assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)", - "max(tensorFromLabels(attribute(double_array_field)),double_array_field)"); - assertTransformedExpression("reduce(tensorFromLabels(attribute(double_array_field),x),max,x)", - "max(tensorFromLabels(attribute(double_array_field),x),x)"); - assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)", - "max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)"); - assertTransformedExpression("reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)", - "max(tensorFromWeightedSet(attribute(weightedset_field),x),x)"); + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field)),double_array_field)", "reduce(tensorFromLabels(attribute(double_array_field)),max,double_array_field)"); + assertContainsExpression("max(tensorFromLabels(attribute(double_array_field),x),x)", "reduce(tensorFromLabels(attribute(double_array_field),x),max,x)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field)),weightedset_field)", "reduce(tensorFromWeightedSet(attribute(weightedset_field)),max,weightedset_field)"); + assertContainsExpression("max(tensorFromWeightedSet(attribute(weightedset_field),x),x)", "reduce(tensorFromWeightedSet(attribute(weightedset_field),x),max,x)"); } @Test public void requireThatMaxAndMinWithTensorInQueryIsReplaced() throws ParseException { - assertTransformedExpression("reduce(query(q),max,x)", "max(query(q),x)"); - assertTransformedExpression("max(query(n),x)", "max(query(n),x)"); + assertContainsExpression("max(query(q),x)", "reduce(query(q),max,x)"); + assertContainsExpression("max(query(n),x)", "max(query(n),x)"); } @Test public void requireThatMaxAndMinWithTensoresReturnedFromMacrosAreReplaced() throws ParseException { - assertTransformedExpression("reduce(rankingExpression(returns_tensor),max,x)", - "max(returns_tensor,x)"); - assertTransformedExpression("reduce(rankingExpression(wraps_returns_tensor),max,x)", - "max(wraps_returns_tensor,x)"); - assertTransformedExpression("reduce(rankingExpression(tensor_inheriting),max,x)", - "max(tensor_inheriting,x)"); - assertTransformedExpression("reduce(rankingExpression(returns_tensor_with_arg@),max,x)", - "max(returns_tensor_with_arg(attribute(tensor_field_1)),x)"); + assertContainsExpression("max(returns_tensor,x)", "reduce(rankingExpression(returns_tensor),max,x)"); + assertContainsExpression("max(wraps_returns_tensor,x)", "reduce(rankingExpression(wraps_returns_tensor),max,x)"); + assertContainsExpression("max(tensor_inheriting,x)", "reduce(rankingExpression(tensor_inheriting),max,x)"); + assertContainsExpression("max(returns_tensor_with_arg(attribute(tensor_field_1)),x)", "reduce(rankingExpression(returns_tensor_with_arg@),max,x)"); } - private void assertTransformedExpression(String expected, String original) throws ParseException { - for (Pair rankPropertyExpression : buildSearch(original)) { + private void assertContainsExpression(String expr, String transformedExpression) throws ParseException { + assertTrue("Expected expression '" + transformedExpression + "' found", + containsExpression(expr, transformedExpression)); + } + + private boolean containsExpression(String expr, String transformedExpression) throws ParseException { + for (Pair rankPropertyExpression : buildSearch(expr)) { String rankProperty = rankPropertyExpression.getFirst(); if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) { String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); - assertEquals(expected, rankExpression); - return; + return rankExpression.equals(transformedExpression); } } - fail("No 'rankingExpression(firstphase).rankingScript' property produced"); + return false; } private List> buildSearch(String expression) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfileRegistry queryProfiles = setupQueryProfileTypes(); - SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfiles); + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( "search test {\n" + " document test { \n" + @@ -198,16 +167,16 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { " }\n" + " }\n" + "}\n"); - builder.build(new BaseDeployLogger()); + builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(queryProfiles); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); List> testRankProperties = new RawRankProfile(test, - queryProfiles, + new QueryProfileRegistry(), new AttributeFields(s)).configProperties(); return testRankProperties; } - private static QueryProfileRegistry setupQueryProfileTypes() { + private static QueryProfiles setupQueryProfileTypes() { QueryProfileRegistry registry = new QueryProfileRegistry(); QueryProfileTypeRegistry typeRegistry = registry.getTypeRegistry(); QueryProfileType type = new QueryProfileType(new ComponentId("testtype")); @@ -216,7 +185,7 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { type.addField(new FieldDescription("ranking.features.query(n)", FieldType.fromString("integer", typeRegistry)), typeRegistry); typeRegistry.register(type); - return registry; + return new QueryProfiles(registry); } private String censorBindingHash(String s) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index b9b377dc0ec..05a6773c5cb 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -60,10 +60,6 @@ public final class ReferenceNode extends CompositeNode { @Override public String toString(SerializationContext context, Deque path, CompositeNode parent) { - return toString(context, path, true); - } - - private String toString(SerializationContext context, Deque path, boolean includeOutput) { if (path == null) path = new ArrayDeque<>(); String myName = this.name; @@ -105,21 +101,14 @@ public final class ReferenceNode extends CompositeNode { } ret.append(")"); } - if (includeOutput) - ret.append(myOutput != null ? "." + myOutput : ""); + ret.append(myOutput != null ? "." + myOutput : ""); return ret.toString(); } @Override public TensorType type(TypeContext context) { - // Ensure base name (excluding output exists, - // but don't support outputs of different tensor types (not used, so no need) - String name = toString(new SerializationContext(), null, false); - TensorType type = context.getType(name); - - if (type == null) - throw new IllegalArgumentException("Unknown feature '" + toString() + "'"); - return type; + // Don't support outputs of different type, for simplicity + return context.getType(toString()); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java index 416b74e7f94..76a938b9fe2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java @@ -106,7 +106,6 @@ public class Reduce extends PrimitiveTensorFunction { } private TensorType type(TensorType argumentType) { - if (dimensions.isEmpty()) return TensorType.empty; // means reduce all TensorType.Builder builder = new TensorType.Builder(); for (TensorType.Dimension dimension : argumentType.dimensions()) if ( ! dimensions.contains(dimension.name())) // keep -- cgit v1.2.3