diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-02-05 12:43:36 +0100 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-02-05 12:43:36 +0100 |
commit | 5794c369b4dcfdc85163158d49788d6303bbf564 (patch) | |
tree | 065786cd1ca8d4e0852252ca4dacbe479e268520 /config-model/src | |
parent | ab84971140f1782c0c7a693dd45c324766094c7c (diff) |
Use query profiles already read
Diffstat (limited to 'config-model/src')
34 files changed, 405 insertions, 214 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 24e68965e9c..eb0c6067fca 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -308,10 +308,12 @@ public class DeployState implements ConfigDefinitionStore { zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion, disableFiledistributor); } - private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) { + private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, + DeployLogger logger, + QueryProfiles queryProfiles) { Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); Map<String, String> names = new LinkedHashMap<>(); - SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry); + SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry, queryProfiles.getRegistry()); for (NamedReader reader : readers) { try { String readerName = reader.getName(); diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 271ec6958ec..6a8d754af1c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.Version; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.searchdefinition.*; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -36,9 +38,11 @@ public class MockApplicationPackage implements ApplicationPackage { private final Optional<String> deploymentSpec; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; + private final QueryProfileRegistry queryProfileRegistry; protected MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir, - String deploymentSpec, String validationOverrides, boolean failOnValidateXml) { + String deploymentSpec, String validationOverrides, boolean failOnValidateXml, + String queryProfile, String queryProfileType) { this.hostsS = hosts; this.servicesS = services; this.searchDefinitions = searchDefinitions; @@ -46,6 +50,8 @@ public class MockApplicationPackage implements ApplicationPackage { this.deploymentSpec = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; + queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), + asNamedReaderList(queryProfile)); } @Override @@ -67,7 +73,9 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getSearchDefinitions() { ArrayList<NamedReader> readers = new ArrayList<>(); - SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry()); + SearchBuilder searchBuilder = new SearchBuilder(this, + new RankProfileRegistry(), + queryProfileRegistry); for (String sd : searchDefinitions) { try { String name = searchBuilder.importString(sd); @@ -123,6 +131,8 @@ public class MockApplicationPackage implements ApplicationPackage { return Collections.emptyList(); } + public QueryProfileRegistry getQueryProfiles() { return queryProfileRegistry; } + @Override public Reader getRankingExpression(String name) { File expressionFile = new File(searchDefinitionDir, name); @@ -147,6 +157,7 @@ public class MockApplicationPackage implements ApplicationPackage { } public static class Builder { + private String hosts = null; private String services = null; private List<String> searchDefinitions = Collections.emptyList(); @@ -154,6 +165,8 @@ public class MockApplicationPackage implements ApplicationPackage { private String deploymentSpec = null; private String validationOverrides = null; private boolean failOnValidateXml = false; + private String queryProfile = null; + private String queryProfileType = null; public Builder() { } @@ -206,9 +219,20 @@ public class MockApplicationPackage implements ApplicationPackage { return this; } + public Builder queryProfile(String queryProfile) { + this.queryProfile = queryProfile; + return this; + } + + public Builder queryProfileType(String queryProfileType) { + this.queryProfileType = queryProfileType; + return this; + } + public ApplicationPackage build() { return new MockApplicationPackage(hosts, services, searchDefinitions, searchDefinitionDir, - deploymentSpec, validationOverrides, failOnValidateXml); + deploymentSpec, validationOverrides, failOnValidateXml, + queryProfile, queryProfileType); } } @@ -242,4 +266,16 @@ public class MockApplicationPackage implements ApplicationPackage { } } + private List<NamedReader> asNamedReaderList(String value) { + if (value == null) return Collections.emptyList(); + return Collections.singletonList(new NamedReader(extractId(value) + ".xml", new StringReader(value))); + } + + private String extractId(String xmlStringWithIdAttribute) { + int idStart = xmlStringWithIdAttribute.indexOf("id="); + int idEnd = Math.min(xmlStringWithIdAttribute.indexOf(" ", idStart), + xmlStringWithIdAttribute.indexOf(">", idStart)); + return xmlStringWithIdAttribute.substring(idStart + 4, idEnd - 1); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 90ea5a11486..b4d72f3a456 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -43,20 +43,18 @@ import static java.util.stream.Collectors.toSet; */ public class DocumentModelBuilder { - public static class RetryLaterException extends IllegalArgumentException { - public RetryLaterException(String message) { - super(message); - } - } private DocumentModel model; private final Map<NewDocumentType, List<SDDocumentType>> scratchInheritsMap = new HashMap<>(); + public DocumentModelBuilder(DocumentModel model) { this.model = model; model.getDocumentManager().add(VespaDocumentType.INSTANCE); } + public boolean valid() { return scratchInheritsMap.isEmpty(); } + public void addToModel(Collection<Search> searchList) { List<SDDocumentType> docList = new LinkedList<>(); for (Search search : searchList) { @@ -65,7 +63,8 @@ public class DocumentModelBuilder { docList = sortDocumentTypes(docList); addDocumentTypes(docList); for (Collection<Search> toAdd = tryAdd(searchList); - !toAdd.isEmpty() && (toAdd.size() < searchList.size()); toAdd = tryAdd(searchList)) { + ! toAdd.isEmpty() && (toAdd.size() < searchList.size()); + toAdd = tryAdd(searchList)) { searchList = toAdd; } } @@ -126,6 +125,7 @@ public class DocumentModelBuilder { } return left; } + public void addToModel(Search search) { // Then we add the search specific stuff SearchDef searchDef = new SearchDef(search.getName()); @@ -133,9 +133,9 @@ public class DocumentModelBuilder { for (Field f : search.getDocument().fieldSet()) { addSearchField((SDField) f, searchDef); } - for(SDField field : search.allConcreteFields()) { - for(Attribute attribute : field.getAttributes().values()) { - if (!searchDef.getFields().containsKey(attribute.getName())) { + for (SDField field : search.allConcreteFields()) { + for (Attribute attribute : field.getAttributes().values()) { + if ( ! searchDef.getFields().containsKey(attribute.getName())) { searchDef.add(new SearchField(new Field(attribute.getName(), field), !field.getIndices().isEmpty(), true)); } } @@ -146,14 +146,15 @@ public class DocumentModelBuilder { } model.getSearchManager().add(searchDef); } + private static void addSearchFields(Collection<SDField> fields, SearchDef searchDef) { for (SDField field : fields) { addSearchField(field, searchDef); } } - private static void addSearchField(SDField field, SearchDef searchDef) { - SearchField searchField = + private static void addSearchField(SDField field, SearchDef searchDef) { + SearchField searchField = new SearchField(field, field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), field.getAttributes().containsKey(field.getName())); @@ -453,4 +454,11 @@ public class DocumentModelBuilder { } return false; } + + public static class RetryLaterException extends IllegalArgumentException { + public RetryLaterException(String message) { + super(message); + } + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 135d983c1ca..18deab70f3e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.reader.NamedReader; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.profile.QueryProfile; @@ -676,10 +677,10 @@ public class RankProfile implements Serializable, Cloneable { * Returns a copy of this where the content is optimized for execution. * Compiled profiles should never be modified. */ - public RankProfile compile() { + public RankProfile compile(QueryProfileRegistry queryProfiles) { try { RankProfile compiled = this.clone(); - compiled.compileThis(); + compiled.compileThis(queryProfiles); return compiled; } catch (IllegalArgumentException e) { @@ -687,7 +688,7 @@ public class RankProfile implements Serializable, Cloneable { } } - private void compileThis() { + private void compileThis(QueryProfileRegistry queryProfiles) { parseExpressions(); checkNameCollisions(getMacros(), getConstants()); @@ -696,13 +697,14 @@ public class RankProfile implements Serializable, Cloneable { for (Map.Entry<String, Macro> macroEntry : getMacros().entrySet()) { Macro compiledMacro = macroEntry.getValue().clone(); compiledMacro.setRankingExpression(compile(macroEntry.getValue().getRankingExpression(), - getConstants(), Collections.<String, Macro>emptyMap())); + queryProfiles, + getConstants(), Collections.<String, Macro>emptyMap())); compiledMacros.put(macroEntry.getKey(), compiledMacro); } macros = compiledMacros; Map<String, Macro> inlineMacros = keepInline(compiledMacros); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), getConstants(), inlineMacros); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), getConstants(), inlineMacros); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, getConstants(), inlineMacros); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, getConstants(), inlineMacros); } private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) { @@ -722,12 +724,14 @@ public class RankProfile implements Serializable, Cloneable { } private RankingExpression compile(RankingExpression expression, + QueryProfileRegistry queryProfiles, Map<String, Value> constants, Map<String, Macro> inlineMacros) { if (expression == null) return null; Map<String, String> rankPropertiesOutput = new HashMap<>(); RankProfileTransformContext context = new RankProfileTransformContext(this, + queryProfiles, constants, inlineMacros, rankPropertiesOutput); @@ -742,7 +746,7 @@ public class RankProfile implements Serializable, Cloneable { * Creates a context containing the type information of all constants, attributes and query profiles * referable from this rank profile. */ - public TypeContext typeContext() { + public TypeContext typeContext(QueryProfileRegistry queryProfiles) { TypeMapContext context = new TypeMapContext(); // Add constants @@ -754,7 +758,7 @@ public class RankProfile implements Serializable, Cloneable { } // Add query features from rank profile types reached from the "default" profile - QueryProfile profile = queryProfilesOf(getSearch().sourceApplication()).getComponent("default"); + QueryProfile profile = queryProfiles.getComponent("default"); if (profile != null && profile.getType() != null) { profile.listTypes(CompoundName.empty, Collections.emptyMap()).forEach((prefix, queryProfileType) -> { for (FieldDescription field : queryProfileType.declaredFields().values()) { @@ -771,20 +775,6 @@ public class RankProfile implements Serializable, Cloneable { return context; } - private QueryProfileRegistry queryProfilesOf(ApplicationPackage applicationPackage) { - List<NamedReader> queryProfileFiles = null; - List<NamedReader> queryProfileTypeFiles = null; - try { - queryProfileFiles = applicationPackage.getQueryProfileFiles(); - queryProfileTypeFiles = applicationPackage.getQueryProfileTypeFiles(); - return new QueryProfileXMLReader().read(queryProfileTypeFiles, queryProfileFiles); - } - finally { - NamedReader.closeAll(queryProfileFiles); - NamedReader.closeAll(queryProfileTypeFiles); - } - } - /** * A rank setting. The identity of a rank setting is its field name and type (not value). * A rank setting is immutable. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 0bccbc9c433..762c0fec838 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.SearchOrderer; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; @@ -32,8 +33,6 @@ import java.util.List; * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built * search objects using the {@link #getSearch(String)} method. - * - * @author TODO: Who created this? */ // TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage public class SearchBuilder { @@ -43,23 +42,35 @@ public class SearchBuilder { private ApplicationPackage app = null; private boolean isBuilt = false; private DocumentModel model = new DocumentModel(); - private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private final RankProfileRegistry rankProfileRegistry; + private final QueryProfileRegistry queryProfileRegistry; + /** For testing only */ public SearchBuilder() { - this.app = MockApplicationPackage.createEmpty(); + this(MockApplicationPackage.createEmpty(), new RankProfileRegistry(), new QueryProfileRegistry()); } + /** For testing only */ public SearchBuilder(ApplicationPackage app) { - this.app = app; + this(app, new RankProfileRegistry(), new QueryProfileRegistry()); } - public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) { - this.app = app; - this.rankProfileRegistry = rankProfileRegistry; + /** For testing only */ + public SearchBuilder(RankProfileRegistry rankProfileRegistry) { + this(MockApplicationPackage.createEmpty(), rankProfileRegistry, new QueryProfileRegistry()); } - public SearchBuilder(RankProfileRegistry rankProfileRegistry) { - this(MockApplicationPackage.createEmpty(), rankProfileRegistry); + /** For testing only */ + public SearchBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + this(MockApplicationPackage.createEmpty(), rankProfileRegistry, queryProfileRegistry); + } + + public SearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + this.app = app; + this.rankProfileRegistry = rankProfileRegistry; + this.queryProfileRegistry = queryProfileRegistry; } /** @@ -164,12 +175,12 @@ public class SearchBuilder { String rawName = rawSearch.getName(); if (rawSearch.isProcessed()) { throw new IllegalArgumentException("A search definition with a search section called '" + rawName + - "' has already been processed."); + "' has already been processed."); } for (Search search : searchList) { if (rawName.equals(search.getName())) { throw new IllegalArgumentException("A search definition with a search section called '" + rawName + - "' has already been added."); + "' has already been added."); } } searchList.add(rawSearch); @@ -319,7 +330,7 @@ public class SearchBuilder { * @throws ParseException if there was a problem parsing the file content. */ public static SearchBuilder createFromFile(String fileName) throws IOException, ParseException { - return createFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry()); + return createFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry()); } /** @@ -332,19 +343,28 @@ public class SearchBuilder { * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - public static SearchBuilder createFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) + public static SearchBuilder createFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { - SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), rankProfileRegistry); + SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), + rankProfileRegistry, + queryprofileRegistry); builder.importFile(fileName); builder.build(deployLogger, new QueryProfiles()); return builder; } public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException { - return createFromDirectory(dir, new RankProfileRegistry()); - } - public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry); + return createFromDirectory(dir, new RankProfileRegistry(), new QueryProfileRegistry()); + } + public static SearchBuilder createFromDirectory(String dir, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), + rankProfileRegistry, + queryProfileRegistry); for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } @@ -363,7 +383,7 @@ public class SearchBuilder { * @throws ParseException Thrown if there was a problem parsing the file content. */ public static Search buildFromFile(String fileName) throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry()); + return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry()); } /** @@ -375,9 +395,11 @@ public class SearchBuilder { * @throws IOException Thrown if there was a problem reading the file. * @throws ParseException Thrown if there was a problem parsing the file content. */ - public static Search buildFromFile(String fileName, RankProfileRegistry rankProfileRegistry) + public static Search buildFromFile(String fileName, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry); + return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry); } /** @@ -390,20 +412,25 @@ public class SearchBuilder { * @throws IOException Thrown if there was a problem reading the file. * @throws ParseException Thrown if there was a problem parsing the file content. */ - public static Search buildFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) + public static Search buildFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - return createFromFile(fileName, deployLogger, rankProfileRegistry).getSearch(); + return createFromFile(fileName, deployLogger, rankProfileRegistry, queryProfileRegistry).getSearch(); } /** * Convenience factory method to import and build a {@link Search} object from a raw object. * - * @param rawSearch The raw object to build from. - * @return The built {@link SearchBuilder} object. + * @param rawSearch the raw object to build from. + * @return the built {@link SearchBuilder} object. * @see #importRawSearch(Search) */ - public static SearchBuilder createFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) { - SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + public static SearchBuilder createFromRawSearch(Search rawSearch, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfileRegistry); builder.importRawSearch(rawSearch); builder.build(); return builder; @@ -416,12 +443,18 @@ public class SearchBuilder { * @return The built {@link Search} object. * @see #importRawSearch(Search) */ - public static Search buildFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) { - return createFromRawSearch(rawSearch, rankProfileRegistry).getSearch(); + public static Search buildFromRawSearch(Search rawSearch, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + return createFromRawSearch(rawSearch, rankProfileRegistry, queryProfileRegistry).getSearch(); } public RankProfileRegistry getRankProfileRegistry() { return rankProfileRegistry; } + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java index b448005c6a5..1201de86d8d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -13,16 +14,19 @@ import java.io.IOException; */ public class UnprocessingSearchBuilder extends SearchBuilder { - public UnprocessingSearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) { - super(app, rankProfileRegistry); + public UnprocessingSearchBuilder(ApplicationPackage app, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + super(app, rankProfileRegistry, queryProfileRegistry); } public UnprocessingSearchBuilder() { super(); } - public UnprocessingSearchBuilder(RankProfileRegistry rankProfileRegistry) { - super(rankProfileRegistry); + public UnprocessingSearchBuilder(RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + super(rankProfileRegistry, queryProfileRegistry); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index fa202770e26..985087e905b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -8,6 +8,7 @@ import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.IOUtils; import com.yahoo.protect.Validator; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.derived.validation.Validation; @@ -45,8 +46,8 @@ public class DerivedConfiguration { * modified. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} */ - public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry) { - this(search, null, new BaseDeployLogger(), rankProfileRegistry); + public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { + this(search, null, new BaseDeployLogger(), rankProfileRegistry, queryProfiles); } /** @@ -60,8 +61,12 @@ public class DerivedConfiguration { * @param deployLogger a {@link DeployLogger} for logging when * doing operations on this * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + * @param queryProfiles the query profiles of this application */ - public DerivedConfiguration(Search search, List<Search> abstractSearchList, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public DerivedConfiguration(Search search, List<Search> abstractSearchList, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfiles) { Validator.ensureNotNull("Search definition", search); if ( ! search.isProcessed()) { throw new IllegalArgumentException("Search '" + search.getName() + "' not processed."); @@ -83,7 +88,7 @@ public class DerivedConfiguration { summaries = new Summaries(search, deployLogger); summaryMap = new SummaryMap(search, summaries); juniperrc = new Juniperrc(search); - rankProfileList = new RankProfileList(search, attributeFields, rankProfileRegistry); + rankProfileList = new RankProfileList(search, attributeFields, rankProfileRegistry, queryProfiles); indexingScript = new IndexingScript(search); indexInfo = new IndexInfo(search); indexSchema = new IndexSchema(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 5a8996d4e53..77645331d9e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.searchdefinition.RankProfile; @@ -22,19 +23,27 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ * @param search the search definition this is a rank profile from * @param attributeFields the attribute fields to create a ranking for */ - public RankProfileList(Search search, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry) { + public RankProfileList(Search search, + AttributeFields attributeFields, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfiles) { setName(search.getName()); - deriveRankProfiles(rankProfileRegistry, search, attributeFields); + deriveRankProfiles(rankProfileRegistry, queryProfiles, search, attributeFields); } - private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, Search search, AttributeFields attributeFields) { - RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "default"), attributeFields); + private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfiles, + Search search, + AttributeFields attributeFields) { + RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "default"), + queryProfiles, + attributeFields); rankProfiles.put(defaultProfile.getName(), defaultProfile); for (RankProfile rank : rankProfileRegistry.localRankProfiles(search)) { if ("default".equals(rank.getName())) continue; - RawRankProfile rawRank = new RawRankProfile(rank, attributeFields); + RawRankProfile rawRank = new RawRankProfile(rank, queryProfiles, attributeFields); rankProfiles.put(rawRank.getName(), rawRank); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 3c7b99afefc..ea02f960800 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.derived; import com.google.common.collect.ImmutableList; import com.yahoo.collections.Pair; import com.yahoo.compress.Compressor; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; @@ -46,9 +47,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public RawRankProfile(RankProfile rankProfile, AttributeFields attributeFields) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, AttributeFields attributeFields) { this.name = rankProfile.getName(); - compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, attributeFields).derive())); + compressedProperties = compress(removePartFromKeys(new Deriver(rankProfile, queryProfiles, attributeFields).derive())); } private List<Pair<String, String>> removePartFromKeys(Map<String, String> map) { @@ -153,8 +154,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public Deriver(RankProfile rankProfile, AttributeFields attributeFields) { - this.rankProfile = rankProfile.compile(); + public Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, AttributeFields attributeFields) { + this.rankProfile = rankProfile.compile(queryProfiles); deriveRankingFeatures(this.rankProfile); deriveRankTypeSetting(this.rankProfile, attributeFields); deriveFilterFields(this.rankProfile); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 7fcd2ed357a..5da5b3dabda 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.expressiontransforms; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.transform.TransformContext; @@ -15,20 +16,24 @@ import java.util.Map; public class RankProfileTransformContext extends TransformContext { private final RankProfile rankProfile; + private final QueryProfileRegistry queryProfiles; private final Map<String, RankProfile.Macro> inlineMacros; private final Map<String, String> rankPropertiesOutput; public RankProfileTransformContext(RankProfile rankProfile, - Map<String, Value> constants, - Map<String, RankProfile.Macro> inlineMacros, - Map<String, String> rankPropertiesOutput) { + QueryProfileRegistry queryProfiles, + Map<String, Value> constants, + Map<String, RankProfile.Macro> inlineMacros, + Map<String, String> rankPropertiesOutput) { super(constants); this.rankProfile = rankProfile; + this.queryProfiles = queryProfiles; this.inlineMacros = inlineMacros; this.rankPropertiesOutput = rankPropertiesOutput; } public RankProfile rankProfile() { return rankProfile; } + public QueryProfileRegistry queryProfiles() { return queryProfiles; } public Map<String, RankProfile.Macro> inlineMacros() { return inlineMacros; } public Map<String, String> rankPropertiesOutput() { return rankPropertiesOutput; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index da85e9f65ec..6c4a5dae4cb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -68,14 +69,16 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil if (store.hasStoredModel()) return transformFromStoredModel(store, context.rankProfile()); else // not converted yet - access TensorFlow model files - return transformFromTensorFlowModel(store, context.rankProfile()); + return transformFromTensorFlowModel(store, context.rankProfile(), context.queryProfiles()); } catch (IllegalArgumentException | UncheckedIOException e) { throw new IllegalArgumentException("Could not use tensorflow model from " + feature, e); } } - private ExpressionNode transformFromTensorFlowModel(ModelStore store, RankProfile profile) { + private ExpressionNode transformFromTensorFlowModel(ModelStore store, + RankProfile profile, + QueryProfileRegistry queryProfiles) { TensorFlowModel model = importedModels.computeIfAbsent(store.arguments().modelPath(), k -> tensorFlowImporter.importModel(store.tensorFlowModelDir())); @@ -83,7 +86,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil Signature signature = chooseSignature(model, store.arguments().signature()); String output = chooseOutput(signature, store.arguments().output()); RankingExpression expression = model.expressions().get(output); - verifyRequiredMacros(expression, model.requiredMacros(), profile); + verifyRequiredMacros(expression, model.requiredMacros(), profile, queryProfiles); store.writeConverted(expression); model.constants().forEach((k, v) -> transformConstant(store, profile, k, v)); @@ -172,7 +175,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil * and return tensors of the types specified in requiredMacros. */ private void verifyRequiredMacros(RankingExpression expression, Map<String, TensorType> requiredMacros, - RankProfile profile) { + RankProfile profile, QueryProfileRegistry queryProfiles) { List<String> macroNames = new ArrayList<>(); addMacroNamesIn(expression.getRoot(), macroNames); for (String macroName : macroNames) { @@ -184,7 +187,7 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + " but this macro is not present in " + profile); - TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext()); + TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java index b7cfad6c052..296e3952454 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java @@ -5,9 +5,9 @@ import java.util.TreeMap; /** * @author baldersheim - * @since 2010-02-19 */ public class SearchManager { + /// This is the list of all known search definitions private TreeMap<String, SearchDef> defs = new TreeMap<>(); @@ -24,4 +24,5 @@ public class SearchManager { defs.put(def.getName(), def); return this; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4925b88e608..c3069f699d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -281,11 +281,15 @@ public abstract class IndexedSearchCluster extends SearchCluster } } protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, - List<com.yahoo.searchdefinition.Search> globalSearches) { + List<com.yahoo.searchdefinition.Search> globalSearches) { for (SearchDefinitionSpec spec : localSearches) { com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch(); - if (!(search instanceof UnproperSearch)) { - DocumentDatabase db = new DocumentDatabase(this, search.getName(), new DerivedConfiguration(search, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry())); + if ( ! (search instanceof UnproperSearch)) { + DocumentDatabase db = new DocumentDatabase(this, + search.getName(), + new DerivedConfiguration(search, globalSearches, deployLogger(), + getRoot().getDeployState().rankProfileRegistry(), + getRoot().getDeployState().getQueryProfiles().getRegistry())); // TODO: remove explicit adding of user configs when the complete content model is built using builders. db.mergeUserConfigs(spec.getUserConfigs()); documentDbs.add(db); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 632e33b85b0..85f98c27365 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -92,7 +92,9 @@ public class StreamingSearchCluster extends SearchCluster implements } private void deriveSingleSearchDefinition(com.yahoo.searchdefinition.Search localSearch, List<com.yahoo.searchdefinition.Search> globalSearches) { - this.sdConfig = new DerivedConfiguration(localSearch, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry()); + this.sdConfig = new DerivedConfiguration(localSearch, globalSearches, deployLogger(), + getRoot().getDeployState().rankProfileRegistry(), + getRoot().getDeployState().getQueryProfiles().getRegistry()); } @Override public DerivedConfiguration getSdConfig() { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index bf6376983a4..86b136a1dd2 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -65,24 +65,29 @@ import org.apache.commons.lang.StringUtils; /** * A search definition parser * - * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> et.al. + * @author bratseth */ public class SDParser { + private DocumentTypeManager docMan = null; - private ApplicationPackage app = MockApplicationPackage.createEmpty(); - private DeployLogger deployLogger = new BaseDeployLogger(); - private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private ApplicationPackage app; + private DeployLogger deployLogger; + private RankProfileRegistry rankProfileRegistry; + /** For testing only */ public SDParser(String input, DeployLogger deployLogger) { this(new SimpleCharStream(input), deployLogger); } + /** For testing only */ public SDParser(SimpleCharStream stream, DeployLogger deployLogger) { - this(stream); - this.deployLogger = deployLogger; + this(stream, deployLogger, MockApplicationPackage.createEmpty(), new RankProfileRegistry()); } - public SDParser(SimpleCharStream stream, DeployLogger deployLogger, ApplicationPackage applicationPackage, RankProfileRegistry rankProfileRegistry) { + public SDParser(SimpleCharStream stream, + DeployLogger deployLogger, + ApplicationPackage applicationPackage, + RankProfileRegistry rankProfileRegistry) { this(stream); this.deployLogger = deployLogger; this.app = applicationPackage; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java index 27bab31d709..bff34411d44 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -19,8 +20,10 @@ public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionT public void testIncorrectRef() throws IOException, ParseException { try { RankProfileRegistry registry = new RankProfileRegistry(); - Search search = SearchBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", registry); - new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + Search search = SearchBuilder.buildFromFile("src/test/examples/incorrectrankingexpressionfileref.sd", + registry, + new QueryProfileRegistry()); + new DerivedConfiguration(search, registry, new QueryProfileRegistry()); // cause rank profile parsing fail("parsing should have failed"); } catch (IllegalArgumentException e) { e.printStackTrace(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java index 960a3b7d6db..442c8bd41bd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertTrue; * @author bratseth */ public class RankProfileTestCase extends SearchDefinitionTestCase { + @Test public void testRankProfileInheritance() { Search search = new Search("test", null); @@ -90,7 +91,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { assertEquals(8, rankProfile.getNumThreadsPerSearch()); assertEquals(70, rankProfile.getMinHitsPerThread()); assertEquals(1200, rankProfile.getNumSearchPartitions()); - RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, attributeFields); + RawRankProfile rawRankProfile = new RawRankProfile(rankProfile, new QueryProfileRegistry(), attributeFields); assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").isPresent()); assertEquals("0.78", findProperty(rawRankProfile.configProperties(), "vespa.matching.termwise_limit").get()); assertTrue(findProperty(rawRankProfile.configProperties(), "vespa.matching.numthreadspersearch").isPresent()); @@ -125,7 +126,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { } private static void assertAttributeTypeSettings(RankProfile profile, Search search) { - RawRankProfile rawProfile = new RawRankProfile(profile, new AttributeFields(search)); + RawRankProfile rawProfile = new RawRankProfile(profile, new QueryProfileRegistry(), new AttributeFields(search)); assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.a").get()); assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.attribute.b").get()); assertEquals("tensor(x[])", findProperty(rawProfile.configProperties(), "vespa.type.attribute.c").get()); @@ -167,7 +168,7 @@ public class RankProfileTestCase extends SearchDefinitionTestCase { } private static void assertQueryFeatureTypeSettings(RankProfile profile, Search search) { - RawRankProfile rawProfile = new RawRankProfile(profile, new AttributeFields(search)); + RawRankProfile rawProfile = new RawRankProfile(profile, new QueryProfileRegistry(), new AttributeFields(search)); assertEquals("tensor(x[10])", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor1").get()); assertEquals("tensor(y{})", findProperty(rawProfile.configProperties(), "vespa.type.query.tensor2").get()); assertFalse(findProperty(rawProfile.configProperties(), "vespa.type.query.tensor3").isPresent()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java index c42336b300b..15ddef60807 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -59,7 +60,7 @@ public class RankPropertiesTestCase extends SearchDefinitionTestCase { assertEquals("query(a) = 1500", parent.getRankProperties().get(0).toString()); // Check derived model - RawRankProfile rawParent = new RawRankProfile(parent, attributeFields); + RawRankProfile rawParent = new RawRankProfile(parent, new QueryProfileRegistry(), attributeFields); assertEquals("(query(a),1500)", rawParent.configProperties().get(0).toString()); } @@ -69,7 +70,9 @@ public class RankPropertiesTestCase extends SearchDefinitionTestCase { assertEquals("query(a) = 2000", parent.getRankProperties().get(0).toString()); // Check derived model - RawRankProfile rawChild = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "child"), attributeFields); + RawRankProfile rawChild = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "child"), + new QueryProfileRegistry(), + attributeFields); assertEquals("(query(a),2000)", rawChild.configProperties().get(0).toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java index 1f4f18b5a47..e94880e61c7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.yolean.Exceptions; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; @@ -24,6 +25,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase @Test public void testConstants() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfileRegistry queryProfileRegistry = new QueryProfileRegistry(); SearchBuilder builder = new SearchBuilder(rankProfileRegistry); builder.importString( "search test {\n" + @@ -67,17 +69,19 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(queryProfileRegistry); assertEquals("0.0", parent.getFirstPhaseRanking().getRoot().toString()); - RankProfile child1 = rankProfileRegistry.getRankProfile(s, "child1").compile(); + RankProfile child1 = rankProfileRegistry.getRankProfile(s, "child1").compile(queryProfileRegistry); assertEquals("6.5", child1.getFirstPhaseRanking().getRoot().toString()); assertEquals("11.5", child1.getSecondPhaseRanking().getRoot().toString()); - RankProfile child2 = rankProfileRegistry.getRankProfile(s, "child2").compile(); + RankProfile child2 = rankProfileRegistry.getRankProfile(s, "child2").compile(queryProfileRegistry); assertEquals("16.6", child2.getFirstPhaseRanking().getRoot().toString()); assertEquals("foo: 14.0", child2.getMacros().get("foo").getRankingExpression().toString()); - List<Pair<String, String>> rankProperties = new RawRankProfile(child2, new AttributeFields(s)).configProperties(); + List<Pair<String, String>> rankProperties = new RawRankProfile(child2, + queryProfileRegistry, + new AttributeFields(s)).configProperties(); assertEquals("(rankingExpression(foo).rankingScript,14.0)", rankProperties.get(0).toString()); assertEquals("(rankingExpression(firstphase).rankingScript,16.6)", rankProperties.get(2).toString()); } @@ -107,7 +111,7 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); try { - rankProfileRegistry.getRankProfile(s, "test").compile(); + rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); fail("Should have caused an exception"); } catch (IllegalArgumentException e) { @@ -169,7 +173,8 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); profile.parseExpressions(); // TODO: Do differently assertEquals("safeLog(popShareSlowDecaySignal,myValue)", profile.getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); - assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", profile.compile().getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); + assertEquals("safeLog(popShareSlowDecaySignal,-9.21034037)", + profile.compile(new QueryProfileRegistry()).getMacros().get("POP_SLOW_SCORE").getRankingExpression().getRoot().toString()); } @Test @@ -191,7 +196,8 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); - assertEquals("k1 + (k2 + k3) / 100000000.0", profile.compile().getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + assertEquals("k1 + (k2 + k3) / 100000000.0", + profile.compile(new QueryProfileRegistry()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); } @Test @@ -213,7 +219,8 @@ public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); RankProfile profile = rankProfileRegistry.getRankProfile(s, "test"); - assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", profile.compile().getMacros().get("rank_default").getRankingExpression().getRoot().toString()); + assertEquals("0.5 + 50 * (attribute(rating_yelp) - 3)", + profile.compile(new QueryProfileRegistry()).getMacros().get("rank_default").getRankingExpression().getRoot().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java index a36ac63b2b5..97e1ab9aeb9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -59,10 +60,12 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); - RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); - assertEquals("7.0 * (3 + attribute(a) + attribute(b) * (attribute(a) * 3 + if (7.0 < attribute(a), 1, 2) == 0))", parent.getFirstPhaseRanking().getRoot().toString()); - RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(); - assertEquals("7.0 * (9 + attribute(a))", child.getFirstPhaseRanking().getRoot().toString()); + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(new QueryProfileRegistry()); + assertEquals("7.0 * (3 + attribute(a) + attribute(b) * (attribute(a) * 3 + if (7.0 < attribute(a), 1, 2) == 0))", + parent.getFirstPhaseRanking().getRoot().toString()); + RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(new QueryProfileRegistry()); + assertEquals("7.0 * (9 + attribute(a))", + child.getFirstPhaseRanking().getRoot().toString()); } @Test public void testConstants() throws ParseException { @@ -116,26 +119,39 @@ public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase builder.build(); Search s = builder.getSearch(); - RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(); + RankProfile parent = rankProfileRegistry.getRankProfile(s, "parent").compile(new QueryProfileRegistry()); assertEquals("17.0", parent.getFirstPhaseRanking().getRoot().toString()); assertEquals("0.0", parent.getSecondPhaseRanking().getRoot().toString()); - List<Pair<String, String>> parentRankProperties = new RawRankProfile(parent, new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(foo).rankingScript,10.0)", parentRankProperties.get(0).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript,17.0)", parentRankProperties.get(2).toString()); - assertEquals("(rankingExpression(secondphase).rankingScript,0.0)", parentRankProperties.get(4).toString()); + List<Pair<String, String>> parentRankProperties = new RawRankProfile(parent, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(foo).rankingScript,10.0)", + parentRankProperties.get(0).toString()); + assertEquals("(rankingExpression(firstphase).rankingScript,17.0)", + parentRankProperties.get(2).toString()); + assertEquals("(rankingExpression(secondphase).rankingScript,0.0)", + parentRankProperties.get(4).toString()); - RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(); + RankProfile child = rankProfileRegistry.getRankProfile(s, "child").compile(new QueryProfileRegistry()); assertEquals("31.0 + bar + arg(4.0)", child.getFirstPhaseRanking().getRoot().toString()); assertEquals("24.0", child.getSecondPhaseRanking().getRoot().toString()); - List<Pair<String, String>> childRankProperties = new RawRankProfile(child, new AttributeFields(s)).configProperties(); - for (Object o : childRankProperties) System.out.println(o); - assertEquals("(rankingExpression(foo).rankingScript,12.0)", childRankProperties.get(0).toString()); - assertEquals("(rankingExpression(bar).rankingScript,14.0)", childRankProperties.get(1).toString()); - assertEquals("(rankingExpression(boz).rankingScript,3.0)", childRankProperties.get(2).toString()); - assertEquals("(rankingExpression(baz).rankingScript,9.0 + rankingExpression(boz))", childRankProperties.get(3).toString()); - assertEquals("(rankingExpression(arg).rankingScript,a1 * 2)", childRankProperties.get(4).toString()); - assertEquals("(rankingExpression(firstphase).rankingScript,31.0 + rankingExpression(bar) + rankingExpression(arg@))", censorBindingHash(childRankProperties.get(7).toString())); - assertEquals("(rankingExpression(secondphase).rankingScript,24.0)", childRankProperties.get(9).toString()); + List<Pair<String, String>> childRankProperties = new RawRankProfile(child, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(foo).rankingScript,12.0)", + childRankProperties.get(0).toString()); + assertEquals("(rankingExpression(bar).rankingScript,14.0)", + childRankProperties.get(1).toString()); + assertEquals("(rankingExpression(boz).rankingScript,3.0)", + childRankProperties.get(2).toString()); + assertEquals("(rankingExpression(baz).rankingScript,9.0 + rankingExpression(boz))", + childRankProperties.get(3).toString()); + assertEquals("(rankingExpression(arg).rankingScript,a1 * 2)", + childRankProperties.get(4).toString()); + assertEquals("(rankingExpression(firstphase).rankingScript,31.0 + rankingExpression(bar) + rankingExpression(arg@))", + censorBindingHash(childRankProperties.get(7).toString())); + assertEquals("(rankingExpression(secondphase).rankingScript,24.0)", + childRankProperties.get(9).toString()); } /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 92896429f26..5100ac15c40 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition; import com.yahoo.collections.Pair; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; import com.yahoo.searchdefinition.parser.ParseException; @@ -40,13 +41,16 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(sin@).rankingScript,2 * 2)", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(2).toString())); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(sin@).rankingScript,2 * 2)", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(2).toString())); } @@ -80,19 +84,28 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(tan).rankingScript,x * x)", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(tan@).rankingScript,x * x)", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(4).toString())); - assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", censorBindingHash(testRankProperties.get(5).toString())); - assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(6).toString())); - assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(7).toString())); - assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(8).toString())); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(tan).rankingScript,x * x)", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(tan@).rankingScript,x * x)", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(3).toString())); + assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(4).toString())); + assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + censorBindingHash(testRankProperties.get(5).toString())); + assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(6).toString())); + assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(7).toString())); + assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(8).toString())); } @@ -120,15 +133,20 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(sin@).rankingScript,4.0 * 4.0)", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(sin@).rankingScript,cos(5.0) * cos(5.0))", censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(vespa.rank.firstphase,rankingExpression(firstphase))", censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(firstphase).rankingScript,cos(rankingExpression(sin@)) + rankingExpression(sin@))", censorBindingHash(testRankProperties.get(4).toString())); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(sin@).rankingScript,4.0 * 4.0)", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin@).rankingScript,cos(5.0) * cos(5.0))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(vespa.rank.firstphase,rankingExpression(firstphase))", + censorBindingHash(testRankProperties.get(3).toString())); + assertEquals("(rankingExpression(firstphase).rankingScript,cos(rankingExpression(sin@)) + rankingExpression(sin@))", + censorBindingHash(testRankProperties.get(4).toString())); } @@ -162,16 +180,22 @@ public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase "}\n"); builder.build(); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); - assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(0).toString()); - assertEquals("(rankingExpression(relu@).rankingScript,max(1.0,reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input)))", censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", testRankProperties.get(3).toString()); - assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", testRankProperties.get(4).toString()); - assertEquals("(rankingExpression(secondphase).rankingScript,reduce(rankingExpression(final_layer), sum))", testRankProperties.get(5).toString()); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", + testRankProperties.get(0).toString()); + assertEquals("(rankingExpression(relu@).rankingScript,max(1.0,reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input)))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", + censorBindingHash(testRankProperties.get(2).toString())); + assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", + testRankProperties.get(3).toString()); + assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", + testRankProperties.get(4).toString()); + assertEquals("(rankingExpression(secondphase).rankingScript,reduce(rankingExpression(final_layer), sum))", + testRankProperties.get(5).toString()); } private String censorBindingHash(String s) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java index b652bcd6ce6..a07fea69592 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Ignore; @@ -23,7 +24,7 @@ public class RankingExpressionValidationTestCase extends SearchDefinitionTestCas try { RankProfileRegistry registry = new RankProfileRegistry(); Search search = importWithExpression(expression, registry); - new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + new DerivedConfiguration(search, registry, new QueryProfileRegistry()); // cause rank profile parsing fail("No exception on incorrect ranking expression " + expression); } catch (IllegalArgumentException e) { // Success diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index f9b62799171..d1a5bbad217 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; import com.yahoo.document.Document; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.document.*; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.processing.MakeAliases; @@ -26,7 +27,7 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { @Test public void testSimpleImporting() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - SearchBuilder sb = new UnprocessingSearchBuilder(rankProfileRegistry); + SearchBuilder sb = new UnprocessingSearchBuilder(rankProfileRegistry, new QueryProfileRegistry()); sb.importFile("src/test/examples/simple.sd"); sb.build(); Search search = sb.getSearch(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index 84edb50aed3..3d2bce62713 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -78,12 +78,16 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase } protected DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException { - DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), builder.getRankProfileRegistry()); + DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), + builder.getRankProfileRegistry(), + builder.getQueryProfileRegistry()); return export(dirName, builder, config); } protected DerivedConfiguration derive(String dirName, SearchBuilder builder, Search search) throws IOException { - DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); + DerivedConfiguration config = new DerivedConfiguration(search, + builder.getRankProfileRegistry(), + builder.getQueryProfileRegistry()); return export(dirName, builder, config); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java index 4e9a802841a..21467776ad9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; @@ -21,7 +22,7 @@ import java.io.IOException; public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { @Test - public void testDeriving() throws IOException, ParseException { + public void testDeriving() { Search search = new Search("test", null); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(search); SDDocumentType doc = new SDDocumentType("test"); @@ -32,7 +33,8 @@ public class EmptyRankProfileTestCase extends SearchDefinitionTestCase { doc.addField(field); doc.addField(new SDField("c", DataType.STRING)); - search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); - new DerivedConfiguration(search, rankProfileRegistry); + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); + new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index 2cb7c0e86b9..77ad5051c19 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -40,7 +41,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()); - DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry); + DerivedConfiguration derived=new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); // Check attribute fields derived.getAttributeFields(); // TODO: assert content @@ -70,8 +71,8 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { rankProfileRegistry.addRankProfile(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); - search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); // Check il script addition assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }", @@ -97,8 +98,8 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { field2.parseIndexingScript("{ summary | index }"); field2.setLiteralBoost(20); - search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry); - new DerivedConfiguration(search, rankProfileRegistry); + search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); + new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }", "clear_state | guard { input body | tokenize normalize stem:\"SHORTEST\" | summary body | index body; }", "clear_state | guard { input title | tokenize | index title_literal; }", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index f7794d3439e..f4edc1dd0ae 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; @@ -31,7 +32,9 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { toDir.mkdirs(); deleteContent(toDir); - DerivedConfiguration config = new DerivedConfiguration(search, builder.getRankProfileRegistry()); + DerivedConfiguration config = new DerivedConfiguration(search, + builder.getRankProfileRegistry(), + new QueryProfileRegistry()); config.export(toDirName); checkDir(toDirName, expectedResultsDirName); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java index d09f56c31b3..3d2e1d11e28 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchDefinitionTestCase; @@ -20,9 +21,7 @@ import static org.junit.Assert.assertFalse; */ public class TypeConversionTestCase extends SearchDefinitionTestCase { - /** - * Tests that exact-string stuff is not spilled over to the default index - */ + /** Tests that exact-string stuff is not spilled over to the default index */ @Test public void testExactStringToStringTypeConversion() { Search search = new Search("test", null); @@ -34,7 +33,7 @@ public class TypeConversionTestCase extends SearchDefinitionTestCase { document.addField(a); Processing.process(search, new BaseDeployLogger(), rankProfileRegistry, new QueryProfiles()); - DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry); + DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry()); IndexInfo indexInfo = derived.getIndexInfo(); assertFalse(indexInfo.hasCommand("default", "compact-to-term")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java index 3ffa89b612b..d743f60201e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.processing; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.SearchDefinitionTestCase; @@ -98,7 +99,7 @@ public class ImplicitSearchFieldsTestCase extends SearchDefinitionTestCase { sb.importFile("src/test/examples/nextgen/simple.sd"); sb.build(); assertNotNull(sb.getSearch()); - new DerivedConfiguration(sb.getSearch(), sb.getRankProfileRegistry()); + new DerivedConfiguration(sb.getSearch(), sb.getRankProfileRegistry(), new QueryProfileRegistry()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 86094458b34..800697b3430 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -22,20 +23,22 @@ import static org.junit.Assert.assertEquals; class RankProfileSearchFixture { private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private final QueryProfileRegistry queryProfileRegistry; private Search search; RankProfileSearchFixture(String rankProfiles) throws ParseException { - this(MockApplicationPackage.createEmpty(), rankProfiles); + this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); } - RankProfileSearchFixture(ApplicationPackage applicationpackage, String rankProfiles) throws ParseException { - this(applicationpackage, rankProfiles, null, null); + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, + String rankProfiles) throws ParseException { + this(applicationpackage, queryProfileRegistry, rankProfiles, null, null); } - RankProfileSearchFixture(ApplicationPackage applicationpackage, + RankProfileSearchFixture(ApplicationPackage applicationpackage, QueryProfileRegistry queryProfileRegistry, String rankProfiles, String constant, String field) throws ParseException { - SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry); + SearchBuilder builder = new SearchBuilder(applicationpackage, rankProfileRegistry, new QueryProfileRegistry()); String sdContent = "search test {\n" + " " + (constant != null ? constant : "") + "\n" + " document test {\n" + @@ -47,6 +50,7 @@ class RankProfileSearchFixture { builder.importString(sdContent); builder.build(); search = builder.getSearch(); + this.queryProfileRegistry = queryProfileRegistry; } public void assertFirstPhaseExpression(String expExpression, String rankProfile) { @@ -68,7 +72,7 @@ class RankProfileSearchFixture { } public RankProfile rankProfile(String rankProfile) { - return rankProfileRegistry.getRankProfile(search, rankProfile).compile(); + return rankProfileRegistry.getRankProfile(search, rankProfile).compile(queryProfileRegistry); } public Search search() { return search; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java index bdfad96ef87..df2bcca63dd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile.RankProperty; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; @@ -16,10 +17,14 @@ import java.util.List; import static org.junit.Assert.fail; public class RankPropertyVariablesTestCase extends SearchDefinitionTestCase { + @Test public void testRankPropVariables() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Search search = SearchBuilder.buildFromFile("src/test/examples/rankpropvars.sd", new BaseDeployLogger(), rankProfileRegistry); + Search search = SearchBuilder.buildFromFile("src/test/examples/rankpropvars.sd", + new BaseDeployLogger(), + rankProfileRegistry, + new QueryProfileRegistry()); assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvar1", "foo"); assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvar_2", "bar"); assertRankPropEquals(rankProfileRegistry.getRankProfile(search, "other").getRankProperties(), "$testvarOne23", "baz"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index e6db6abdc95..73c622cba7f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -8,6 +8,7 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.tensor.Tensor; @@ -148,6 +149,7 @@ public class RankingExpressionWithTensorFlowTestCase { try { RankProfileSearchFixture search = new RankProfileSearchFixture( new StoringApplicationPackage(applicationDir), + new QueryProfileRegistry(), " rank-profile my_profile {\n" + " first-phase {\n" + " expression: tensorflow('mnist_softmax/saved')" + @@ -290,6 +292,7 @@ public class RankingExpressionWithTensorFlowTestCase { try { return new RankProfileSearchFixture( application, + application.getQueryProfiles(), " rank-profile my_profile {\n" + " macro Placeholder() {\n" + " expression: " + placeholderExpression + @@ -319,7 +322,7 @@ public class RankingExpressionWithTensorFlowTestCase { StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { super(null, null, Collections.emptyList(), null, - null, null, false); + null, null, false, queryProfile, queryProfileType); this.root = new File(applicationPackageWritableRoot.toString()); this.queryProfile = queryProfile; this.queryProfileType = queryProfileType; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index cdb77f5d0e1..18c3e43ae7e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.collections.Pair; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.*; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.derived.AttributeFields; @@ -16,13 +17,14 @@ import java.util.Map; import static org.junit.Assert.assertEquals; -// TODO: WHO? public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test public void testMacros() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", rankProfileRegistry).getSearch(); + Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", + rankProfileRegistry, + new QueryProfileRegistry()).getSearch(); final RankProfile macrosRankProfile = rankProfileRegistry.getRankProfile(search, "macros"); macrosRankProfile.parseExpressions(); final Map<String, RankProfile.Macro> macros = macrosRankProfile.getMacros(); @@ -35,7 +37,9 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { assertEquals("78 + closeness(distance)", macros.get("artistmatch").getTextualExpression().trim()); assertEquals(0, macros.get("artistmatch").getFormalParams().size()); - List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, new AttributeFields(search)).configProperties(); + List<Pair<String, String>> rankProperties = new RawRankProfile(macrosRankProfile, + new QueryProfileRegistry(), + new AttributeFields(search)).configProperties(); assertEquals(6, rankProperties.size()); assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(0).getFirst()); @@ -57,8 +61,10 @@ public class RankingExpressionsTestCase extends SearchDefinitionTestCase { @Test(expected = IllegalArgumentException.class) public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", registry).getSearch(); - new DerivedConfiguration(search, registry); // rank profile parsing happens during deriving + Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", + registry, + new QueryProfileRegistry()).getSearch(); + new DerivedConfiguration(search, registry, new QueryProfileRegistry()); // rank profile parsing happens during deriving } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index 28e1f310078..c18cfcfe1aa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -100,7 +100,6 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { String rankProperty = rankPropertyExpression.getFirst(); if (rankProperty.equals("rankingExpression(firstphase).rankingScript")) { String rankExpression = censorBindingHash(rankPropertyExpression.getSecond().replace(" ","")); - System.out.println("expression is --> " + rankExpression); return rankExpression.equals(transformedExpression); } } @@ -170,10 +169,10 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { "}\n"); builder.build(new BaseDeployLogger(), setupQueryProfileTypes()); Search s = builder.getSearch(); - RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(); - List<Pair<String, String>> testRankProperties = new RawRankProfile(test, new AttributeFields(s)).configProperties(); - for (Object o : testRankProperties) - System.out.println(o); + RankProfile test = rankProfileRegistry.getRankProfile(s, "test").compile(new QueryProfileRegistry()); + List<Pair<String, String>> testRankProperties = new RawRankProfile(test, + new QueryProfileRegistry(), + new AttributeFields(s)).configProperties(); return testRankProperties; } |