diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-08-19 12:23:32 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2019-08-19 12:23:32 +0200 |
commit | 13d0bb503aacfd4ce020e600e87dd22e73f787cd (patch) | |
tree | 00da5fb37d14bf38d0c0aadd3f3c6109798b29e8 /config-model/src/main/java/com/yahoo | |
parent | 0e1adb57b5f6621c6adc37e7509d7acff2bb96e6 (diff) |
Add feature flag for controlling termwise limit default.
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
8 files changed, 50 insertions, 26 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index d974db73547..0dc08c19b92 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties { private boolean useFdispatchByDefault = true; private boolean dispatchWithProtobuf = true; private boolean useAdaptiveDispatch = false; + private double defaultTermwiseLimit = 1.0; private Optional<TlsSecrets> tlsSecrets = Optional.empty(); @@ -62,7 +63,12 @@ public class TestProperties implements ModelContext.Properties { @Override public boolean useFdispatchByDefault() { return useFdispatchByDefault; } @Override public boolean dispatchWithProtobuf() { return dispatchWithProtobuf; } @Override public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; } + @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } + public TestProperties setDefaultTermwiseLimit(double limit) { + defaultTermwiseLimit = limit; + return this; + } public TestProperties setApplicationId(ApplicationId applicationId) { this.applicationId = applicationId; return this; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 7a34ff258b7..d3e32868f62 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -501,10 +501,10 @@ public class RankProfile implements Serializable, Cloneable { : numSearchPartitions; } - public double getTermwiseLimit() { + public Optional<Double> getTermwiseLimit() { return ((termwiseLimit == null) && (getInherited() != null)) ? getInherited().getTermwiseLimit() - : (termwiseLimit != null) ? termwiseLimit : 1.0; + : (termwiseLimit != null) ? Optional.of(termwiseLimit) : Optional.empty(); } public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index f25dd0be7b2..1e45ea22567 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -3,8 +3,10 @@ package com.yahoo.searchdefinition.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.IOUtils; @@ -50,7 +52,7 @@ public class DerivedConfiguration { RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { - this(search, new BaseDeployLogger(), rankProfileRegistry, queryProfiles, importedModels); + this(search, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, importedModels); } /** @@ -59,13 +61,14 @@ public class DerivedConfiguration { * @param search The search to derive a configuration from. Derived objects will be snapshots, but this * argument is live. Which means that this object will be inconsistent when the given * search definition is later modified. - * @param deployLogger a {@link DeployLogger} for logging when - * doing operations on this + * @param deployLogger a {@link DeployLogger} for logging when doing operations on this + * @param deployProperties Properties set on deploy. * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} * @param queryProfiles the query profiles of this application */ public DerivedConfiguration(Search search, DeployLogger deployLogger, + ModelContext.Properties deployProperties, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { @@ -80,7 +83,7 @@ public class DerivedConfiguration { summaries = new Summaries(search, deployLogger); summaryMap = new SummaryMap(search); juniperrc = new Juniperrc(search); - rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels); + rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels, deployProperties); indexingScript = new IndexingScript(search); indexInfo = new IndexInfo(search); indexSchema = new IndexSchema(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index d8f2141d837..f6d4889b55e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstant; @@ -45,29 +46,29 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels) { + ImportedMlModels importedModels, + ModelContext.Properties deployProperties) { setName(search == null ? "default" : search.getName()); this.rankingConstants = rankingConstants; - deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields); + deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, Search search, - AttributeFields attributeFields) { + AttributeFields attributeFields, + ModelContext.Properties deployProperties) { if (search != null) { // profiles belonging to a search have a default profile RawRankProfile defaultProfile = new RawRankProfile(rankProfileRegistry.get(search, "default"), - queryProfiles, - importedModels, - attributeFields); + queryProfiles, importedModels, attributeFields, deployProperties); rankProfiles.put(defaultProfile.getName(), defaultProfile); } for (RankProfile rank : rankProfileRegistry.rankProfilesOf(search)) { if (search != null && "default".equals(rank.getName())) continue; - RawRankProfile rawRank = new RawRankProfile(rank, queryProfiles, importedModels, attributeFields); + RawRankProfile rawRank = new RawRankProfile(rank, queryProfiles, importedModels, attributeFields, deployProperties); rankProfiles.put(rawRank.getName(), rawRank); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index bad46955dd9..e2955d4b3ff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -5,6 +5,8 @@ import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.google.common.collect.ImmutableList; import com.yahoo.collections.Pair; import com.yahoo.compress.Compressor; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.RankProfile; @@ -16,7 +18,7 @@ import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.RankProfilesConfig; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; @@ -50,20 +52,28 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { + this.name = rankProfile.getName(); + compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, deployProperties).derive()); + } + + /** + * Only for testing + */ public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) { this.name = rankProfile.getName(); - compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields).derive()); + compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, new TestProperties()).derive()); } private Compressor.Compression compress(List<Pair<String, String>> properties) { StringBuilder b = new StringBuilder(); for (Pair<String, String> property : properties) b.append(property.getFirst()).append(keyEndMarker).append(property.getSecond()).append(valueEndMarker); - return compressor.compress(b.toString().getBytes(Charset.forName("utf8"))); + return compressor.compress(b.toString().getBytes(StandardCharsets.UTF_8)); } private List<Pair<String, String>> decompress(Compressor.Compression compression) { - String propertiesString = new String(compressor.decompress(compression), Charset.forName("utf8")); + String propertiesString = new String(compressor.decompress(compression), StandardCharsets.UTF_8); if (propertiesString.isEmpty()) return ImmutableList.of(); ImmutableList.Builder<Pair<String, String>> properties = new ImmutableList.Builder<>(); @@ -148,11 +158,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) { + Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + AttributeFields attributeFields, ModelContext.Properties deployProperties) + { RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); - deriveRankingFeatures(compiled); + deriveRankingFeatures(compiled, deployProperties); deriveRankTypeSetting(compiled, attributeFields); deriveFilterFields(compiled); deriveWeightProperties(compiled); @@ -162,7 +174,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { filterFields.addAll(rp.allFilterFields()); } - public void deriveRankingFeatures(RankProfile rankProfile) { + private void deriveRankingFeatures(RankProfile rankProfile, ModelContext.Properties deployProperties) { firstPhaseRanking = rankProfile.getFirstPhaseRanking(); secondPhaseRanking = rankProfile.getSecondPhaseRanking(); summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures()); @@ -172,7 +184,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { numThreadsPerSearch = rankProfile.getNumThreadsPerSearch(); minHitsPerThread = rankProfile.getMinHitsPerThread(); numSearchPartitions = rankProfile.getNumSearchPartitions(); - termwiseLimit = rankProfile.getTermwiseLimit(); + termwiseLimit = rankProfile.getTermwiseLimit().orElse(deployProperties.defaultTermwiseLimit()); keepRankCount = rankProfile.getKeepRankCount(); rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); @@ -263,7 +275,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } } - public void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) { + private void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) { if (isDefaultSetting) return; NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType); @@ -290,7 +302,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return setting != null && setting.getValue().equals(RankType.DEFAULT); } - public FieldRankSettings deriveFieldRankSettings(String fieldName) { + private FieldRankSettings deriveFieldRankSettings(String fieldName) { FieldRankSettings settings = fieldRankSettings.get(fieldName); if (settings == null) { settings = new FieldRankSettings(fieldName); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index f36cc3b195c..31746c39f80 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -164,11 +164,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(), deployState.rankProfileRegistry(), deployState.getQueryProfiles()); this.rankProfileList = new RankProfileList(null, // null search -> global - rankingConstants, - AttributeFields.empty, + rankingConstants, AttributeFields.empty, deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels()); + deployState.getImportedModels(), + deployState.getProperties()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 1d804ba3b27..c12ef3cf30f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -304,6 +304,7 @@ public class IndexedSearchCluster extends SearchCluster if ( ! (search instanceof DocumentOnlySearch)) { DocumentDatabase db = new DocumentDatabase(this, search.getName(), new DerivedConfiguration(search, deployState.getDeployLogger(), + deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), deployState.getImportedModels())); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 832f5bf7d6b..d668adea116 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -103,6 +103,7 @@ public class StreamingSearchCluster extends SearchCluster implements throw new IllegalStateException("Mismatch between document type name (" + docTypeName + ") and name of search definition (" + localSearch.getName() + ")"); } this.sdConfig = new DerivedConfiguration(localSearch, deployState.getDeployLogger(), + deployState.getProperties(), deployState.rankProfileRegistry(), deployState.getQueryProfiles().getRegistry(), deployState.getImportedModels()); |