From 7ea7f1fd2917698cc30c9cea3a81ebf2f99f3d70 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 11 May 2022 16:05:57 +0200 Subject: Pass DeployState --- .../com/yahoo/config/model/deploy/DeployState.java | 22 ++++++++++- .../derived/DerivedConfiguration.java | 45 +++++++--------------- .../searchdefinition/derived/RankProfileList.java | 37 ++++++++++-------- .../java/com/yahoo/vespa/model/VespaModel.java | 6 +-- .../vespa/model/search/IndexedSearchCluster.java | 8 +--- .../vespa/model/search/StreamingSearchCluster.java | 8 +--- 6 files changed, 57 insertions(+), 69 deletions(-) (limited to 'config-model/src/main/java/com') diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 9eef1dc2447..74d92bcfd02 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -28,6 +28,7 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; +import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.Application; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; @@ -334,6 +335,8 @@ public class DeployState implements ConfigDefinitionStore { private boolean accessLoggingEnabledByDefault = true; private Optional wantedDockerImageRepo = Optional.empty(); private Reindexing reindexing = null; + private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private QueryProfiles queryProfiles = null; public Builder() {} @@ -431,6 +434,21 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder rankProfileRegistry(RankProfileRegistry rankProfileRegistry) { + this.rankProfileRegistry = rankProfileRegistry; + return this; + } + + public Builder queryProfiles(QueryProfiles queryProfiles) { + this.queryProfiles = queryProfiles; + return this; + } + + public Builder queryProfiles(QueryProfileRegistry queryProfileRegistry) { + this.queryProfiles = new QueryProfiles(queryProfileRegistry, logger); + return this; + } + public Builder reindexing(Reindexing reindexing) { this.reindexing = Objects.requireNonNull(reindexing); return this; } public DeployState build() { @@ -438,8 +456,8 @@ public class DeployState implements ConfigDefinitionStore { } public DeployState build(ValidationParameters validationParameters) { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger); + if (queryProfiles == null) + queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); Application application = new ApplicationBuilder(applicationPackage, fileRegistry, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 0c7c507f32d..8dec0e8339a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -1,13 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.IOUtils; @@ -21,7 +16,6 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import java.io.IOException; import java.io.Writer; -import java.util.concurrent.ExecutorService; /** * A set of all derived configuration of a schema. Use this as a facade to individual configurations when @@ -62,47 +56,36 @@ public class DerivedConfiguration implements AttributesConfig.Producer { } DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(schema, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, new ImportedMlModels(), new InThreadExecutorService()); + this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build()); } /** - * Creates a complete derived configuration snapshot from a search definition. + * Creates a complete derived configuration snapshot from a schema. * - * @param schema the search to derive a configuration from. Derived objects will be snapshots, but this - * argument is live. Which means that this object will be inconsistent when the given - * search definition is later modified. - * @param deployLogger a {@link DeployLogger} for logging when doing operations on this - * @param deployProperties properties set on deploy - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} - * @param queryProfiles the query profiles of this application + * @param schema the schema to derive a configuration from. Derived objects will be snapshots, but this + * argument is live. Which means that this object will be inconsistent if the given + * schema is later modified. */ - public DerivedConfiguration(Schema schema, - DeployLogger deployLogger, - ModelContext.Properties deployProperties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - ExecutorService executor) { - Validator.ensureNotNull("Search definition", schema); + public DerivedConfiguration(Schema schema, DeployState deployState) { + Validator.ensureNotNull("Schema", schema); this.schema = schema; - this.queryProfiles = queryProfiles; - this.maxUncommittedMemory = deployProperties.featureFlags().maxUnCommittedMemory(); - this.enableBitVectors = deployProperties.featureFlags().enableBitVectors(); + this.queryProfiles = deployState.getQueryProfiles().getRegistry(); + this.maxUncommittedMemory = deployState.getProperties().featureFlags().maxUnCommittedMemory(); + this.enableBitVectors = deployState.getProperties().featureFlags().enableBitVectors(); if ( ! schema.isDocumentsOnly()) { streamingFields = new VsmFields(schema); streamingSummary = new VsmSummary(schema); } if ( ! schema.isDocumentsOnly()) { attributeFields = new AttributeFields(schema); - summaries = new Summaries(schema, deployLogger, deployProperties.featureFlags()); + summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); summaryMap = new SummaryMap(schema); juniperrc = new Juniperrc(schema); rankProfileList = new RankProfileList(schema, schema.rankingConstants(), schema.rankExpressionFiles(), - schema.onnxModels(), attributeFields, rankProfileRegistry, - queryProfiles, importedModels, deployProperties, executor); + schema.onnxModels(), attributeFields, deployState); indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema); - schemaInfo = new SchemaInfo(schema, rankProfileRegistry, summaries, summaryMap); + schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries, summaryMap); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index c36bd96d309..a55c97f4ebb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -3,6 +3,7 @@ package com.yahoo.searchdefinition.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; @@ -60,16 +61,12 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ LargeRankExpressions largeRankExpressions, OnnxModels onnxModels, AttributeFields attributeFields, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - ModelContext.Properties deployProperties, - ExecutorService executor) { + DeployState deployState) { setName(schema == null ? "default" : schema.getName()); this.rankingConstants = rankingConstants; this.largeRankExpressions = largeRankExpressions; this.onnxModels = onnxModels; // as ONNX models come from parsing rank expressions - deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); + deriveRankProfiles(schema, attributeFields, deployState); } private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) { @@ -78,28 +75,34 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null)); } - private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - Schema schema, + private void deriveRankProfiles(Schema schema, AttributeFields attributeFields, - ModelContext.Properties deployProperties, - ExecutorService executor) { + DeployState deployState) { if (schema != null) { // profiles belonging to a search have a default profile - RawRankProfile rawRank = new RawRankProfile(rankProfileRegistry.get(schema, "default"), - largeRankExpressions, queryProfiles, importedModels, attributeFields, deployProperties); + RawRankProfile rawRank = new RawRankProfile(deployState.rankProfileRegistry().get(schema, "default"), + largeRankExpressions, + deployState.getQueryProfiles().getRegistry(), + deployState.getImportedModels(), + attributeFields, + deployState.getProperties()); rankProfiles.put(rawRank.getName(), rawRank); } Map remaining = new LinkedHashMap<>(); - rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); + deployState.rankProfileRegistry().rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); remaining.remove("default"); while (!remaining.isEmpty()) { List ready = new ArrayList<>(); remaining.forEach((name, rank) -> { - if (areDependenciesReady(rank, rankProfileRegistry)) ready.add(rank); + if (areDependenciesReady(rank, deployState.rankProfileRegistry())) ready.add(rank); }); - processRankProfiles(ready, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); + processRankProfiles(ready, + deployState.getQueryProfiles().getRegistry(), + deployState.getImportedModels(), + schema, + attributeFields, + deployState.getProperties(), + deployState.getExecutor()); ready.forEach(rank -> remaining.remove(rank.name())); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index c671ad5bd17..6c04ebbc31e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -186,11 +186,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri new LargeRankExpressions(deployState.getFileRegistry()), new OnnxModels(deployState.getFileRegistry(), Optional.empty()), AttributeFields.empty, - deployState.rankProfileRegistry(), - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - deployState.getProperties(), - deployState.getExecutor()); + deployState); HostSystem hostSystem = root.hostSystem(); if (complete) { // create a completed, frozen model diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index d5bf12df559..44e574199ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -189,13 +189,7 @@ public class IndexedSearchCluster extends SearchCluster for (SchemaInfo spec : schemas().values()) { if (spec.fullSchema() instanceof DocumentOnlySchema) continue; DocumentDatabase db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(spec.fullSchema(), - deployState.getDeployLogger(), - deployState.getProperties(), - deployState.rankProfileRegistry(), - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - deployState.getExecutor())); + new DerivedConfiguration(spec.fullSchema(), deployState)); documentDbs.add(db); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index df266ab77dd..09ed8bad403 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -77,13 +77,7 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - this.schemaConfig = new DerivedConfiguration(schema, - deployState.getDeployLogger(), - deployState.getProperties(), - deployState.rankProfileRegistry(), - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - deployState.getExecutor()); + this.schemaConfig = new DerivedConfiguration(schema, deployState); } @Override -- cgit v1.2.3