diff options
Diffstat (limited to 'config-model/src/main/java')
9 files changed, 66 insertions, 27 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 574c25a2f84..d784722f77a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -21,8 +21,9 @@ import com.yahoo.config.provision.Zone; import com.yahoo.io.reader.NamedReader; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.SearchBuilder; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModels; import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ModelImporter; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -39,6 +40,7 @@ import java.io.IOException; import java.io.Reader; import java.time.Instant; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -80,11 +82,23 @@ public class DeployState implements ConfigDefinitionStore { return new Builder().applicationPackage(applicationPackage).build(); } - private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, - FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, - Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, - java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, - SemanticRules semanticRules, Instant now, Version wantedNodeVespaVersion) { + private DeployState(ApplicationPackage applicationPackage, + SearchDocumentModel searchDocumentModel, + RankProfileRegistry rankProfileRegistry, + FileRegistry fileRegistry, + DeployLogger deployLogger, + Optional<HostProvisioner> hostProvisioner, + DeployProperties properties, + Optional<ApplicationPackage> permanentApplicationPackage, + Optional<ConfigDefinitionRepo> configDefinitionRepo, + java.util.Optional<Model> previousModel, + Set<Rotation> rotations, + Collection<ModelImporter> modelImporters, + Zone zone, + QueryProfiles queryProfiles, + SemanticRules semanticRules, + Instant now, + Version wantedNodeVespaVersion) { this.logger = deployLogger; this.fileRegistry = fileRegistry; this.rankProfileRegistry = rankProfileRegistry; @@ -100,7 +114,8 @@ public class DeployState implements ConfigDefinitionStore { this.zone = zone; this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated - this.importedModels = new ImportedModels(applicationPackage.getFileReference(ApplicationPackage.MODELS_DIR)); + this.importedModels = new ImportedModels(applicationPackage.getFileReference(ApplicationPackage.MODELS_DIR), + modelImporters); this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml).orElse(ValidationOverrides.empty); this.wantedNodeVespaVersion = wantedNodeVespaVersion; @@ -232,6 +247,7 @@ public class DeployState implements ConfigDefinitionStore { private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty(); private Optional<Model> previousModel = Optional.empty(); private Set<Rotation> rotations = new HashSet<>(); + private Collection<ModelImporter> modelImporters = Collections.emptyList(); private Zone zone = Zone.defaultZone(); private Instant now = Instant.now(); private Version wantedNodeVespaVersion = Vtag.currentVersion; @@ -281,6 +297,11 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder modelImporters(Collection<ModelImporter> modelImporters) { + this.modelImporters = modelImporters; + return this; + } + public Builder zone(Zone zone) { this.zone = zone; return this; @@ -305,9 +326,23 @@ public class DeployState implements ConfigDefinitionStore { QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles, validationParameters); - return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, - properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, - zone, queryProfiles, semanticRules, now, wantedNodeVespaVersion); + return new DeployState(applicationPackage, + searchDocumentModel, + rankProfileRegistry, + fileRegistry, + logger, + hostProvisioner, + properties, + permanentApplicationPackage, + configDefinitionRepo, + previousModel, + rotations, + modelImporters, + zone, + queryProfiles, + semanticRules, + now, + wantedNodeVespaVersion); } private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 78f61d7192d..a0cbc4271f3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -16,7 +16,7 @@ import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModels; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 0c5c7733dda..eb221a80638 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -12,7 +12,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchdefinition.derived.validation.Validation; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModels; import java.io.IOException; import java.io.Writer; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index fcbfb47c597..c955cd9956b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -5,7 +5,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.RankingConstants; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModels; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.Search; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index a1b0e72051b..fd468c8eca7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -9,7 +9,7 @@ import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModels; import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 2fe2dacf2ce..f52eedc8d08 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.expressiontransforms; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModels; import com.yahoo.searchlib.rankingexpression.transform.TransformContext; import java.util.HashMap; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index fef198f7939..b4de5e925f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -33,8 +33,8 @@ import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.ml.ConvertedModel; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModels; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModel; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModels; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; @@ -150,7 +150,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri this(configModelRegistry, deployState, true, null); } - private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) throws IOException, SAXException { + private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) + throws IOException, SAXException { super("vespamodel"); this.validationOverrides = deployState.validationOverrides(); configModelRegistry = new VespaConfigModelRegistry(configModelRegistry); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java index 2af9b297e9e..ff64b6753f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -21,6 +21,7 @@ import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.Version; import com.yahoo.config.provision.Zone; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ModelImporter; import com.yahoo.vespa.config.VespaVersion; import com.yahoo.vespa.model.application.validation.Validation; @@ -29,6 +30,8 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.time.Clock; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.logging.Logger; @@ -41,19 +44,17 @@ public class VespaModelFactory implements ModelFactory { private static final Logger log = Logger.getLogger(VespaModelFactory.class.getName()); private final ConfigModelRegistry configModelRegistry; + private final Collection<ModelImporter> modelImporters; private final Zone zone; private final Clock clock; private final Version version; /** Creates a factory for vespa models for this version of the source */ @Inject - public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) { - this(Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro), pluginRegistry, zone); - } - - /** Creates a factory for vespa models of a particular version */ - public VespaModelFactory(Version version, ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) { - this.version = version; + public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, + ComponentRegistry<ModelImporter> modelImporters, + Zone zone) { + this.version = Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); List<ConfigModelBuilder> modelBuilders = new ArrayList<>(); for (ConfigModelPlugin plugin : pluginRegistry.allComponents()) { if (plugin instanceof ConfigModelBuilder) { @@ -61,6 +62,7 @@ public class VespaModelFactory implements ModelFactory { } } this.configModelRegistry = new MapConfigModelRegistry(modelBuilders); + this.modelImporters = modelImporters.allComponents(); this.zone = zone; this.clock = Clock.systemUTC(); } @@ -79,6 +81,7 @@ public class VespaModelFactory implements ModelFactory { } else { this.configModelRegistry = configModelRegistry; } + this.modelImporters = Collections.emptyList(); this.zone = Zone.defaultZone(); this.clock = clock; } @@ -137,6 +140,7 @@ public class VespaModelFactory implements ModelFactory { .properties(createDeployProperties(modelContext.properties())) .modelHostProvisioner(createHostProvisioner(modelContext)) .rotations(modelContext.properties().rotations()) + .modelImporters(modelImporters) .zone(zone) .now(clock.instant()) .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 30586b1e677..18f3cd6e088 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -19,7 +19,7 @@ import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.integration.ml.ImportedModel; +import com.yahoo.searchlib.rankingexpression.integration.ml.importer.ImportedModel; import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -41,7 +41,6 @@ import com.yahoo.tensor.serialization.TypedBinaryFormat; import java.io.BufferedReader; import java.io.File; import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.io.UncheckedIOException; import java.util.ArrayList; |