Wire in fileregistry to RankingConstants and OnnxModels.

8 files changed, 48 insertions, 33 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java
index 60733a4f5ba..96ee8c0c2f4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java
@@ -1,6 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition;
 
+import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.vespa.model.AbstractService;
 
 import java.util.Collection;
@@ -16,7 +17,11 @@ import java.util.Map;
 public class OnnxModels {
 
     private final Map<String, OnnxModel> models = new HashMap<>();
+    private final FileRegistry fileRegistry;
 
+    public OnnxModels(FileRegistry fileRegistry) {
+        this.fileRegistry = fileRegistry;
+    }
     public void add(OnnxModel model) {
         model.validate();
         String name = model.getName();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index 68e03ecb188..3c1f00f1252 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -120,9 +120,6 @@ public class RankProfile implements Cloneable {
 
     private List<ImmutableSDField> allFieldsList;
 
-    /** Global onnx models not tied to a search definition */
-    private OnnxModels onnxModels = new OnnxModels();
-
     /**
      * Creates a new rank profile for a particular search definition
      *
@@ -144,12 +141,11 @@ public class RankProfile implements Cloneable {
      * @param name  the name of the new profile
      * @param model the model owning this profile
      */
-    public RankProfile(String name, VespaModel model, RankProfileRegistry rankProfileRegistry, OnnxModels onnxModels) {
+    public RankProfile(String name, VespaModel model, RankProfileRegistry rankProfileRegistry) {
         this.name = Objects.requireNonNull(name, "name cannot be null");
         this.search = null;
         this.model = Objects.requireNonNull(model, "model cannot be null");
         this.rankProfileRegistry = rankProfileRegistry;
-        this.onnxModels = onnxModels;
     }
 
     public String getName() { return name; }
@@ -168,7 +164,7 @@ public class RankProfile implements Cloneable {
     }
 
     public Map<String, OnnxModel> onnxModels() {
-        return search != null ? search.onnxModels().asMap() : onnxModels.asMap();
+        return search != null ? search.onnxModels().asMap() : model.onnxModels().asMap();
     }
 
     private Stream<ImmutableSDField> allFields() {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
index 82381aa63fc..a61e1ae3efb 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition;
 
+import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.vespa.model.AbstractService;
 
 import java.util.Collection;
@@ -17,6 +18,11 @@ import java.util.Map;
 public class RankingConstants {
 
     private final Map<String, RankingConstant> constants = new HashMap<>();
+    private final FileRegistry fileRegistry;
+
+    public RankingConstants(FileRegistry fileRegistry) {
+        this.fileRegistry = fileRegistry;
+    }
 
     public void add(RankingConstant constant) {
         constant.validate();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index 973ac6655cc..f3d6e2f78a2 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -86,10 +86,10 @@ public class Search implements ImmutableSearch {
     private final LargeRankExpressions largeRankExpressions;
 
     /** Ranking constants of this */
-    private final RankingConstants rankingConstants = new RankingConstants();
+    private final RankingConstants rankingConstants;
 
     /** Onnx models of this */
-    private final OnnxModels onnxModels = new OnnxModels();
+    private final OnnxModels onnxModels;
 
     private Optional<TemporaryImportedFields> temporaryImportedFields = Optional.of(new TemporaryImportedFields());
     private Optional<ImportedFields> importedFields = Optional.empty();
@@ -123,6 +123,8 @@ public class Search implements ImmutableSearch {
         this.properties = properties;
         this.documentsOnly = documentsOnly;
         largeRankExpressions = new LargeRankExpressions(fileRegistry);
+        rankingConstants = new RankingConstants(fileRegistry);
+        onnxModels = new OnnxModels(fileRegistry);
     }
 
     protected void setName(String name) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index 52d99c523ea..16de8fd4b9e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -85,8 +85,9 @@ public class DerivedConfiguration {
             summaries = new Summaries(search, deployLogger);
             summaryMap = new SummaryMap(search);
             juniperrc = new Juniperrc(search);
-            rankProfileList = new RankProfileList(search, search.rankingConstants(), search.rankExpressionFiles(), attributeFields,
-                                                  rankProfileRegistry, queryProfiles, importedModels, deployProperties);
+            rankProfileList = new RankProfileList(search, search.rankingConstants(), search.rankExpressionFiles(),
+                                                  search.onnxModels(), attributeFields, rankProfileRegistry,
+                                                  queryProfiles, importedModels, deployProperties);
             indexingScript = new IndexingScript(search);
             indexInfo = new IndexInfo(search);
             indexSchema = new IndexSchema(search);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
index a9a04fccf5d..7f19fcd24d4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
@@ -41,9 +41,9 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
     public static RankProfileList empty = new RankProfileList();
 
     private RankProfileList() {
-        rankingConstants = new RankingConstants();
-        largeRankExpressions = new LargeRankExpressions(new MockFileRegistry());
-        onnxModels = new OnnxModels();
+        rankingConstants = new RankingConstants(null);
+        largeRankExpressions = new LargeRankExpressions(null);
+        onnxModels = new OnnxModels(null);
         dryRunOnnxOnSetup = true;
     }
 
@@ -56,6 +56,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
     public RankProfileList(Search search,
                            RankingConstants rankingConstants,
                            LargeRankExpressions largeRankExpressions,
+                           OnnxModels onnxModels,
                            AttributeFields attributeFields,
                            RankProfileRegistry rankProfileRegistry,
                            QueryProfileRegistry queryProfiles,
@@ -64,7 +65,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
         setName(search == null ? "default" : search.getName());
         this.rankingConstants = rankingConstants;
         this.largeRankExpressions = largeRankExpressions;
-        onnxModels = search == null ? new OnnxModels() : search.onnxModels();  // as ONNX models come from parsing rank expressions
+        this.onnxModels = onnxModels;  // as ONNX models come from parsing rank expressions
         dryRunOnnxOnSetup = deployProperties.featureFlags().dryRunOnnxOnSetup();
         deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java
index 247f61b3de0..ce7899b8711 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java
@@ -5,7 +5,8 @@ import com.yahoo.config.model.ConfigModelRegistry;
 import com.yahoo.config.model.admin.AdminModel;
 import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
 import com.yahoo.config.model.builder.xml.ConfigModelId;
-import com.yahoo.vespa.model.builder.xml.dom.*;
+import com.yahoo.vespa.model.builder.xml.dom.DomClientsBuilder;
+import com.yahoo.vespa.model.builder.xml.dom.DomRoutingBuilder;
 import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
 import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking;
 import com.yahoo.vespa.model.content.Content;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index 7b8c7ef6421..ca3bc7d66b1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -125,11 +125,14 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
     private final RankProfileList rankProfileList;
 
     /** The global ranking constants of this model */
-    private final RankingConstants rankingConstants = new RankingConstants();
+    private final RankingConstants rankingConstants;
 
-    /** External rank expression files of this */
+    /** Large rank expression files of this */
     private final LargeRankExpressions largeRankExpressions;
 
+    /** Large rank expression files of this */
+    private final OnnxModels onnxModels;
+
     /** The validation overrides of this. This is never null. */
     private final ValidationOverrides validationOverrides;
 
@@ -172,13 +175,14 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
     private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor)
             throws IOException, SAXException {
         super("vespamodel");
-        this.version = deployState.getVespaVersion();
+        version = deployState.getVespaVersion();
         largeRankExpressions = new LargeRankExpressions(deployState.getFileRegistry());
-        this.validationOverrides = deployState.validationOverrides();
-        configModelRegistry = new VespaConfigModelRegistry(configModelRegistry);
+        rankingConstants = new RankingConstants(deployState.getFileRegistry());
+        onnxModels = new OnnxModels(deployState.getFileRegistry());
+        validationOverrides = deployState.validationOverrides();
+        applicationPackage = deployState.getApplicationPackage();
+        provisioned = deployState.provisioned();
         VespaModelBuilder builder = new VespaDomBuilder();
-        this.applicationPackage = deployState.getApplicationPackage();
-        this.provisioned = deployState.provisioned();
         root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this);
 
         createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(),
@@ -186,6 +190,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
         rankProfileList = new RankProfileList(null, // null search -> global
                                               rankingConstants,
                                               largeRankExpressions,
+                                              onnxModels,
                                               AttributeFields.empty,
                                               deployState.rankProfileRegistry(),
                                               deployState.getQueryProfiles().getRegistry(),
@@ -194,7 +199,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
 
         HostSystem hostSystem = root.hostSystem();
         if (complete) { // create a completed, frozen model
-            configModelRepo.readConfigModels(deployState, this, builder, root, configModelRegistry);
+            configModelRepo.readConfigModels(deployState, this, builder, root, new VespaConfigModelRegistry(configModelRegistry));
             addServiceClusters(deployState, builder);
             setupRouting(deployState);
             this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor();
@@ -266,6 +271,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
 
     public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; }
 
+    public OnnxModels onnxModels() { return onnxModels; }
+
     /** Creates a mutable model with no services instantiated */
     public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException {
         return new VespaModel(new NullConfigModelRegistry(), deployState, false,
@@ -296,8 +303,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
                                           QueryProfiles queryProfiles) {
         if ( ! importedModels.all().isEmpty()) { // models/ directory is available
             for (ImportedMlModel model : importedModels.all()) {
-                OnnxModels onnxModels = onnxModelInfoFromSource(model);
-                RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry, onnxModels);
+                onnxModelInfoFromSource(model);
+                RankProfile profile = new RankProfile(model.name(), this, rankProfileRegistry);
                 rankProfileRegistry.add(profile);
                 ConvertedModel convertedModel = ConvertedModel.fromSource(new ModelName(model.name()),
                                                                           model.name(), profile, queryProfiles.getRegistry(), model);
@@ -309,8 +316,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
             for (ApplicationFile generatedModelDir : generatedModelsDir.listFiles()) {
                 String modelName = generatedModelDir.getPath().last();
                 if (modelName.contains(".")) continue; // Name space: Not a global profile
-                OnnxModels onnxModels = onnxModelInfoFromStore(modelName);
-                RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry, onnxModels);
+                onnxModelInfoFromStore(modelName);
+                RankProfile profile = new RankProfile(modelName, this, rankProfileRegistry);
                 rankProfileRegistry.add(profile);
                 ConvertedModel convertedModel = ConvertedModel.fromStore(new ModelName(modelName), modelName, profile);
                 convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false));
@@ -319,8 +326,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
         new Processing().processRankProfiles(deployLogger, rankProfileRegistry, queryProfiles, true, false);
     }
 
-    private OnnxModels onnxModelInfoFromSource(ImportedMlModel model) {
-        OnnxModels onnxModels = new OnnxModels();
+    private void onnxModelInfoFromSource(ImportedMlModel model) {
         if (model.modelType().equals(ImportedMlModel.ModelType.ONNX)) {
             String path = model.source();
             String applicationPath = this.applicationPackage.getFileReference(Path.fromString("")).toString();
@@ -329,14 +335,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
             }
             loadOnnxModelInfo(onnxModels, model.name(), path);
         }
-        return onnxModels;
     }
 
-    private OnnxModels onnxModelInfoFromStore(String modelName) {
-        OnnxModels onnxModels = new OnnxModels();
+    private void onnxModelInfoFromStore(String modelName) {
         String path = ApplicationPackage.MODELS_DIR.append(modelName + ".onnx").toString();
         loadOnnxModelInfo(onnxModels, modelName, path);
-        return onnxModels;
     }
 
     private void loadOnnxModelInfo(OnnxModels onnxModels, String name, String path) {