Collect constants from compiled profiles

3 files changed, 48 insertions, 42 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
index 4e718714ded..d415f7d0d6e 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
@@ -40,7 +41,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
 
     private static final Logger log = Logger.getLogger(RankProfileList.class.getName());
 
-    private final Map<String, RawRankProfile> rankProfiles = new java.util.LinkedHashMap<>();
+    private final Map<String, RawRankProfile> rankProfiles;
     private final FileDistributedConstants constants;
     private final LargeRankExpressions largeRankExpressions;
     private final OnnxModels onnxModels;
@@ -51,6 +52,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
         constants = new FileDistributedConstants(null, List.of());
         largeRankExpressions = new LargeRankExpressions(null);
         onnxModels = new OnnxModels(null, Optional.empty());
+        rankProfiles = Map.of();
     }
 
     /**
@@ -66,39 +68,33 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
                            AttributeFields attributeFields,
                            DeployState deployState) {
         setName(schema == null ? "default" : schema.getName());
-        this.constants = deriveFileDistributedConstants(schema, constantsFromSchema, deployState);
         this.largeRankExpressions = largeRankExpressions;
         this.onnxModels = onnxModels;  // as ONNX models come from parsing rank expressions
-        deriveRankProfiles(schema, attributeFields, deployState);
+        this.rankProfiles = deriveRankProfiles(schema, attributeFields, deployState);
+        this.constants = deriveFileDistributedConstants(schema, constantsFromSchema, rankProfiles.values(), deployState);
     }
 
     private static FileDistributedConstants deriveFileDistributedConstants(Schema schema,
                                                                            Map<Reference, RankProfile.Constant> constantsFromSchema,
+                                                                           Collection<RawRankProfile> rankProfiles,
                                                                            DeployState deployState) {
         Map<Reference, RankProfile.Constant> allFileConstants = new HashMap<>();
-        addFileConstants(constantsFromSchema.values(), allFileConstants, schema != null ? schema.toString() : "[global]", deployState.getDeployLogger());
-        for (var profile : deployState.rankProfileRegistry().rankProfilesOf(schema))
-            addFileConstants(profile.constants().values(), allFileConstants, profile.toString(), deployState.getDeployLogger());
-        for (var profile : deployState.rankProfileRegistry().rankProfilesOf(null))
-            addFileConstants(profile.constants().values(), allFileConstants, profile.toString(), deployState.getDeployLogger());
+        addFileConstants(constantsFromSchema.values(), allFileConstants, schema != null ? schema.toString() : "[global]");
+        for (var profile : rankProfiles)
+            addFileConstants(profile.compiled().constants().values(), allFileConstants, profile.toString());
         return new FileDistributedConstants(deployState.getFileRegistry(), allFileConstants.values());
     }
 
     private static void addFileConstants(Collection<RankProfile.Constant> source,
                                          Map<Reference, RankProfile.Constant> destination,
-                                         String sourceName,
-                                         DeployLogger logger) {
+                                         String sourceName) {
         for (var constant : source) {
             if (constant.valuePath().isEmpty()) continue;
             var existing = destination.get(constant.name());
             if ( existing != null && ! constant.equals(existing)) {
-                String message = "Duplicate constants: " + sourceName + " have " + constant +
-                                 ", but we already have " + existing +
-                                 ": Value reference constants must be unique across all rank profiles and models";
-                if (! constant.type().equals(existing.type()))
-                    throw new IllegalArgumentException(message);
-                else // different paths are allowed
-                    logger.logApplicationPackage(Level.WARNING, message);
+                throw new IllegalArgumentException("Duplicate constants: " + sourceName + " have " + constant +
+                                                   ", but we already have " + existing +
+                                                   ": Value reference constants must be unique across all rank profiles/models");
             }
             destination.put(constant.name(), constant);
         }
@@ -106,15 +102,16 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
 
     public FileDistributedConstants constants() { return constants; }
 
-    private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) {
+    private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry, Set<String> processedProfiles) {
         return rank.inheritedNames().isEmpty() ||
-               rankProfiles.keySet().containsAll(rank.inheritedNames()) ||
+               processedProfiles.containsAll(rank.inheritedNames()) ||
                (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null));
     }
 
-    private void deriveRankProfiles(Schema schema,
-                                    AttributeFields attributeFields,
-                                    DeployState deployState) {
+    private Map<String, RawRankProfile>  deriveRankProfiles(Schema schema,
+                                                            AttributeFields attributeFields,
+                                                            DeployState deployState) {
+        Map<String,  RawRankProfile> rawRankProfiles = new LinkedHashMap<>();
         if (schema != null) { // profiles belonging to a schema have a default profile
             RawRankProfile rawRank = new RawRankProfile(deployState.rankProfileRegistry().get(schema, "default"),
                                                         largeRankExpressions,
@@ -122,7 +119,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
                                                         deployState.getImportedModels(),
                                                         attributeFields,
                                                         deployState.getProperties());
-            rankProfiles.put(rawRank.getName(), rawRank);
+            rawRankProfiles.put(rawRank.getName(), rawRank);
         }
 
         Map<String, RankProfile> remaining = new LinkedHashMap<>();
@@ -130,27 +127,29 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
         remaining.remove("default");
         while (!remaining.isEmpty()) {
             List<RankProfile> ready = new ArrayList<>();
-            remaining.forEach((name, rank) -> {
-                if (areDependenciesReady(rank, deployState.rankProfileRegistry())) ready.add(rank);
+            remaining.forEach((name, profile) -> {
+                if (areDependenciesReady(profile, deployState.rankProfileRegistry(), rawRankProfiles.keySet()))
+                    ready.add(profile);
             });
-            processRankProfiles(ready,
-                                deployState.getQueryProfiles().getRegistry(),
-                                deployState.getImportedModels(),
-                                schema,
-                                attributeFields,
-                                deployState.getProperties(),
-                                deployState.getExecutor());
+            rawRankProfiles.putAll(processRankProfiles(ready,
+                                                       deployState.getQueryProfiles().getRegistry(),
+                                                       deployState.getImportedModels(),
+                                                       schema,
+                                                       attributeFields,
+                                                       deployState.getProperties(),
+                                                       deployState.getExecutor()));
             ready.forEach(rank -> remaining.remove(rank.name()));
         }
+        return rawRankProfiles;
     }
 
-    private void processRankProfiles(List<RankProfile> ready,
-                                     QueryProfileRegistry queryProfiles,
-                                     ImportedMlModels importedModels,
-                                     Schema schema,
-                                     AttributeFields attributeFields,
-                                     ModelContext.Properties deployProperties,
-                                     ExecutorService executor) {
+    private Map<String, RawRankProfile> processRankProfiles(List<RankProfile> ready,
+                                                            QueryProfileRegistry queryProfiles,
+                                                            ImportedMlModels importedModels,
+                                                            Schema schema,
+                                                            AttributeFields attributeFields,
+                                                            ModelContext.Properties deployProperties,
+                                                            ExecutorService executor) {
         Map<String, Future<RawRankProfile>> futureRawRankProfiles = new LinkedHashMap<>();
         for (RankProfile rank : ready) {
             if (schema == null) {
@@ -161,10 +160,12 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ
                                                                                             attributeFields, deployProperties)));
         }
         try {
+            Map<String,  RawRankProfile> rawRankProfiles = new LinkedHashMap<>();
             for (Future<RawRankProfile> rawFuture : futureRawRankProfiles.values()) {
                 RawRankProfile rawRank = rawFuture.get();
-                rankProfiles.put(rawRank.getName(), rawRank);
+                rawRankProfiles.put(rawRank.getName(), rawRank);
             }
+            return rawRankProfiles;
         } catch (InterruptedException | ExecutionException e) {
             throw new IllegalStateException(e);
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index 77245da5ddd..7b70e818e0f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -53,15 +53,21 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
     private final String name;
     private final Compressor.Compression compressedProperties;
 
+    /**  The compiled profile this is created from. */
+    private final RankProfile compiled;
+
     /** Creates a raw rank profile from the given rank profile. */
     public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions,
                           QueryProfileRegistry queryProfiles, ImportedMlModels importedModels,
                           AttributeFields attributeFields, ModelContext.Properties deployProperties) {
         this.name = rankProfile.name();
-        compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), attributeFields, deployProperties, queryProfiles)
+        compiled = rankProfile.compile(queryProfiles, importedModels);
+        compressedProperties = compress(new Deriver(compiled, attributeFields, deployProperties, queryProfiles)
                                                 .derive(largeExpressions));
     }
 
+    public RankProfile compiled() { return compiled; }
+
     private Compressor.Compression compress(List<Pair<String, String>> properties) {
         StringBuilder b = new StringBuilder();
         for (Pair<String, String> property : properties)
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index 4662f68c31d..03fc7592b40 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -35,7 +35,6 @@ import com.yahoo.searchdefinition.OnnxModel;
 import com.yahoo.searchdefinition.OnnxModels;
 import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.RankProfileRegistry;
-import com.yahoo.searchdefinition.derived.FileDistributedConstants;
 import com.yahoo.searchdefinition.derived.AttributeFields;
 import com.yahoo.searchdefinition.derived.RankProfileList;
 import com.yahoo.searchdefinition.document.SDField;