Merge pull request #5639 from vespa-engine/balder/reduce-memory-footprint-for-rankprofiles

Balder/reduce memory footprint for rankprofiles

2 files changed, 26 insertions, 38 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index b02362154d9..07b79c4f604 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -145,9 +145,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
         /**
          * The rank type definitions used to derive settings for the native rank features
          */
-        private NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default");
+        private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default");
 
-        private RankProfile rankProfile;
+        private final Map<String, String> attributeTypes;
+        private final Map<String, String> queryFeatureTypes;
 
         private Set<String> filterFields = new java.util.LinkedHashSet<>();
 
@@ -155,11 +156,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
          * Creates a raw rank profile from the given rank profile
          */
         public Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, AttributeFields attributeFields) {
-            this.rankProfile = rankProfile.compile(queryProfiles);
-            deriveRankingFeatures(this.rankProfile);
-            deriveRankTypeSetting(this.rankProfile, attributeFields);
-            deriveFilterFields(this.rankProfile);
-            deriveWeightProperties(this.rankProfile);
+            RankProfile compiled = rankProfile.compile(queryProfiles);
+            attributeTypes = compiled.getAttributeTypes();
+            queryFeatureTypes = compiled.getQueryFeatureTypes();
+            deriveRankingFeatures(compiled);
+            deriveRankTypeSetting(compiled, attributeFields);
+            deriveFilterFields(compiled);
+            deriveWeightProperties(compiled);
         }
 
         private void deriveFilterFields(RankProfile rp) {
@@ -209,7 +212,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
         private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) {
             SerializationContext context = new SerializationContext(eMacros);
             for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) {
-                String script = e.getValue().getBody().getRoot().toString(context, null, null);
+                String script = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString();
                 context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), script);
             }
             return context.serializedFunctions();
@@ -223,7 +226,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
                 // Is the feature a macro?
                 if (context.getFunction(referenceNode.getName()) != null) {
                     context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()),
-                                                     referenceNode.toString(context, null, null));
+                                                     referenceNode.toString(new StringBuilder(), context, null, null).toString());
                     ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput());
                     macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode);
                     i.remove(); // Will add the expanded one in next block
@@ -363,7 +366,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
                 properties.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "");
                 properties.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "");
                 properties.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "");
-                RankProfile.DiversitySettings diversitySettings = rankProfile.getMatchPhaseSettings().getDiversity();
+                RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity();
                 if (diversitySettings != null) {
                     properties.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute());
                     properties.put("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()));
@@ -388,10 +391,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
                 String fieldName = (String) filterFieldsIterator.next();
                 properties.put("vespa.isfilterfield." + fieldName + ".part42", String.valueOf(true));
             }
-            for (Map.Entry<String, String> attributeType : rankProfile.getAttributeTypes().entrySet()) {
+            for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) {
                 properties.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue());
             }
-            for (Map.Entry<String, String> queryFeatureType : rankProfile.getQueryFeatureTypes().entrySet()) {
+            for (Map.Entry<String, String> queryFeatureType : queryFeatureTypes.entrySet()) {
                 properties.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue());
             }
             if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties");
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index be76a8dfa40..83c6374fe9a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -133,38 +133,23 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
      * @param deployState the global deploy state to use for this model.
      */
     public VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState) throws IOException, SAXException {
-        this(configModelRegistry, deployState, true, null);
-    }
-
-    private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) throws IOException, SAXException {
         super("vespamodel");
         this.deployState = deployState;
         this.validationOverrides = deployState.validationOverrides();
         configModelRegistry = new VespaConfigModelRegistry(configModelRegistry);
         VespaModelBuilder builder = new VespaDomBuilder();
         root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this);
-        if (complete) { // create a a completed, frozen model
-            configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry);
-            addServiceClusters(deployState.getApplicationPackage(), builder);
-            this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); // must happen after the two lines above
-            setupRouting();
-            this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor();
-            getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState);
-            freezeModelTopology();
-            root.prepare(configModelRepo);
-            configModelRepo.prepareConfigModels();
-            validateWrapExceptions();
-            this.deployState = null;
-        }
-        else { // create a model with no services instantiated and the given file distributor
-            this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs());
-            this.fileDistributor = fileDistributor;
-        }
-    }
-    
-    /** Creates a mutable model with no services instantiated */
-    public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException {
-        return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry(), null));
+        configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry);
+        addServiceClusters(deployState.getApplicationPackage(), builder);
+        this.allocatedHosts = AllocatedHosts.withHosts(root.getHostSystem().getHostSpecs()); // must happen after the two lines above
+        setupRouting();
+        this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor();
+        getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState);
+        freezeModelTopology();
+        root.prepare(configModelRepo);
+        configModelRepo.prepareConfigModels();
+        validateWrapExceptions();
+        this.deployState = null;
     }
 
     private void validateWrapExceptions() {