Merge pull request #16710 from vespa-engine/hmusum/use-flag-for-metrics-proxy-max-heap

Control max heap size for metrics proxy with feature flag

11 files changed, 123 insertions, 38 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index f5765c6cd61..53ecc5892b6 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
@@ -85,6 +86,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return false; }
         @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.2; }
         @ModelFeatureFlag(owners = {"hmusum"}) default int clusterControllerMaxHeapSizeInMb() { return 512; }
+        @ModelFeatureFlag(owners = {"hmusum"}) default int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return 512; }
         @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); }
         @ModelFeatureFlag(owners = {"tokle"}) default boolean tenantIamRole() { return false; }
         @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; }
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 3d06fdf6758..be15abc5ac8 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -10,6 +10,7 @@ import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.Quota;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.Zone;
 
@@ -56,6 +57,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private boolean enableFeedBlockInDistributor = false;
     private double maxDeadBytesRatio = 0.2;
     private int clusterControllerMaxHeapSizeInMb = 512;
+    private int metricsProxyMaxHeapSizeInMb = 512;
     private int maxActivationInhibitedOutOfSyncGroups = 0;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
@@ -95,6 +97,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; }
     @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; }
     @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; }
+    @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb; }
     @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; }
 
     public TestProperties setFeedConcurrency(double feedConcurrency) {
@@ -227,6 +230,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties metricsProxyMaxHeapSizeInMb(int heapSize) {
+        metricsProxyMaxHeapSizeInMb = heapSize;
+        return this;
+    }
+
     public TestProperties maxActivationInhibitedOutOfSyncGroups(int nGroups) {
         maxActivationInhibitedOutOfSyncGroups = nGroups;
         return this;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
index 08f7ff82f4e..0f006c31959 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java
@@ -75,7 +75,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable
     private Optional<LogserverContainerCluster> logServerContainerCluster = Optional.empty();
 
     private ZooKeepersConfigProvider zooKeepersConfigProvider;
-    private FileDistributionConfigProducer fileDistribution;
+    private final FileDistributionConfigProducer fileDistribution;
     private final boolean multitenant;
 
     public Admin(AbstractConfigProducer parent,
@@ -220,7 +220,7 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable
         for (var host : hosts) {
             // Send hostname to be used in configId (instead of index), as the sorting of hosts seems to be unstable
             // between config changes, even when the set of hosts is unchanged.
-            var container = new MetricsProxyContainer(metricsProxyCluster, host.getHostname(), index, deployState.isHosted());
+            var container = new MetricsProxyContainer(metricsProxyCluster, host, index, deployState);
             addAndInitializeService(deployState.getDeployLogger(), host, container);
             metricsProxyCluster.addContainer(container);
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
index aef7c846a19..690900865ad 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
@@ -11,14 +11,19 @@ import ai.vespa.metricsproxy.rpc.RpcConnector;
 import ai.vespa.metricsproxy.rpc.RpcConnectorConfig;
 import ai.vespa.metricsproxy.service.VespaServices;
 import ai.vespa.metricsproxy.service.VespaServicesConfig;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.container.ContainerServiceType;
-import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.search.config.QrStartConfig;
+import com.yahoo.vespa.model.HostResource;
 import com.yahoo.vespa.model.PortAllocBridge;
 import com.yahoo.vespa.model.container.Container;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME;
@@ -33,15 +38,23 @@ public class MetricsProxyContainer extends Container implements
         NodeDimensionsConfig.Producer,
         NodeInfoConfig.Producer,
         RpcConnectorConfig.Producer,
-        VespaServicesConfig.Producer
+        VespaServicesConfig.Producer,
+        QrStartConfig.Producer
 {
     public static final int BASEPORT = 19092;
 
     final boolean isHostedVespa;
+    private final Optional<ClusterMembership> clusterMembership;
+    private final ModelContext.FeatureFlags featureFlags;
+    private final MetricsProxyContainerCluster cluster;
 
-    public MetricsProxyContainer(AbstractConfigProducer<?> parent, String hostname, int index, boolean isHostedVespa) {
-        super(parent, hostname, index, isHostedVespa);
-        this.isHostedVespa = isHostedVespa;
+
+    public MetricsProxyContainer(MetricsProxyContainerCluster cluster, HostResource host, int index, DeployState deployState) {
+        super(cluster, host.getHostname(), index, deployState.isHosted());
+        this.isHostedVespa = deployState.isHosted();
+        this.clusterMembership = host.spec().membership();
+        this.featureFlags = deployState.featureFlags();
+        this.cluster = cluster;
         setProp("clustertype", "admin");
         setProp("index", String.valueOf(index));
         addNodeSpecificComponents();
@@ -131,6 +144,19 @@ public class MetricsProxyContainer extends Container implements
                 .hostname(getHostName());
     }
 
+    @Override
+    public void getConfig(QrStartConfig.Builder builder) {
+        cluster.getConfig(builder);
+
+        if (clusterMembership.isPresent()) {
+            int maxHeapSize = featureFlags.metricsProxyMaxHeapSizeInMb(clusterMembership.get().cluster().type());
+            boolean verboseGc = (maxHeapSize < 512);
+            builder.jvm
+                    .verbosegc(verboseGc)
+                    .heapsize(maxHeapSize);
+        }
+    }
+
     private String getNodeRole() {
         String hostConfigId = getHost().getHost().getConfigId();
         if (! isHostedVespa) return hostConfigId;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index a0b99516ce3..194967f9868 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -86,7 +86,6 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
     private final AbstractConfigProducer<?> parent;
     private final ApplicationId applicationId;
 
-
     public MetricsProxyContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) {
         super(parent, name, name, deployState, true);
         this.parent = parent;
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index eeded9e6c2f..f1c81e29923 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -1795,7 +1795,7 @@ public class ModelProvisioningTest {
             assertTrue("Initial servers are not joining", config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining));
         }
         {
-            VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, 0, Optional.of(model));
+            VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, 0, Optional.of(model), new DeployState.Builder());
             ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk");
             ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder();
             cluster.getContainers().forEach(c -> c.getConfig(config));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
index 7ed546c445d..1a7258db7e2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
@@ -13,6 +13,9 @@ import ai.vespa.metricsproxy.http.yamas.YamasHandler;
 import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
 import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
 import com.yahoo.component.ComponentSpecification;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.core.ApplicationMetadataConfig;
@@ -23,9 +26,9 @@ import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.App
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.Handler;
 import org.junit.Test;
+
 import java.util.Collection;
 
-import static com.yahoo.vespa.model.container.ContainerCluster.G1GC;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_FILE;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.zoneString;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
@@ -39,6 +42,7 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.g
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly;
+import static com.yahoo.vespa.model.container.ContainerCluster.G1GC;
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.CoreMatchers.endsWith;
 import static org.hamcrest.CoreMatchers.hasItem;
@@ -74,24 +78,39 @@ public class MetricsProxyContainerClusterTest {
     }
 
     private void metrics_proxy_has_expected_qr_start_options(MetricsProxyModelTester.TestMode mode) {
-        VespaModel model = getModel(servicesWithAdminOnly(), mode);
-        QrStartConfig qrStartConfig = getQrStartConfig(model);
-        assertEquals(32, qrStartConfig.jvm().minHeapsize());
-        assertEquals(512, qrStartConfig.jvm().heapsize());
-        assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
-        assertEquals(2, qrStartConfig.jvm().availableProcessors());
-        assertFalse(qrStartConfig.jvm().verbosegc());
-        assertEquals(G1GC, qrStartConfig.jvm().gcopts());
-        assertEquals(512, qrStartConfig.jvm().stacksize());
-        assertEquals(0, qrStartConfig.jvm().directMemorySizeCache());
-        assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
-        assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize());
+        metrics_proxy_has_expected_qr_start_options(mode, 0);
+    }
+
+    private void metrics_proxy_has_expected_qr_start_options(MetricsProxyModelTester.TestMode mode, int maxHeapForAdminClusterNodes) {
+        DeployState.Builder builder = new DeployState.Builder();
+        if (maxHeapForAdminClusterNodes > 0) {
+            builder.properties(new TestProperties().metricsProxyMaxHeapSizeInMb(maxHeapForAdminClusterNodes));
+        }
+
+        VespaModel model = getModel(servicesWithAdminOnly(), mode, builder);
+        for (HostInfo host : model.getHosts()) {
+            QrStartConfig qrStartConfig = getQrStartConfig(model, host.getHostname());
+            assertEquals(32, qrStartConfig.jvm().minHeapsize());
+            assertEquals(maxHeapForAdminClusterNodes > 0 ? maxHeapForAdminClusterNodes : 512, qrStartConfig.jvm().heapsize());
+            assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
+            assertEquals(2, qrStartConfig.jvm().availableProcessors());
+            assertFalse(qrStartConfig.jvm().verbosegc());
+            assertEquals(G1GC, qrStartConfig.jvm().gcopts());
+            assertEquals(512, qrStartConfig.jvm().stacksize());
+            assertEquals(0, qrStartConfig.jvm().directMemorySizeCache());
+            assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
+            assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize());
+        }
     }
 
     @Test
     public void metrics_proxy_has_expected_qr_start_options() {
         metrics_proxy_has_expected_qr_start_options(self_hosted);
         metrics_proxy_has_expected_qr_start_options(hosted);
+
+        // With max heap from feature flag
+        metrics_proxy_has_expected_qr_start_options(self_hosted, 123);
+        metrics_proxy_has_expected_qr_start_options(hosted, 123);
     }
 
     @Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
index b6037d2614e..7fcd5c14a0f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
@@ -1,7 +1,4 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/*
- * Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
- */
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 package com.yahoo.vespa.model.admin.metricsproxy;
 
@@ -11,12 +8,15 @@ import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
 import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
 import ai.vespa.metricsproxy.rpc.RpcConnectorConfig;
 import ai.vespa.metricsproxy.service.VespaServicesConfig;
+import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.admin.monitoring.Metric;
 import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
 import com.yahoo.vespa.model.test.VespaModelTester;
 
+import java.util.Optional;
+
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
 
@@ -40,12 +40,16 @@ class MetricsProxyModelTester {
     }
 
     static VespaModel getModel(String servicesXml, TestMode testMode) {
+        return getModel(servicesXml, testMode, new DeployState.Builder());
+    }
+
+    static VespaModel getModel(String servicesXml, TestMode testMode, DeployState.Builder builder) {
         var numberOfHosts = testMode == hosted ? 2 : 1;
         var tester = new VespaModelTester();
         tester.addHosts(numberOfHosts);
         tester.setHosted(testMode == hosted);
         if (testMode == hosted) tester.setApplicationId(MY_TENANT, MY_APPLICATION, MY_INSTANCE);
-        return tester.createModel(servicesXml, true);
+        return tester.createModel(servicesXml, true, builder);
     }
 
     static String containerConfigId(VespaModel model, MetricsProxyModelTester.TestMode mode) {
@@ -98,8 +102,8 @@ class MetricsProxyModelTester {
         return model.getConfig(ApplicationDimensionsConfig.class, CLUSTER_CONFIG_ID);
     }
 
-    static QrStartConfig getQrStartConfig(VespaModel model) {
-        return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID);
+    static QrStartConfig getQrStartConfig(VespaModel model, String hostname) {
+        return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID + "/" + hostname);
     }
 
     static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
index e779b5e43cb..220f87001aa 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
@@ -6,14 +6,12 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.ConfigModelRegistry;
 import com.yahoo.config.model.NullConfigModelRegistry;
 import com.yahoo.config.model.api.HostProvisioner;
-import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.provision.Host;
 import com.yahoo.config.model.provision.Hosts;
 import com.yahoo.config.model.provision.InMemoryProvisioner;
 import com.yahoo.config.model.provision.SingleNodeProvisioner;
-import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
@@ -21,7 +19,6 @@ import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.ProvisionLogger;
-import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
@@ -116,22 +113,38 @@ public class VespaModelTester {
 
     /** Creates a model which uses 0 as start index */
     public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) {
-        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, Optional.empty(), retiredHostNames);
+        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0,
+                           Optional.empty(), new DeployState.Builder(), retiredHostNames);
+    }
+
+    /** Creates a model which uses 0 as start index */
+    public VespaModel createModel(String services, boolean failOnOutOfCapacity, DeployState.Builder builder) {
+        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, Optional.empty(), builder);
     }
 
     /** Creates a model which uses 0 as start index */
     public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) {
-        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, Optional.empty(), retiredHostNames);
+        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0,
+                           Optional.empty(), new DeployState.Builder(), retiredHostNames);
     }
 
     /** Creates a model which uses 0 as start index */
     public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
-        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, Optional.empty(), retiredHostNames);
+        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters,
+                           Optional.empty(), new DeployState.Builder(), retiredHostNames);
     }
 
     /** Creates a model which uses 0 as start index */
     public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) {
-        return createModel(zone, services, failOnOutOfCapacity, false, 0, Optional.empty(), retiredHostNames);
+        return createModel(zone, services, failOnOutOfCapacity, false, 0,
+                           Optional.empty(), new DeployState.Builder(), retiredHostNames);
+    }
+
+    /** Creates a model which uses 0 as start index */
+    public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity,
+                                  DeployState.Builder deployStateBuilder, String ... retiredHostNames) {
+        return createModel(zone, services, failOnOutOfCapacity, false, 0,
+                           Optional.empty(), deployStateBuilder, retiredHostNames);
     }
 
     /**
@@ -144,7 +157,8 @@ public class VespaModelTester {
      * @return the resulting model
      */
     public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources,
-                                  int startIndexForClusters, Optional<VespaModel> previousModel, String ... retiredHostNames) {
+                                  int startIndexForClusters, Optional<VespaModel> previousModel,
+                                  DeployState.Builder deployStatebuilder, String ... retiredHostNames) {
         VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1"));
         ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg;
 
@@ -164,7 +178,7 @@ public class VespaModelTester {
                 .setUseDedicatedNodeForLogserver(useDedicatedNodeForLogserver)
                 .setDedicatedClusterControllerCluster(dedicatedClusterControllerCluster);
 
-        DeployState.Builder deployState = new DeployState.Builder()
+        DeployState.Builder deployState = deployStatebuilder
                 .applicationPackage(appPkg)
                 .modelHostProvisioner(provisioner)
                 .properties(properties)
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index a5b8c57ccaa..2568ff23f9e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -19,6 +19,7 @@ import com.yahoo.config.model.api.Quota;
 import com.yahoo.config.model.api.Reindexing;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
@@ -28,6 +29,7 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.IntFlag;
 import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.flags.UnboundFlag;
 
@@ -36,8 +38,10 @@ import java.net.URI;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.ToIntFunction;
 
 import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig;
+import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE;
 
 /**
  * Implementation of {@link ModelContext} for configserver.
@@ -169,6 +173,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean enableFeedBlockInDistributor;
         private final double maxDeadBytesRatio;
         private final int clusterControllerMaxHeapSizeInMb;
+        private final ToIntFunction<ClusterSpec.Type> metricsProxyMaxHeapSizeInMb;
         private final List<String> allowedAthenzProxyIdentities;
         private final boolean tenantIamRole;
         private final int maxActivationInhibitedOutOfSyncGroups;
@@ -193,6 +198,7 @@ public class ModelContextImpl implements ModelContext {
             this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR);
             this.maxDeadBytesRatio = flagValue(source, appId, Flags.MAX_DEAD_BYTES_RATIO);
             this.clusterControllerMaxHeapSizeInMb = flagValue(source, appId, Flags.CLUSTER_CONTROLLER_MAX_HEAP_SIZE_IN_MB);
+            this.metricsProxyMaxHeapSizeInMb = type -> Flags.METRICS_PROXY_MAX_HEAP_SIZE_IN_MB.bindTo(source).with(CLUSTER_TYPE, type.name()).value();
             this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES);
             this.tenantIamRole = flagValue(source, appId.tenant(), Flags.TENANT_IAM_ROLE);
             this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS);
@@ -217,6 +223,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; }
         @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; }
         @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; }
+        @Override public int metricsProxyMaxHeapSizeInMb(ClusterSpec.Type type) { return metricsProxyMaxHeapSizeInMb.applyAsInt(type); }
         @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; }
         @Override public boolean tenantIamRole() { return tenantIamRole; }
         @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 5e030014c83..d3ca08c5dba 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -239,6 +239,12 @@ public class Flags {
             "JVM max heap size for cluster controller in Mb",
             "Takes effect when restarting cluster controller");
 
+    public static final UnboundIntFlag METRICS_PROXY_MAX_HEAP_SIZE_IN_MB = defineIntFlag(
+            "metrics-proxy-max-heap-size-in-mb", 512,
+            List.of("hmusum"), "2021-03-01", "2021-05-01",
+            "JVM max heap size for metrics proxy in Mb",
+            "Takes effect when restarting metrics proxy");
+
     public static final UnboundBooleanFlag DEDICATED_CLUSTER_CONTROLLER_CLUSTER = defineFeatureFlag(
             "dedicated-cluster-controller-cluster", false,
             List.of("jonmv"), "2021-02-15", "2021-04-15",