Merge pull request #565 from yahoo/balder/control-malloc-options

Balder/control malloc options

7 files changed, 251 insertions, 72 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index 2101ac25314..0d41fe34d6b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -53,6 +53,11 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
     // If this is true it will dump core when OOM
     private boolean coreOnOOM = false;
 
+    private String noVespaMalloc = "";
+    private String vespaMalloc = "";
+    private String vespaMallocDebug = "";
+    private String vespaMallocDebugStackTrace = "";
+
     /** The ports metainfo object */
     protected PortsMeta portsMeta = new PortsMeta();
 
@@ -410,12 +415,52 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
         this.preload = preload;
     }
     public long getMMapNoCoreLimit() { return mmapNoCoreLimit; }
-    public void setMMapNoCoreLimit(long noCoreLimit) {
-        this.mmapNoCoreLimit = noCoreLimit;
-    }
+    public void setMMapNoCoreLimit(long noCoreLimit) { this.mmapNoCoreLimit = noCoreLimit; }
     public boolean getCoreOnOOM() { return coreOnOOM; }
-    public void setCoreOnOOM(boolean coreOnOOM) {
-        this.coreOnOOM = coreOnOOM;
+    public void setCoreOnOOM(boolean coreOnOOM) { this.coreOnOOM = coreOnOOM; }
+
+    public String getNoVespaMalloc() { return noVespaMalloc; }
+    public String getVespaMalloc() { return vespaMalloc; }
+    public String getVespaMallocDebug() { return vespaMallocDebug; }
+    public String getVespaMallocDebugStackTrace() { return vespaMallocDebugStackTrace; }
+    public void setNoVespaMalloc(String s) { noVespaMalloc = s; }
+    public void setVespaMalloc(String s) { vespaMalloc = s; }
+    public void setVespaMallocDebug(String s) { vespaMallocDebug = s; }
+    public void setVespaMallocDebugStackTrace(String s) { vespaMallocDebugStackTrace = s; }
+
+    public String getMMapNoCoreEnvVariable() {
+        return (getMMapNoCoreLimit() >= 0l)
+                ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " "
+                : "";
+    }
+
+    public String getCoreOnOOMEnvVariable() {
+        return getCoreOnOOM() ? "" : "VESPA_SILENCE_CORE_ON_OOM=true ";
+    }
+    public String getNoVespaMallocEnvVariable() {
+        return "".equals(getNoVespaMalloc())
+                ? ""
+                : "VESPA_USE_NO_VESPAMALLOC=\"" + getNoVespaMalloc() + "\" ";
+    }
+    public String getVespaMallocEnvVariable() {
+        return "".equals(getVespaMalloc())
+                ? ""
+                : "VESPA_USE_VESPAMALLOC=\"" + getVespaMalloc() + "\" ";
+    }
+    public String getVespaMallocDebugEnvVariable() {
+        return "".equals(getVespaMallocDebug())
+                ? ""
+                : "VESPA_USE_VESPAMALLOC_D=\"" + getVespaMallocDebug() + "\" ";
+    }
+    public String getVespaMallocDebugStackTraceEnvVariable() {
+        return "".equals(getVespaMallocDebugStackTrace())
+                ? ""
+                : "VESPA_USE_VESPAMALLOC_DST=\"" + getVespaMallocDebugStackTrace() + "\" ";
+    }
+
+    public String getEnvVariables() {
+        return getCoreOnOOMEnvVariable() + getMMapNoCoreEnvVariable() + getNoVespaMallocEnvVariable() +
+                getVespaMallocEnvVariable() + getVespaMallocDebugEnvVariable() + getVespaMallocDebugStackTraceEnvVariable();
     }
 
     /**
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
index d1261c1e578..e24fae3e3b5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
@@ -38,9 +38,13 @@ import java.util.logging.Logger;
 public class VespaDomBuilder extends VespaModelBuilder {
 
     public static final String JVMARGS_ATTRIB_NAME = "jvmargs";
-    public static final String PRELOAD_ATTRIB_NAME = "preload";
-    public static final String MMAP_NOCORE_LIMIT = "mmap-core-limit";
-    public static final String CORE_ON_OOM = "core-on-oom";
+    public static final String PRELOAD_ATTRIB_NAME = "preload";        // Intended for vespa engineers
+    public static final String MMAP_NOCORE_LIMIT = "mmap-core-limit";  // Intended for vespa engineers
+    public static final String CORE_ON_OOM = "core-on-oom";            // Intended for vespa engineers
+    public static final String NO_VESPAMALLOC = "no-vespamalloc";      // Intended for vespa engineers
+    public static final String VESPAMALLOC = "vespamalloc";            // Intended for vespa engineers
+    public static final String VESPAMALLOC_DEBUG = "vespamalloc-debug"; // Intended for vespa engineers
+    public static final String VESPAMALLOC_DEBUG_STACKTRACE = "vespamalloc-debug-stacktrace"; // Intended for vespa engineers
     private static final String CPU_SOCKET_ATTRIB_NAME = "cpu-socket";
     public static final String CPU_SOCKET_AFFINITY_ATTRIB_NAME = "cpu-socket-affinity";
     public static final String Allocated_MEMORY_ATTRIB_NAME = "allocated-memory";
@@ -146,6 +150,18 @@ public class VespaDomBuilder extends VespaModelBuilder {
                 if (producerSpec.hasAttribute(CORE_ON_OOM)) {
                     t.setCoreOnOOM(Boolean.parseBoolean(producerSpec.getAttribute(CORE_ON_OOM)));
                 }
+                if (producerSpec.hasAttribute(NO_VESPAMALLOC)) {
+                    t.setNoVespaMalloc(producerSpec.getAttribute(NO_VESPAMALLOC));
+                }
+                if (producerSpec.hasAttribute(VESPAMALLOC)) {
+                    t.setVespaMalloc(producerSpec.getAttribute(VESPAMALLOC));
+                }
+                if (producerSpec.hasAttribute(VESPAMALLOC_DEBUG)) {
+                    t.setVespaMallocDebug(producerSpec.getAttribute(VESPAMALLOC_DEBUG));
+                }
+                if (producerSpec.hasAttribute(VESPAMALLOC_DEBUG_STACKTRACE)) {
+                    t.setVespaMallocDebugStackTrace(producerSpec.getAttribute(VESPAMALLOC_DEBUG_STACKTRACE));
+                }
                 if (producerSpec.hasAttribute(CPU_SOCKET_ATTRIB_NAME)) {
                     t.setAffinity(new Affinity.Builder().cpuSocket(Integer.parseInt(producerSpec.getAttribute(CPU_SOCKET_ATTRIB_NAME))).build());
                 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
index d6e34f66b44..7625374b0a0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
@@ -171,6 +171,26 @@ public class Content extends ConfigModel {
                 s.setCoreOnOOM(cluster.getRootGroup().getCoreOnOOM().get());
             }
         }
+        if (cluster.getRootGroup().getNoVespaMalloc().isPresent()) {
+            for (AbstractService s : cluster.getSearch().getSearchNodes()) {
+                s.setNoVespaMalloc(cluster.getRootGroup().getNoVespaMalloc().get());
+            }
+        }
+        if (cluster.getRootGroup().getVespaMalloc().isPresent()) {
+            for (AbstractService s : cluster.getSearch().getSearchNodes()) {
+                s.setVespaMalloc(cluster.getRootGroup().getVespaMalloc().get());
+            }
+        }
+        if (cluster.getRootGroup().getVespaMallocDebug().isPresent()) {
+            for (AbstractService s : cluster.getSearch().getSearchNodes()) {
+                s.setVespaMallocDebug(cluster.getRootGroup().getVespaMallocDebug().get());
+            }
+        }
+        if (cluster.getRootGroup().getVespaMallocDebugStackTrace().isPresent()) {
+            for (AbstractService s : cluster.getSearch().getSearchNodes()) {
+                s.setVespaMallocDebugStackTrace(cluster.getRootGroup().getVespaMallocDebugStackTrace().get());
+            }
+        }
         cluster.prepare();
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
index d720ea41a98..367b6c03968 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
@@ -37,6 +37,10 @@ public class StorageGroup {
     private final ContentCluster owner;
     private final Optional<Long> mmapNoCoreLimit;
     private final Optional<Boolean> coreOnOOM;
+    private final Optional<String> noVespaMalloc;
+    private final Optional<String> vespaMalloc;
+    private final Optional<String> vespaMallocDebug;
+    private final Optional<String> vespaMallocDebugStackTrace;
 
     private final List<StorageGroup> subgroups = new ArrayList<>();
     private final List<StorageNode> nodes = new ArrayList<>();
@@ -53,7 +57,9 @@ public class StorageGroup {
      * @param useCpuSocketAffinity whether processes should be started with socket affinity
      */
     private StorageGroup(ContentCluster owner, String name, String index, Optional<String> partitions,
-                         boolean useCpuSocketAffinity, Optional<Long> mmapNoCoreLimit, Optional<Boolean> coreOnOOM)
+                         boolean useCpuSocketAffinity, Optional<Long> mmapNoCoreLimit, Optional<Boolean> coreOnOOM,
+                         Optional<String> noVespaMalloc, Optional<String> vespaMalloc,
+                         Optional<String> vespaMallocDebug, Optional<String> vespaMallocDebugStackTrace)
     {
         this.owner = owner;
         this.index = index;
@@ -62,6 +68,14 @@ public class StorageGroup {
         this.useCpuSocketAffinity = useCpuSocketAffinity;
         this.mmapNoCoreLimit = mmapNoCoreLimit;
         this.coreOnOOM = coreOnOOM;
+        this.noVespaMalloc = noVespaMalloc;
+        this.vespaMalloc = vespaMalloc;
+        this.vespaMallocDebug = vespaMallocDebug;
+        this.vespaMallocDebugStackTrace = vespaMallocDebugStackTrace;
+    }
+    private StorageGroup(ContentCluster owner, String name, String index) {
+        this(owner, name, index, Optional.empty(), false, Optional.empty(),Optional.empty(), Optional.empty(),
+             Optional.empty(), Optional.empty(), Optional.empty());
     }
 
     /** Returns the name of this group, or null if it is the root group */
@@ -82,6 +96,10 @@ public class StorageGroup {
     public boolean useCpuSocketAffinity() { return useCpuSocketAffinity; }
     public Optional<Long> getMmapNoCoreLimit() { return mmapNoCoreLimit; }
     public Optional<Boolean> getCoreOnOOM() { return coreOnOOM; }
+    public Optional<String> getNoVespaMalloc() { return noVespaMalloc; }
+    public Optional<String> getVespaMalloc() { return vespaMalloc; }
+    public Optional<String> getVespaMallocDebug() { return vespaMallocDebug; }
+    public Optional<String> getVespaMallocDebugStackTrace() { return vespaMallocDebugStackTrace; }
 
     /** Returns all the nodes below this group */
     public List<StorageNode> recursiveGetNodes() {
@@ -272,8 +290,7 @@ public class StorageGroup {
                     // create subgroups as returned from allocation
                     for (Map.Entry<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroup : hostGroups.entrySet()) {
                         String groupIndex = String.valueOf(hostGroup.getKey().get().index());
-                        StorageGroup subgroup = new StorageGroup(owner, groupIndex, groupIndex, Optional.empty(),
-                                                                 false, Optional.empty(),Optional.empty());
+                        StorageGroup subgroup = new StorageGroup(owner, groupIndex, groupIndex);
                         for (Map.Entry<HostResource, ClusterMembership> host : hostGroup.getValue().entrySet()) {
                             subgroup.nodes.add(createStorageNode(owner, host.getKey(), subgroup, host.getValue()));
                         }
@@ -350,11 +367,16 @@ public class StorageGroup {
          * </ul>
          */
         private GroupBuilder collectGroup(Optional<ModelElement> groupElement, Optional<ModelElement> nodesElement, String name, String index) {
-            StorageGroup group = new StorageGroup(owner, name, index,
-                                                  childAsString(groupElement, "distribution.partitions"),
-                                                  booleanAttributeOr(groupElement, VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME, false),
-                                                  childAsLong(groupElement, VespaDomBuilder.MMAP_NOCORE_LIMIT),
-                                                  childAsBoolean(groupElement, VespaDomBuilder.CORE_ON_OOM));
+            StorageGroup group = new StorageGroup(
+                    owner, name, index,
+                    childAsString(groupElement, "distribution.partitions"),
+                    booleanAttributeOr(groupElement, VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME, false),
+                    childAsLong(groupElement, VespaDomBuilder.MMAP_NOCORE_LIMIT),
+                    childAsBoolean(groupElement, VespaDomBuilder.CORE_ON_OOM),
+                    childAsString(groupElement, VespaDomBuilder.NO_VESPAMALLOC),
+                    childAsString(groupElement, VespaDomBuilder.VESPAMALLOC),
+                    childAsString(groupElement, VespaDomBuilder.VESPAMALLOC_DEBUG),
+                    childAsString(groupElement, VespaDomBuilder.VESPAMALLOC_DEBUG_STACKTRACE));
 
             List<GroupBuilder> subGroups = groupElement.isPresent() ? collectSubGroups(group, groupElement.get()) : Collections.emptyList();
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 22f3b8fa2a0..53fdad64715 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -229,19 +229,9 @@ public class SearchNode extends AbstractService implements
         return serviceLayerService;
     }
 
-    public String getMMapNoCoreEnvVariable() {
-        return (getMMapNoCoreLimit() >= 0l)
-                ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " "
-                : "";
-    }
-
-    public String getCoreOnOOMEnvVariable() {
-        return getCoreOnOOM() ? "" : "VESPA_SILENCE_CORE_ON_OOM=true ";
-    }
-
     @Override
     public String getStartupCommand() {
-        String startup = getCoreOnOOMEnvVariable() + getMMapNoCoreEnvVariable() + "exec $ROOT/sbin/proton " + "--identity " + getConfigId();
+        String startup = getEnvVariables() + "exec $ROOT/sbin/proton " + "--identity " + getConfigId();
         if (serviceLayerService != null) {
             startup = startup + " --serviceidentity " + serviceLayerService.getConfigId();
         }
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index 110bcc5e3bc..30b931053d5 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -197,6 +197,10 @@ ContentNode = element node {
     attribute capacity { xsd:double { minExclusive = "0.0" } }? &
     attribute mmap-core-limit { xsd:nonNegativeInteger }? &
     attribute core-on-oom { xsd:boolean }? &
+    attribute no-vespamalloc { xsd:string }? &
+    attribute vespamalloc { xsd:string }? &
+    attribute vespamalloc-debug { xsd:string }? &
+    attribute vespamalloc-debug-stacktrace { xsd:string }? &
     attribute cpu-socket { xsd:nonNegativeInteger }?
 }
 
@@ -204,6 +208,10 @@ ContentNodes = element nodes {
     attribute cpu-socket-affinity { xsd:string }? &
     attribute mmap-core-limit { xsd:nonNegativeInteger }? &
     attribute core-on-oom { xsd:boolean }? &
+    attribute no-vespamalloc { xsd:string }? &
+    attribute vespamalloc { xsd:string }? &
+    attribute vespamalloc-debug { xsd:string }? &
+    attribute vespamalloc-debug-stacktrace { xsd:string }? &
     (
       (
         attribute count { xsd:positiveInteger } &
@@ -225,6 +233,10 @@ TopGroup = element group {
     attribute cpu-socket-affinity { xsd:string }? &
     attribute mmap-core-limit { xsd:nonNegativeInteger }? &
     attribute core-on-oom { xsd:boolean }? &
+    attribute no-vespamalloc { xsd:string }? &
+    attribute vespamalloc { xsd:string }? &
+    attribute vespamalloc-debug { xsd:string }? &
+    attribute vespamalloc-debug-stacktrace { xsd:string }? &
     attribute distribution-key { xsd:nonNegativeInteger }? &
     (
         ContentNode +
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index f097854efde..138bf21b6c6 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -456,15 +456,15 @@ public class ContentBuilderTest extends DomBuilderTest {
     public void canConfigureMmapNoCoreLimit() throws Exception {
         ContentCluster b = createContent(
                 "<content version =\"1.0\" id=\"b\">" +
-                        "    <redundancy>2</redundancy>" +
-                        "      <documents>" +
-                        "        <document type='music' mode='index'/>" +
-                        "      </documents>" +
-                        "    <group mmap-core-limit=\"200000\">" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
-                        "    </group>" +
-                        "</content>");
+                "    <redundancy>2</redundancy>" +
+                "      <documents>" +
+                "        <document type='music' mode='index'/>" +
+                "      </documents>" +
+                "    <group mmap-core-limit=\"200000\">" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
+                "    </group>" +
+                "</content>");
         ContentSearchCluster s;
 
         s = b.getSearch();
@@ -485,15 +485,15 @@ public class ContentBuilderTest extends DomBuilderTest {
     public void canConfigureCoreOnOOM() throws Exception {
         ContentCluster b = createContent(
                 "<content version =\"1.0\" id=\"b\">" +
-                        "    <redundancy>2</redundancy>" +
-                        "      <documents>" +
-                        "        <document type='music' mode='index'/>" +
-                        "      </documents>" +
-                        "    <group core-on-oom=\"true\">" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
-                        "    </group>" +
-                        "</content>");
+                "    <redundancy>2</redundancy>" +
+                "      <documents>" +
+                "        <document type='music' mode='index'/>" +
+                "      </documents>" +
+                "    <group core-on-oom=\"true\">" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
+                "    </group>" +
+                "</content>");
         ContentSearchCluster s;
 
         s = b.getSearch();
@@ -514,15 +514,15 @@ public class ContentBuilderTest extends DomBuilderTest {
     public void defaultCoreOnOOMIsFalse() throws Exception {
         ContentCluster b = createContent(
                 "<content version =\"1.0\" id=\"b\">" +
-                        "    <redundancy>2</redundancy>" +
-                        "      <documents>" +
-                        "        <document type='music' mode='index'/>" +
-                        "      </documents>" +
-                        "    <group>" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
-                        "    </group>" +
-                        "</content>");
+                "    <redundancy>2</redundancy>" +
+                "      <documents>" +
+                "        <document type='music' mode='index'/>" +
+                "      </documents>" +
+                "    <group>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"0\" />" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
+                "    </group>" +
+                "</content>");
         ContentSearchCluster s = b.getSearch();
         assertTrue(s.hasIndexedCluster());
         assertNotNull(s.getIndexed());
@@ -540,15 +540,15 @@ public class ContentBuilderTest extends DomBuilderTest {
     public void canConfigureMmapNoCoreLimitPerHost() throws Exception {
         ContentCluster b = createContent(
                 "<content version =\"1.0\" id=\"b\">" +
-                        "    <redundancy>2</redundancy>" +
-                        "      <documents>" +
-                        "        <document type='music' mode='index'/>" +
-                        "      </documents>" +
-                        "    <group>" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"0\"  mmap-core-limit=\"200000\"/>" +
-                        "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
-                        "    </group>" +
-                        "</content>");
+                "    <redundancy>2</redundancy>" +
+                "      <documents>" +
+                "        <document type='music' mode='index'/>" +
+                "      </documents>" +
+                "    <group>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"0\"  mmap-core-limit=\"200000\"/>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"1\" />" +
+                "    </group>" +
+                "</content>");
         ContentSearchCluster s = b.getSearch();
         assertTrue(s.hasIndexedCluster());
         assertNotNull(s.getIndexed());
@@ -589,6 +589,79 @@ public class ContentBuilderTest extends DomBuilderTest {
     }
 
     @Test
+    public void canConfigureVespaMalloc() throws Exception {
+        ContentCluster b = createContent(
+                "<content version =\"1.0\" id=\"b\">" +
+                "    <redundancy>2</redundancy>" +
+                "      <documents>" +
+                "        <document type='music' mode='index'/>" +
+                "      </documents>" +
+                "    <group no-vespamalloc=\"proton\" vespamalloc-debug=\"distributord\" vespamalloc-debug-stacktrace=\"all\" vespamalloc=\"storaged\">" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"0\"/>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"1\"/>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"2\"/>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"3\"/>" +
+                "    </group>" +
+                "</content>");
+        ContentSearchCluster s = b.getSearch();
+        assertTrue(s.hasIndexedCluster());
+        assertNotNull(s.getIndexed());
+        assertEquals(4, b.getStorageNodes().getChildren().size());
+        assertTrue(b.getRootGroup().getNoVespaMalloc().isPresent());
+        assertEquals("proton", b.getRootGroup().getNoVespaMalloc().get());
+        assertTrue(b.getRootGroup().getVespaMalloc().isPresent());
+        assertEquals("storaged", b.getRootGroup().getVespaMalloc().get());
+        assertTrue(b.getRootGroup().getVespaMallocDebug().isPresent());
+        assertEquals("distributord", b.getRootGroup().getVespaMallocDebug().get());
+        assertTrue(b.getRootGroup().getVespaMallocDebugStackTrace().isPresent());
+        assertEquals("all", b.getRootGroup().getVespaMallocDebugStackTrace().get());
+
+        assertThat(s.getSearchNodes().size(), is(4));
+        for (SearchNode n : s.getSearchNodes()) {
+            assertEquals("proton", n.getNoVespaMalloc());
+            assertEquals("VESPA_USE_NO_VESPAMALLOC=\"proton\" ", n.getNoVespaMallocEnvVariable());
+            assertEquals("distributord", n.getVespaMallocDebug());
+            assertEquals("VESPA_USE_VESPAMALLOC=\"storaged\" ", n.getVespaMallocEnvVariable());
+            assertEquals("all", n.getVespaMallocDebugStackTrace());
+            assertEquals("VESPA_USE_VESPAMALLOC_D=\"distributord\" ", n.getVespaMallocDebugEnvVariable());
+            assertEquals("storaged", n.getVespaMalloc());
+            assertEquals("VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getVespaMallocDebugStackTraceEnvVariable());
+            assertEquals("VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\" VESPA_USE_VESPAMALLOC=\"storaged\" VESPA_USE_VESPAMALLOC_D=\"distributord\" VESPA_USE_VESPAMALLOC_DST=\"all\" ", n.getEnvVariables());
+        }
+    }
+
+    @Test
+    public void canConfigureVespaMallocPerHost() throws Exception {
+        ContentCluster b = createContent(
+                "<content version =\"1.0\" id=\"b\">" +
+                "    <redundancy>2</redundancy>" +
+                "      <documents>" +
+                "        <document type='music' mode='index'/>" +
+                "      </documents>" +
+                "    <group>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"0\" no-vespamalloc=\"proton\"/>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"1\" vespamalloc-debug=\"distributord\"/>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"2\" vespamalloc-debug-stacktrace=\"all\"/>" +
+                "      <node hostalias=\"mockhost\" distribution-key=\"3\" vespamalloc=\"storaged\"/>" +
+                "    </group>" +
+                "</content>");
+        ContentSearchCluster s = b.getSearch();
+        assertTrue(s.hasIndexedCluster());
+        assertNotNull(s.getIndexed());
+        assertEquals(4, b.getStorageNodes().getChildren().size());
+        assertFalse(b.getRootGroup().getNoVespaMalloc().isPresent());
+        assertFalse(b.getRootGroup().getVespaMalloc().isPresent());
+        assertFalse(b.getRootGroup().getVespaMallocDebug().isPresent());
+        assertFalse(b.getRootGroup().getVespaMallocDebugStackTrace().isPresent());
+
+        assertThat(s.getSearchNodes().size(), is(4));
+        assertEquals("VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_NO_VESPAMALLOC=\"proton\" ", s.getSearchNodes().get(0).getEnvVariables());
+        assertEquals("VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_D=\"distributord\" ", s.getSearchNodes().get(1).getEnvVariables());
+        assertEquals("VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC_DST=\"all\" ", s.getSearchNodes().get(2).getEnvVariables());
+        assertEquals("VESPA_SILENCE_CORE_ON_OOM=true VESPA_USE_VESPAMALLOC=\"storaged\" ", s.getSearchNodes().get(3).getEnvVariables());
+    }
+
+    @Test
     public void canConfigureCpuAffinity() throws Exception
     {
         ContentCluster b = createContent(
@@ -839,15 +912,16 @@ public class ContentBuilderTest extends DomBuilderTest {
     @Test
     public void ensurePruneRemovedDocumentsAgeForHostedVespa() throws Exception {
         {
-            ContentCluster contentNonHosted = createContent("<content version='1.0' id='search'>" +
-                                                                           "  <redundancy>1</redundancy>" +
-                                                                           "  <documents>" +
-                                                                           "    <document type='music' mode='index'/>" +
-                                                                           "  </documents>" +
-                                                                           "  <nodes>" +
-                                                                           "    <node hostalias='mockhost' distribution-key='0'/>" +
-                                                                           "  </nodes>" +
-                                                                           "</content>");
+            ContentCluster contentNonHosted = createContent(
+                    "<content version='1.0' id='search'>" +
+                    "  <redundancy>1</redundancy>" +
+                    "  <documents>" +
+                    "    <document type='music' mode='index'/>" +
+                    "  </documents>" +
+                    "  <nodes>" +
+                    "    <node hostalias='mockhost' distribution-key='0'/>" +
+                    "  </nodes>" +
+                    "</content>");
             ProtonConfig configNonHosted = getProtonConfig(contentNonHosted);
             ProtonConfig defaultConfig = new ProtonConfig(new ProtonConfig.Builder());
             assertEquals(defaultConfig.pruneremoveddocumentsage(), configNonHosted.pruneremoveddocumentsage(), 0.001);