Merge pull request #497 from yahoo/bratseth/nodes-memory

Add allocated-memory atteribute to the nodes tag

7 files changed, 62 insertions, 10 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
index 08c3db81091..942c972a782 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
@@ -42,6 +42,7 @@ public class VespaDomBuilder extends VespaModelBuilder {
     public static final String MMAP_NOCORE_LIMIT="mmap-nocore-limit";
     private static final String CPU_SOCKET_ATTRIB_NAME = "cpu-socket";
     public static final String CPU_SOCKET_AFFINITY_ATTRIB_NAME = "cpu-socket-affinity";
+    public static final String Allocated_MEMORY_ATTRIB_NAME = "allocated-memory";
 
     public static final Logger log = Logger.getLogger(VespaDomBuilder.class.getPackage().toString());
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index bb852d297a3..268f7a87f2c 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -173,6 +173,7 @@ public final class ContainerCluster
     private Zone zone;
     
     private Optional<String> hostClusterId = Optional.empty();
+    private Optional<Integer> memoryPercentage = Optional.empty();
 
     public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name) {
         super(parent, subId);
@@ -791,6 +792,14 @@ public final class ContainerCluster
      */
     public Optional<String> getHostClusterId() { return hostClusterId; }
 
+    public void setMemoryPercentage(Optional<Integer> memoryPercentage) { this.memoryPercentage = memoryPercentage; }
+
+    /** 
+     * Returns the percentage of host physical memory this application has specified for nodes in this cluster,
+     * or empty if this is not specified by the application.
+     */
+    public Optional<Integer> getMemoryPercentage() { return memoryPercentage; }
+
     @Override
     public String toString() {
         return "container cluster '" + getName() + "'";
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index 2e27dda5a62..a473514a344 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -38,10 +38,11 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
         SemanticRulesConfig.Producer,
     	PageTemplatesConfig.Producer
 {
+
     private final List<AbstractSearchCluster> systems = new LinkedList<>();
     private Options options = null;
 
-    //For legacy qrs clusters only.
+    // For legacy qrs clusters only.
     private BinaryScaledAmount totalCacheSize = new BinaryScaledAmount();
 
     private QueryProfiles queryProfiles;
@@ -49,12 +50,14 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
     private PageTemplates pageTemplates;
     private final boolean hostedVespa;
     private final boolean isCombinedCluster;
+    private final Optional<Integer> memoryPercentage;
 
     public ContainerSearch(ContainerCluster cluster, SearchChains chains, Options options) {
         super(chains);
         this.options = options;
         this.hostedVespa  = cluster.isHostedVespa();
         this.isCombinedCluster = cluster.getHostClusterId().isPresent();
+        this.memoryPercentage = cluster.getMemoryPercentage();
         cluster.addComponent(getFS4ResourcePool());
     }
 
@@ -122,7 +125,10 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
     @Override
     public void getConfig(QrStartConfig.Builder qsB) {
     	QrStartConfig.Jvm.Builder internalBuilder = new QrStartConfig.Jvm.Builder();
-    	if (hostedVespa) {
+        if (memoryPercentage.isPresent()) {
+            internalBuilder.heapSizeAsPercentageOfPhysicalMemory(memoryPercentage.get());
+        }
+    	else if (hostedVespa) {
             if (isCombinedCluster)
                 internalBuilder.heapSizeAsPercentageOfPhysicalMemory(17);
             else
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 8c4a451bb13..6d83c614044 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -387,6 +387,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
             applyNodesTagJvmArgs(nodes, nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME));
             applyRoutingAliasProperties(nodes, cluster);
             applyDefaultPreload(nodes, nodesElement);
+            applyMemoryPercentage(cluster, nodesElement.getAttribute(VespaDomBuilder.Allocated_MEMORY_ATTRIB_NAME));
             if (useCpuSocketAffinity(nodesElement))
                 AbstractService.distributeCpuSocketAffinity(nodes);
 
@@ -416,6 +417,24 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         }
     }
     
+    private void applyMemoryPercentage(ContainerCluster cluster, String memoryPercentage) {
+        if (memoryPercentage == null || memoryPercentage.isEmpty()) return;
+        memoryPercentage = memoryPercentage.trim();
+
+        if ( ! memoryPercentage.endsWith("%"))
+            throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster +
+                                               " must be an integer percentage ending by the '%' sign");
+        memoryPercentage = memoryPercentage.substring(0, memoryPercentage.length()-1).trim();
+
+        try {
+            cluster.setMemoryPercentage(Optional.of(Integer.parseInt(memoryPercentage)));
+        }
+        catch (NumberFormatException e) {
+            throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster +
+                                               " must be an integer percentage ending by the '%' sign");
+        }
+    }
+    
     private HostResource allocateSingleNodeHost(ContainerCluster cluster, DeployLogger logger) {
         if (cluster.isHostedVespa()) {
             ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty());
diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc
index 287fd9f459e..e878b0faf22 100644
--- a/config-model/src/main/resources/schema/containercluster.rnc
+++ b/config-model/src/main/resources/schema/containercluster.rnc
@@ -184,6 +184,7 @@ DocumentApi = element document-api {
 NodesOfContainerCluster = element nodes {
     attribute jvmargs { text }? &
     attribute preload { text }? &
+    attribute allocated-memory { text }? &
     attribute cpu-socket-affinity { xsd:boolean }? &
     (
       attribute of { xsd:string }
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index f2b235a29da..86fabdf26bc 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -7,6 +7,7 @@ import static org.junit.Assert.*;
 import java.io.StringReader;
 import java.util.Collection;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -57,7 +58,7 @@ public class ModelProvisioningTest {
                         "  <handler id='myHandler'>" +
                         "    <component id='injected' />" +
                         "  </handler>" +
-                        "  <nodes count='2' jvmargs='-verbosegc' preload='lib/blablamalloc.so'/>" +
+                        "  <nodes count='2' allocated-memory='45%' jvmargs='-verbosegc' preload='lib/blablamalloc.so'/>" +
                         "</jdisc>" +
                         "</services>";
         String hosts ="<hosts>"
@@ -102,12 +103,14 @@ public class ModelProvisioningTest {
         assertThat(model.getContainerClusters().get("mydisc").getContainers().get(0).getPreLoad(), is(Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so"));
         assertThat(model.getContainerClusters().get("mydisc").getContainers().get(1).getPreLoad(), is(Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so"));
         assertThat(model.getContainerClusters().get("mydisc").getContainers().get(2).getPreLoad(), is(Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so"));
+        assertThat(model.getContainerClusters().get("mydisc").getMemoryPercentage(), is(Optional.empty()));
 
         assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getJvmArgs(), is("-verbosegc"));
         assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getJvmArgs(), is("-verbosegc"));
         assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(0).getPreLoad(), is("lib/blablamalloc.so"));
         assertThat(model.getContainerClusters().get("mydisc2").getContainers().get(1).getPreLoad(), is("lib/blablamalloc.so"));
-
+        assertThat(model.getContainerClusters().get("mydisc2").getMemoryPercentage(), is(Optional.of(45)));
+        
         HostSystem hostSystem = model.getHostSystem();
         assertNotNull(hostSystem.getHostByHostname("myhost0"));
         assertNotNull(hostSystem.getHostByHostname("myhost1"));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index 5efbfe0d0dc..ef99d1a6370 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -23,6 +23,7 @@ import org.junit.Test;
 
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -82,29 +83,41 @@ public class ContainerClusterTest {
     }
 
     private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster) {
+        return createContainerCluster(isHosted, isCombinedCluster, Optional.empty());
+    }
+    private ContainerCluster createContainerCluster(boolean isHosted, boolean isCombinedCluster, 
+                                                    Optional<Integer> memoryPercentage) {
         DeployState state = new DeployState.Builder().properties(new DeployProperties.Builder().hostedVespa(isHosted).build()).build();
         MockRoot root = new MockRoot("foo", state);
         ContainerCluster cluster = new ContainerCluster(root, "container0", "container1");
         if (isCombinedCluster)
             cluster.setHostClusterId("test-content-cluster");
+        cluster.setMemoryPercentage(memoryPercentage);
         cluster.setSearch(new ContainerSearch(cluster, new SearchChains(cluster, "search-chain"), new ContainerSearch.Options()));
         return cluster;
     }
-    private void verifyHeapSizeAsPercentageOfPhysicalMemory(boolean isHosted, boolean isCombinedCluster, int percentage) {
-        ContainerCluster cluster = createContainerCluster(isHosted, isCombinedCluster);
+    private void verifyHeapSizeAsPercentageOfPhysicalMemory(boolean isHosted, boolean isCombinedCluster, 
+                                                            Optional<Integer> explicitMemoryPercentage, 
+                                                            int expectedMemoryPercentage) {
+        ContainerCluster cluster = createContainerCluster(isHosted, isCombinedCluster, explicitMemoryPercentage);
         QrStartConfig.Builder qsB = new QrStartConfig.Builder();
         cluster.getSearch().getConfig(qsB);
         QrStartConfig qsC= new QrStartConfig(qsB);
-        assertEquals(percentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory());
+        assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory());
     }
 
     @Test
     public void requireThatHeapSizeAsPercentageOfPhysicalMemoryForHostedAndNot() {
         boolean hosted = true;
         boolean combined = true; // a cluster running on content nodes (only relevant with hosted)
-        verifyHeapSizeAsPercentageOfPhysicalMemory(  hosted, ! combined, 33);
-        verifyHeapSizeAsPercentageOfPhysicalMemory(  hosted,   combined, 17);
-        verifyHeapSizeAsPercentageOfPhysicalMemory(! hosted, ! combined,  0);
+        verifyHeapSizeAsPercentageOfPhysicalMemory(  hosted, ! combined, Optional.empty(), 33);
+        verifyHeapSizeAsPercentageOfPhysicalMemory(  hosted,   combined, Optional.empty(), 17);
+        verifyHeapSizeAsPercentageOfPhysicalMemory(! hosted, ! combined, Optional.empty(), 0);
+        
+        // Explicit value overrides all defaults
+        verifyHeapSizeAsPercentageOfPhysicalMemory(  hosted, ! combined, Optional.of(67), 67);
+        verifyHeapSizeAsPercentageOfPhysicalMemory(  hosted,   combined, Optional.of(68), 68);
+        verifyHeapSizeAsPercentageOfPhysicalMemory(! hosted, ! combined, Optional.of(69), 69);
     }
 
     private void verifyJvmArgs(boolean isHosted, boolean hasDocproc, String expectedArgs, String jvmArgs) {