Merge pull request #785 from yahoo/bratseth/respect-zone-specific-capacity

Bratseth/respect zone specific capacity

25 files changed, 231 insertions, 110 deletions

diff --git a/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java
index cd68d214d3d..4268e4f835e 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/IncludeProcessor.java
@@ -41,7 +41,7 @@ class IncludeProcessor implements PreProcessor {
             Element elem = (Element) list.item(0);
             Element parent = (Element) elem.getParentNode();
             String filename = elem.getAttribute("file");
-            boolean required = elem.hasAttribute("required") ? Boolean.parseBoolean(elem.getAttribute("required")) : true;
+            boolean required = ! elem.hasAttribute("required") || Boolean.parseBoolean(elem.getAttribute("required"));
             File file = new File(currentFolder, filename);
 
             Document subFile = IncludeProcessor.parseIncludeFile(file, parent.getTagName(), required);
@@ -76,4 +76,5 @@ class IncludeProcessor implements PreProcessor {
         w.append(endTag);
         return XML.getDocument(new StringReader(w.toString()));
     }
+
 }
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
index f3da285f524..32e9aec56cb 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
@@ -20,6 +20,7 @@ import java.util.logging.Logger;
  * @since 5.22
  */
 class OverrideProcessor implements PreProcessor {
+
     private static final Logger log = Logger.getLogger(OverrideProcessor.class.getName());
 
     private final Environment environment;
@@ -140,6 +141,9 @@ class OverrideProcessor implements PreProcessor {
             }
         }
 
+        if (bestMatch > 1) // there was a region/environment specific overriode
+            doElementSpecificProcessingOnOverride(bestMatchElement);
+
         // Remove elements not specific
         for (Element child : children) {
             if (child != bestMatchElement) {
@@ -148,6 +152,14 @@ class OverrideProcessor implements PreProcessor {
         }
     }
 
+    /** Called on each element which is selected by matching some override condition */
+    private void doElementSpecificProcessingOnOverride(Element element) {
+        // if node capacity is specified explicitly for some evn/region we should require that capacity
+        if ( element.getTagName().equals("nodes"))
+            if (element.getChildNodes().getLength() == 0) // specifies capacity, not a list of nodes
+                element.setAttribute("required", "true");
+    }
+    
     /**
      * Retains all elements where at least one element is overridden. Removes non-overridden elements from map.
      */
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
index 4e08e514504..b70a5054563 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
@@ -25,6 +25,7 @@ import java.util.List;
  * @since 5.22
  */
 public class XmlPreProcessor {
+
     final static String deployNamespace = "xmlns:deploy";
     final static String deployNamespaceUri = "vespa";
     final static String preprocessNamespace = "xmlns:preprocess";
@@ -68,4 +69,5 @@ public class XmlPreProcessor {
         chain.add(new PropertiesProcessor());
         return chain;
     }
+
 }
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
index 912b4ad2707..002c31d5910 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
@@ -671,10 +671,10 @@ public class FilesApplicationPackage implements ApplicationPackage {
     @Override
     public ApplicationPackage preprocess(Zone zone, RuleConfigDeriver ignored, DeployLogger logger) throws IOException, TransformerException, ParserConfigurationException, SAXException {
         IOUtils.recursiveDeleteDir(preprocessedDir);
-        IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> !name.equals(".preprocessed") &&
-                !name.equals(SERVICES) &&
-                !name.equals(HOSTS) &&
-                !name.equals(CONFIG_DEFINITIONS_DIR));
+        IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(".preprocessed") &&
+                                                                          ! name.equals(SERVICES) &&
+                                                                          ! name.equals(HOSTS) &&
+                                                                          ! name.equals(CONFIG_DEFINITIONS_DIR));
         preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone);
         if (getHostsFile().exists()) {
             preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone);
diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java
new file mode 100644
index 00000000000..338302e9e57
--- /dev/null
+++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorTest.java
@@ -0,0 +1,128 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.application;
+
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import org.custommonkey.xmlunit.XMLUnit;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.transform.TransformerException;
+import java.io.IOException;
+import java.io.StringReader;
+
+/**
+ * @author bratseth
+ */
+public class HostedOverrideProcessorTest {
+
+    static {
+        XMLUnit.setIgnoreWhitespace(true);
+    }
+
+    private static final String input =
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='1'/>" +
+                "    <nodes deploy:environment=\"staging\" count='2'/>" +
+                "    <nodes deploy:environment=\"prod\" count='3'/>" +
+                "    <nodes deploy:environment=\"prod\" deploy:region=\"us-west\" count='4'/>" +
+                "  </container>" +
+                "</services>";
+
+
+    @Test
+    public void testParsingDefault() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException {
+        String expected = 
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='1'/>" +
+                "  </container>" +
+                "</services>";
+        assertOverride(Environment.test, RegionName.defaultName(), expected);
+    }
+
+    @Test
+    public void testParsingEnvironmentAndRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException {
+        String expected = 
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='4' required='true'/>" +
+                "  </container>" +
+                "</services>";
+        assertOverride(Environment.from("prod"), RegionName.from("us-west"), expected);
+    }
+
+    @Test
+    public void testParsingEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException {
+        String expected =
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='3' required='true'/>" +
+                "  </container>" +
+                "</services>";
+        assertOverride(Environment.valueOf("prod"), RegionName.from("us-east"), expected);
+    }
+
+    @Test
+    public void testParsingEnvironmentNoRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException {
+        String expected =
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='3' required='true'/>" +
+                "  </container>" +
+                "</services>";
+        assertOverride(Environment.from("prod"), RegionName.defaultName(), expected);
+    }
+
+    @Test
+    public void testParsingUnknownEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException {
+        String expected =
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='1'/>" +
+                "  </container>" +
+                "</services>";
+        assertOverride(Environment.from("dev"), RegionName.defaultName(), expected);
+    }
+
+    @Test
+    public void testParsingUnknownEnvironmentUnknownRegion() throws ParserConfigurationException, IOException, SAXException, TransformerException {
+        String expected =
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='1'/>" +
+                "  </container>" +
+                "</services>";
+        assertOverride(Environment.from("test"), RegionName.from("us-west"), expected);
+    }
+
+    @Test
+    public void testParsingInheritEnvironment() throws ParserConfigurationException, IOException, SAXException, TransformerException {
+        String expected =
+                "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" +
+                "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"?\" version=\"1.0\">" +
+                "  <container id=\"foo\" version=\"1.0\">" +
+                "    <nodes count='2' required='true'/>" +
+                "  </container>" +
+                "</services>";
+        assertOverride(Environment.from("staging"), RegionName.from("us-west"), expected);
+    }
+
+    private void assertOverride(Environment environment, RegionName region, String expected) throws TransformerException {
+        Document inputDoc = Xml.getDocument(new StringReader(input));
+        Document newDoc = new OverrideProcessor(environment, region).process(inputDoc);
+        TestBase.assertDocument(expected, newDoc);
+    }
+
+}
diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java
index 07068e236cd..6d9bf2cbfa5 100644
--- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java
+++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java
@@ -17,6 +17,7 @@ import java.nio.file.NoSuchFileException;
  * @since 5.22
  */
 public class IncludeProcessorTest {
+
     @Test
     public void testInclude() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException {
         File app = new File("src/test/resources/multienvapp");
@@ -68,7 +69,7 @@ public class IncludeProcessorTest {
                 "</jdisc></services>";
 
         Document doc = (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app)));
-        System.out.println(Xml.documentAsString(doc));
+        // System.out.println(Xml.documentAsString(doc));
         TestBase.assertDocument(expected, doc);
     }
 
@@ -78,4 +79,5 @@ public class IncludeProcessorTest {
         DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder();
         (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app)));
     }
+
 }
diff --git a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java
index eecbb1e7313..f6528e84368 100644
--- a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java
+++ b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java
@@ -83,7 +83,7 @@ public class XmlPreprocessorTest {
                 "</services>";
 
         Document docUsWest = (new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-west"))).run();
-        System.out.println(Xml.documentAsString(docUsWest));
+        // System.out.println(Xml.documentAsString(docUsWest));
         TestBase.assertDocument(expectedUsWest, docUsWest);
 
         String expectedUsEast = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" +
@@ -162,4 +162,5 @@ public class XmlPreprocessorTest {
         Document docDev = (new XmlPreProcessor(appDir, new StringReader(input), Environment.prod, RegionName.from("default")).run());
         TestBase.assertDocument(expectedProd, docDev);
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index 5c9d03b434f..c4ac4d91001 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -101,8 +101,9 @@ public class InMemoryProvisioner implements HostProvisioner {
             throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " +
                                                groups + " groups, but the node count is not divisible into this number of groups");
 
-        int capacity = failOnOutOfCapacity ? requestedCapacity.nodeCount() :
-                                             Math.min(requestedCapacity.nodeCount(), freeNodes.get("default").size() + totalAllocatedTo(cluster));
+        int capacity = failOnOutOfCapacity || requestedCapacity.isRequired() 
+                       ? requestedCapacity.nodeCount() 
+                       : Math.min(requestedCapacity.nodeCount(), freeNodes.get("default").size() + totalAllocatedTo(cluster));
         if (groups > capacity)
             groups = capacity;
 
@@ -138,7 +139,7 @@ public class InMemoryProvisioner implements HostProvisioner {
 
         int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex);
         while (allocation.size() < nodesInGroup) {
-            if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("No nodes of flavor '" + flavor + "' available");
+            if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("Insufficient capacity of flavor '" + flavor + "'");
             Host newHost = freeNodes.removeValue(flavor, 0);
             ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
             allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), membership));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Client.java b/config-model/src/main/java/com/yahoo/vespa/model/Client.java
index 15685f5f669..2a2498cc310 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/Client.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Client.java
@@ -8,7 +8,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer;
  * This is a placeholder config producer that makes global configuration available through a single identifier. This
  * is added directly to the {@link ApplicationConfigProducerRoot} producer, and so can be accessed by the simple "client" identifier.
  *
- * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a>
+ * @author Simon Thoresen
  */
 public class Client extends AbstractConfigProducer {
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
index 852e4e73331..aaeedf10bc8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
@@ -19,44 +19,38 @@ import com.yahoo.config.model.producer.UserConfigRepo;
  */
 public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer {
 
-    /**
-     * @return the configId of this ConfigProducer.
-     */
-    public String getConfigId();
+    /** Returns the configId of this ConfigProducer. */
+    String getConfigId();
 
-    /**
-     * @return  The one and only HostSystem of the root node
-     */
-    public HostSystem getHostSystem();
+    /** Returns the one and only HostSystem of the root node */
+    HostSystem getHostSystem();
 
     /** Returns the user configs of this */
-    public UserConfigRepo getUserConfigs();
+    UserConfigRepo getUserConfigs();
     
-    /**
-     * @return this ConfigProducer's children (only 1st level)
-     */
-    public Map<String,? extends ConfigProducer> getChildren();
+    /** Returns this ConfigProducer's children (only 1st level) */
+    Map<String,? extends ConfigProducer> getChildren();
 
-    /**
-     * @return a List of all Services that are descendants to this ConfigProducer
-     */
-    public List<Service> getDescendantServices();
+    /** Returns a List of all Services that are descendants to this ConfigProducer */
+    List<Service> getDescendantServices();
 
     /**
      * Writes files that need to be written.  The files will usually
      * only be written when the Vespa model is generated through the
      * deploy-application script.
-     * gv: This is primarily intended for debugging.
+     * This is primarily intended for debugging.
+     * 
      * @param directory directory to write files to
      * @throws java.io.IOException if writing fails
      */
-    public void writeFiles(File directory) throws IOException;
+    void writeFiles(File directory) throws IOException;
 
     /**
      * Dump the three of config producers to the specified stream.
+     * 
      * @param out The stream to print to, e.g. System.out
      */
-    public void dump(PrintStream out);
+    void dump(PrintStream out);
 
     /**
      * Build config from this and all parent ConfigProducers,
@@ -74,11 +68,12 @@ public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer
      * @param builder The ConfigBuilder to add user config overrides.
      * @return true if overrides were added, false if not.
      */
-    public boolean addUserConfig(ConfigInstance.Builder builder);
+    boolean addUserConfig(ConfigInstance.Builder builder);
 
     /**
      * check constraints depending on the state of the vespamodel graph.
      * When overriding, you must invoke super.
      */
-    public void validate() throws Exception;
+    void validate() throws Exception;
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java b/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java
deleted file mode 100644
index d424f4fa31b..00000000000
--- a/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.model;
-
-import java.util.logging.Formatter;
-import java.util.logging.LogRecord;
-
-/**
- * A log formatter that returns a plain log message only with level, not
- * including timestamp and method (as java.util.logging.SimpleFormatter).
- * See bug #1789867.
- *
- * @author gjoranv
- */
-public class PlainFormatter extends Formatter {
-
-    public PlainFormatter() {
-        super();
-    }
-
-    public String format(LogRecord record) {
-        StringBuffer sb = new StringBuffer();
-
-        sb.append(record.getLevel().getName()).append(": ");
-        sb.append(formatMessage(record)).append("\n");
-
-        return sb.toString();
-    }
-}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java
index ea2151f9976..a0b3cc7294b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java
@@ -7,11 +7,12 @@ import java.util.LinkedList;
 import java.util.List;
 
 /**
- * Track metainformation about the ports of a service.
+ * Track meta information about the ports of a service.
  *
  * @author Vidar Larsen
  */
 public class PortsMeta implements Serializable {
+
     /** A list of all ports. The list elements are lists of strings. */
     private List<LinkedList<String>> ports;
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java
index 75e9caefbd5..bcf523e1c99 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java
@@ -27,4 +27,5 @@ public abstract class VespaModelBuilder {
      * @param configModelRepo a {@link com.yahoo.config.model.ConfigModelRepo instance}
      */
     public abstract void postProc(AbstractConfigProducer producerRoot, ConfigModelRepo configModelRepo);
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java
index 876017e16bc..f1829a1d718 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java
@@ -35,4 +35,5 @@ public class DomClientsBuilder extends LegacyConfigModelBuilder<Clients> {
             throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported.");
         }
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
index ff37c0a95fd..b4070c67ae1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java
@@ -1,40 +1,17 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.builder.xml.dom;
 
-import com.yahoo.component.ComponentId;
-import com.yahoo.component.ComponentSpecification;
-import com.yahoo.component.chain.Phase;
-import com.yahoo.component.chain.dependencies.Dependencies;
-import com.yahoo.component.chain.model.ChainSpecification;
-import com.yahoo.component.chain.model.ChainedComponentModel;
-import com.yahoo.config.model.ConfigModelUtils;
 import com.yahoo.vespa.config.content.spooler.SpoolerConfig;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
-import com.yahoo.container.bundle.BundleInstantiationSpecification;
-import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.text.XML;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.model.SimpleConfigProducer;
 import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder;
-import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder;
 import com.yahoo.vespa.model.clients.Clients;
-import com.yahoo.vespa.model.clients.HttpGatewayOwner;
 import com.yahoo.vespa.model.clients.VespaSpoolMaster;
 import com.yahoo.vespa.model.clients.VespaSpooler;
 import com.yahoo.vespa.model.clients.VespaSpoolerProducer;
 import com.yahoo.vespa.model.clients.VespaSpoolerService;
-import com.yahoo.vespa.model.container.Container;
-import com.yahoo.vespa.model.container.ContainerCluster;
-import com.yahoo.vespa.model.container.component.Handler;
-import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
-import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
-import com.yahoo.vespa.model.container.docproc.DocprocChains;
-import com.yahoo.vespa.model.container.search.ContainerHttpGateway;
-import com.yahoo.vespa.model.container.search.ContainerSearch;
-import com.yahoo.vespa.model.container.search.searchchain.SearchChain;
-import com.yahoo.vespa.model.container.search.searchchain.SearchChains;
-import com.yahoo.vespa.model.container.search.searchchain.Searcher;
-import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
 import com.yahoo.vespaclient.config.FeederConfig;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
@@ -43,9 +20,6 @@ import org.w3c.dom.NodeList;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.logging.Level;
 
 /**
  * Builds the Clients plugin
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index 7dabfdc600b..c83f6098a0f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -25,14 +25,22 @@ public class NodesSpecification {
 
     private final int groups;
 
+    /** 
+     * Whether the capacity amount specified is required or can it be relaxed 
+     * at the discretion of the component fulfilling it
+     */
+    private final boolean required;
+
     private final Optional<String> flavor;
 
     private final Optional<String> dockerImage;
 
-    private NodesSpecification(boolean dedicated, int count, int groups, Optional<String> flavor, Optional<String> dockerImage) {
+    private NodesSpecification(boolean dedicated, int count, int groups, boolean required, 
+                               Optional<String> flavor, Optional<String> dockerImage) {
         this.dedicated = dedicated;
         this.count = count;
         this.groups = groups;
+        this.required = required;
         this.flavor = flavor;
         this.dockerImage = dockerImage;
     }
@@ -41,6 +49,7 @@ public class NodesSpecification {
         this(dedicated,
              nodesElement.requiredIntegerAttribute("count"),
              nodesElement.getIntegerAttribute("groups", 1),
+             nodesElement.getBooleanAttribute("required", false),
              Optional.ofNullable(nodesElement.getStringAttribute("flavor")),
              Optional.ofNullable(nodesElement.getStringAttribute("docker-image")));
     }
@@ -78,7 +87,7 @@ public class NodesSpecification {
 
     /** Returns a requirement from <code>count</code> nondedicated nodes in one group */
     public static NodesSpecification nonDedicated(int count) {
-        return new NodesSpecification(false, count, 1, Optional.empty(), Optional.empty());
+        return new NodesSpecification(false, count, 1, false, Optional.empty(), Optional.empty());
     }
 
     /**
@@ -95,7 +104,7 @@ public class NodesSpecification {
 
     public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger) {
         ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, dockerImage);
-        return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor), groups, logger);
+        return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor, required), groups, logger);
     }
 
     @Override
diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc
index 06e7b945c18..b89fe0d7fcb 100644
--- a/config-model/src/main/resources/schema/common.rnc
+++ b/config-model/src/main/resources/schema/common.rnc
@@ -23,6 +23,7 @@ Nodes = element nodes {
 OptionalDedicatedNodes = element nodes {
     attribute count { xsd:positiveInteger } &
     attribute flavor { xsd:string }? &
+    attribute required { xsd:boolean }? &
     attribute docker-image { xsd:string }? &
     attribute dedicated { xsd:boolean }?
 }
diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc
index a5679151a03..17e38883755 100644
--- a/config-model/src/main/resources/schema/containercluster.rnc
+++ b/config-model/src/main/resources/schema/containercluster.rnc
@@ -194,6 +194,7 @@ NodesOfContainerCluster = element nodes {
       (
         attribute count { xsd:positiveInteger } &
         attribute flavor { xsd:string }? &
+        attribute required { xsd:boolean }? &
         attribute docker-image { xsd:string }?
       )
       |
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index 30b931053d5..c3a8386ac5e 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -216,6 +216,7 @@ ContentNodes = element nodes {
       (
         attribute count { xsd:positiveInteger } &
         attribute flavor { xsd:string }? &
+        attribute required { xsd:boolean }? &
         attribute docker-image { xsd:string }? &
         attribute groups { xsd:positiveInteger }?
       )
@@ -260,6 +261,7 @@ Group = element group {
             element nodes {
                 attribute count { xsd:positiveInteger } &
                 attribute flavor { xsd:string }? &
+                attribute required { xsd:boolean }? &
                 attribute docker-image { xsd:string }? &
                 attribute groups { xsd:positiveInteger }?
             }
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 86fabdf26bc..8ed4539456a 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -522,6 +522,7 @@ public class ModelProvisioningTest {
         assertEquals(1, clusterControllers.getContainers().size()); // TODO: Expected 5 with this feature reactivated
     }
 
+    @Test
     public void testClusterControllersAreNotPlacedOnRetiredNodes() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -907,6 +908,26 @@ public class ModelProvisioningTest {
         assertThat(cluster.getRootGroup().getNodes().get(0).getConfigId(), is("bar/storage/0"));
     }
 
+    @Test(expected = IllegalArgumentException.class)
+    public void testRequiringMoreNodesThanAreAvailable() throws ParseException {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>\n" +
+                "<services>" +
+                "  <content version='1.0' id='bar'>" +
+                "     <redundancy>1</redundancy>" +
+                "     <documents>" +
+                "       <document type='type1' mode='index'/>" +
+                "     </documents>" +
+                "     <nodes count='3' required='true'/>" +
+                "  </content>" +
+                "</services>";
+
+        int numberOfHosts = 2;
+        VespaModelTester tester = new VespaModelTester();
+        tester.addHosts(numberOfHosts);
+        tester.createModel(services, false);
+    }
+
     @Test
     public void testUsingNodesCountAttributesAndGettingJustOneNode() {
         String services =
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
index 7c13204c1e7..7894b722b58 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
@@ -75,7 +75,11 @@ public final class Capacity {
     public static Capacity fromRequiredNodeCount(int nodeCount, Optional<String> flavor) {
         return new Capacity(nodeCount, true, flavor, NodeType.tenant);
     }
-    
+
+    public static Capacity fromNodeCount(int nodeCount, Optional<String> flavor, boolean required) {
+        return new Capacity(nodeCount, required, flavor, NodeType.tenant);
+    }
+
     /** Creates this from a node type */
     public static Capacity fromRequiredNodeType(NodeType type) {
         return new Capacity(0, true, Optional.empty(), type);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 99036ee0027..1b32d6bde22 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -120,13 +120,12 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                                                                    ConfigserverConfig configserverConfig,
                                                                    Zone zone,
                                                                    Set<Rotation> rotations) {
-        return new ModelContextImpl.Properties(
-                applicationId,
-                configserverConfig.multitenant(),
-                ConfigServerSpec.fromConfig(configserverConfig),
-                configserverConfig.hostedVespa(),
-                zone,
-                rotations);
+        return new ModelContextImpl.Properties(applicationId, 
+                                               configserverConfig.multitenant(),
+                                               ConfigServerSpec.fromConfig(configserverConfig),
+                                               configserverConfig.hostedVespa(),
+                                               zone,
+                                               rotations);
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
index cacd53cf945..9c1b2b4681e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
@@ -91,11 +91,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
         this.applicationId = params.getApplicationId();
         this.rotations = new Rotations(curator, tenantPath);
         this.rotationsSet = getRotations(params.rotations());
-        this.properties = createModelContextProperties(
-                params.getApplicationId(),
-                configserverConfig,
-                zone,
-                rotationsSet);
+        this.properties = createModelContextProperties(params.getApplicationId(), configserverConfig, zone, rotationsSet);
     }
 
     /** Construct with all dependencies passed separately */
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index b10865f257b..d2ded8ee226 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -163,10 +163,7 @@ public class SessionPreparer {
 
         void preprocess() {
             try {
-                this.applicationPackage = context.getApplicationPackage().preprocess(
-                        properties.zone(),
-                        null,
-                        logger);
+                this.applicationPackage = context.getApplicationPackage().preprocess(properties.zone(), null, logger);
             } catch (IOException | TransformerException | ParserConfigurationException | SAXException e) {
                 throw new RuntimeException("Error deploying application package", e);
             }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index 6665833c1a2..a759a8fca37 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -31,7 +31,7 @@ public class CapacityPolicies {
 
         switch(zone.environment()) {
             case dev : case test : return 1;
-            case perf : return Math.min(requestedCapacity.nodeCount(), 10); // TODO: Decrease to 3 when isRequired is implemented
+            case perf : return Math.min(requestedCapacity.nodeCount(), 3);
             case staging: return requestedNodes <= 1 ? requestedNodes : Math.max(2, requestedNodes / 10);
             case prod : return ensureRedundancy(requestedCapacity.nodeCount());
             default : throw new IllegalArgumentException("Unsupported environment " + zone.environment());
@@ -53,7 +53,7 @@ public class CapacityPolicies {
 
     /**
      * Throw if the node count is 1
-
+     *
      * @return the argument node count
      * @throws IllegalArgumentException if only one node is requested
      */