Add configurable automatic group up/down feature based on node availability

Available under content cluster tuning tag; feature is currently disabled by
default (need prod experience for this first).
Also improves handling of nodes removed from config by ensuring these are
taken out of the core working cluster state instead of just patched away
before each state publish.

3 files changed, 48 insertions, 11 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
index fd1bc8a3362..4a0ce99914a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
@@ -31,8 +31,12 @@ public class ClusterControllerConfig extends AbstractConfigProducer implements F
             ModelElement tuning = null;
 
             ModelElement clusterTuning = clusterElement.getChild("tuning");
+            Integer bucketSplittingMinimumBits = null;
+            Double minNodeRatioPerGroup = null;
             if (clusterTuning != null) {
                 tuning = clusterTuning.getChild("cluster-controller");
+                minNodeRatioPerGroup = clusterTuning.childAsDouble("min-node-ratio-per-group");
+                bucketSplittingMinimumBits = clusterTuning.childAsInteger("bucket-splitting.minimum-bits");
             }
 
             if (tuning != null) {
@@ -43,10 +47,11 @@ public class ClusterControllerConfig extends AbstractConfigProducer implements F
                         tuning.childAsDuration("stable-state-period"),
                         tuning.childAsDouble("min-distributor-up-ratio"),
                         tuning.childAsDouble("min-storage-up-ratio"),
-                        clusterElement.childAsInteger("tuning.bucket-splitting.minimum-bits"));
+                        bucketSplittingMinimumBits,
+                        minNodeRatioPerGroup);
             } else {
                 return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null,
-                        clusterElement.childAsInteger("tuning.bucket-splitting.minimum-bits"));
+                        bucketSplittingMinimumBits, minNodeRatioPerGroup);
             }
         }
     }
@@ -59,7 +64,9 @@ public class ClusterControllerConfig extends AbstractConfigProducer implements F
     Double minDistributorUpRatio;
     Double minStorageUpRatio;
     Integer minSplitBits;
+    private Double minNodeRatioPerGroup;
 
+    // TODO refactor; too many args
     private ClusterControllerConfig(AbstractConfigProducer parent,
                                     String clusterName,
                                     Duration initProgressTime,
@@ -68,7 +75,8 @@ public class ClusterControllerConfig extends AbstractConfigProducer implements F
                                     Duration stableStateTimePeriod,
                                     Double minDistributorUpRatio,
                                     Double minStorageUpRatio,
-                                    Integer minSplitBits) {
+                                    Integer minSplitBits,
+                                    Double minNodeRatioPerGroup) {
         super(parent, "fleetcontroller");
 
         this.clusterName = clusterName;
@@ -79,6 +87,7 @@ public class ClusterControllerConfig extends AbstractConfigProducer implements F
         this.minDistributorUpRatio = minDistributorUpRatio;
         this.minStorageUpRatio = minStorageUpRatio;
         this.minSplitBits = minSplitBits;
+        this.minNodeRatioPerGroup = minNodeRatioPerGroup;
     }
 
     @Override
@@ -117,5 +126,8 @@ public class ClusterControllerConfig extends AbstractConfigProducer implements F
         if (minSplitBits != null) {
             builder.ideal_distribution_bits(minSplitBits);
         }
+        if (minNodeRatioPerGroup != null) {
+            builder.min_node_ratio_per_group(minNodeRatioPerGroup);
+        }
     }
 }
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index 36a8dd276ef..1c20acdc0bd 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -60,6 +60,10 @@ PersistenceThreads = element persistence-threads {
   Thread+
 }
 
+MinNodeRatioPerGroup = element min-node-ratio-per-group {
+  xsd:double { minInclusive = "0" maxInclusive = "1" }
+}
+
 ClusterControllerTuning = element cluster-controller {
     element init-progress-time { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? &
     element transition-time {  xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" }  }? &
@@ -85,7 +89,8 @@ ClusterTuning = element tuning {
    VisitorTuning? &
    ClusterControllerTuning? &
    Maintenance? &
-   PersistenceThreads?
+   PersistenceThreads? &
+   MinNodeRatioPerGroup?
 }
 
 Content = element content {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
index 204491b0724..98cc4e5cc47 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
@@ -10,13 +10,6 @@ import org.w3c.dom.Document;
 
 import static org.junit.Assert.assertEquals;
 
-/**
- * Created with IntelliJ IDEA.
- * User: thomasg
- * Date: 5/10/12
- * Time: 2:29 PM
- * To change this template use File | Settings | File Templates.
- */
 public class FleetControllerClusterTest {
     ClusterControllerConfig parse(String xml) {
         Document doc = XML.getDocument(xml);
@@ -69,4 +62,31 @@ public class FleetControllerClusterTest {
         FleetcontrollerConfig config = new FleetcontrollerConfig(builder);
         assertEquals(13, config.init_progress_time());
     }
+
+    @Test
+    public void min_node_ratio_per_group_tuning_config_is_propagated() {
+        FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder();
+        parse("<cluster id=\"storage\">\n" +
+                "  <documents/>\n" +
+                "  <tuning>\n" +
+                "    <min-node-ratio-per-group>0.75</min-node-ratio-per-group>\n" +
+                "  </tuning>\n" +
+                "</cluster>").
+                getConfig(builder);
+
+        FleetcontrollerConfig config = new FleetcontrollerConfig(builder);
+        assertEquals(0.75, config.min_node_ratio_per_group(), 0.01);
+    }
+
+    @Test
+    public void min_node_ratio_per_group_is_implicitly_zero_when_omitted() {
+        FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder();
+        parse("<cluster id=\"storage\">\n" +
+                "  <documents/>\n" +
+                "</cluster>").
+                getConfig(builder);
+
+        FleetcontrollerConfig config = new FleetcontrollerConfig(builder);
+        assertEquals(0.0, config.min_node_ratio_per_group(), 0.01);
+    }
 }