Support configurable feed block hysteresis on the cluster controller

Adds an absolute number delta that is subtracted from the feed block limit
when a node has a resource already in feed blocked state. This means that
there's a lower watermark threshold that must be crossed before feeding
can be unblocked. Avoids flip-flopping between block states.

Default is currently 0.0, i.e. effectively disabled. To be modified
later for system tests and trial roll-outs.

A couple of caveats with the current implementation:
* The cluster state is not recomputed automatically when just the hysteresis
  threshold is crossed, so the description will be out of date on the
  content nodes. However, if any other feed block event happens (or the
  hysteresis threshold is crossed), the state will be recomputed as expected.
  This does not affect correctness, since the feed is still to be blocked.
* A node event remove/add pair is emitted for feed block status when the
  hysteresis threshold is crossed and there's a cluster state recomputation.

2 files changed, 4 insertions, 1 deletions

diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java
index df778028325..4cb6c5d222a 100644
--- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java
+++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java
@@ -83,6 +83,7 @@ public class ClusterControllerClusterConfigurer {
         options.enableTwoPhaseClusterStateActivation = config.enable_two_phase_cluster_state_transitions();
         options.clusterFeedBlockEnabled = config.enable_cluster_feed_block();
         options.clusterFeedBlockLimit = Map.copyOf(config.cluster_feed_block_limit());
+        options.clusterFeedBlockNoiseLevel = config.cluster_feed_block_noise_level();
     }
 
     private static void configure(FleetControllerOptions options, SlobroksConfig config) {
diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java
index 9d2d7610469..76eff0066b1 100644
--- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java
+++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java
@@ -34,7 +34,8 @@ public class ClusterControllerClusterConfigurerTest {
                 .min_node_ratio_per_group(0.123)
                 .enable_cluster_feed_block(true)
                 .cluster_feed_block_limit("foo", 0.5)
-                .cluster_feed_block_limit("bar", 0.7);
+                .cluster_feed_block_limit("bar", 0.7)
+                .cluster_feed_block_noise_level(0.05);
         SlobroksConfig.Builder slobroksConfig = new SlobroksConfig.Builder();
         SlobroksConfig.Slobrok.Builder slobrok = new SlobroksConfig.Slobrok.Builder();
         slobrok.connectionspec("foo");
@@ -63,6 +64,7 @@ public class ClusterControllerClusterConfigurerTest {
         assertTrue(configurer.getOptions().clusterFeedBlockEnabled);
         assertEquals(0.5, configurer.getOptions().clusterFeedBlockLimit.get("foo"), 0.01);
         assertEquals(0.7, configurer.getOptions().clusterFeedBlockLimit.get("bar"), 0.01);
+        assertEquals(0.05, configurer.getOptions().clusterFeedBlockNoiseLevel, 0.001);
 
         try{
             zookeepersConfig.zookeeperserverlist("");