Require all up when going up, but require all down to go down.

5 files changed, 114 insertions, 34 deletions

diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json
index 91c0e4cc6bf..fe8fb91d71f 100644
--- a/container-core/abi-spec.json
+++ b/container-core/abi-spec.json
@@ -11,6 +11,23 @@
     ],
     "fields": []
   },
+  "com.yahoo.container.handler.ClustersStatus$Require": {
+    "superClass": "java.lang.Enum",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final",
+      "enum"
+    ],
+    "methods": [
+      "public static com.yahoo.container.handler.ClustersStatus$Require[] values()",
+      "public static com.yahoo.container.handler.ClustersStatus$Require valueOf(java.lang.String)"
+    ],
+    "fields": [
+      "public static final enum com.yahoo.container.handler.ClustersStatus$Require ONE",
+      "public static final enum com.yahoo.container.handler.ClustersStatus$Require ALL"
+    ]
+  },
   "com.yahoo.container.handler.ClustersStatus": {
     "superClass": "com.yahoo.component.AbstractComponent",
     "interfaces": [],
@@ -23,7 +40,8 @@
       "public void setReceiveTrafficByDefault(boolean)",
       "public void setUp(java.lang.Object)",
       "public void setDown(java.lang.Object)",
-      "public boolean containerShouldReceiveTraffic()"
+      "public boolean containerShouldReceiveTraffic()",
+      "public boolean containerShouldReceiveTraffic(com.yahoo.container.handler.ClustersStatus$Require)"
     ],
     "fields": []
   },
diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
index aab13a1cc7b..0ed0daa2141 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
@@ -27,6 +27,8 @@ public class ClustersStatus extends AbstractComponent {
     @Inject
     public ClustersStatus() { }
 
+    public enum Require {ONE, ALL}
+
     /** Are there any (in-service influencing) clusters in this container? */
     private boolean containerHasClusters;
 
@@ -72,12 +74,25 @@ public class ClustersStatus extends AbstractComponent {
         setDown((String) clusterIdentifier);
     }
 
-    /** Returns whether this container should receive traffic based on the state of this */
+    @Deprecated // TODO: Remove on Vespa 8
     public boolean containerShouldReceiveTraffic() {
+        return containerShouldReceiveTraffic(Require.ONE);
+    }
+    /**
+     *  Returns whether this container should receive traffic based on the state of this
+     *  @param require Requirement for being up, ALL or ONE.
+     */
+    public boolean containerShouldReceiveTraffic(Require require) {
         synchronized (mutex) {
             if (containerHasClusters) {
-                // Should receive traffic when at least one cluster is up
-                return clusterStatus.values().stream().anyMatch(status -> status==true);
+                switch (require) {
+                    case ONE:
+                        // Should receive traffic when at least one cluster is up
+                        return clusterStatus.values().stream().anyMatch(status -> status == true);
+                    case ALL:
+                    default:
+                        return !clusterStatus.isEmpty() && clusterStatus.values().stream().allMatch(status -> status == true);
+                }
             }
             else {
                 return true;
diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
index b9ef1627ce7..f712690efc5 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
@@ -102,10 +102,15 @@ public class VipStatus {
 
     private void updateCurrentlyInRotation() {
         synchronized (mutex) {
-            if (rotationOverride != null)
+            if (rotationOverride != null) {
                 currentlyInRotation = rotationOverride;
-            else
-                currentlyInRotation = clustersStatus.containerShouldReceiveTraffic();
+            } else {
+                if (healthState.status() == StateMonitor.Status.up) {
+                    currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE);
+                } else {
+                    currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL);
+                }
+            }
 
             // Change to/from 'up' when appropriate but don't change 'initializing' to 'down'
             if (currentlyInRotation)
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
index f690c240537..0be9f47f6cf 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
@@ -53,10 +53,16 @@ public class StateMonitor extends AbstractComponent {
     }
 
     StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) {
+        this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)),
+             Status.valueOf(config.initialStatus()),
+             timer, threadFactory);
+    }
+    /* For Testing */
+    public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) {
         this.timer = timer;
-        this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1));
+        this.snapshotIntervalMs = snapshotIntervalMS;
         this.lastSnapshotTimeMs = timer.currentTimeMillis();
-        this.status = Status.valueOf(config.initialStatus());
+        this.status = status;
         thread = threadFactory.newThread(this::run);
         thread.start();
     }
diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
index 52679c15957..d3479936544 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
@@ -4,6 +4,8 @@ package com.yahoo.container.handler;
 import static org.junit.Assert.*;
 
 import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.jdisc.state.StateMonitor;
+import com.yahoo.jdisc.core.SystemTimer;
 import org.junit.Test;
 
 /**
@@ -12,45 +14,79 @@ import org.junit.Test;
  * @author steinar
  */
 public class VipStatusTestCase {
+    private static final String [] clusters = {"cluster1", "cluster2", "cluster3"};
 
-    @Test
-    public void testVipStatusWorksWithClusters() {
+    private static QrSearchersConfig getSearchersCfg() {
         var b = new QrSearchersConfig.Builder();
         var searchClusterB = new QrSearchersConfig.Searchcluster.Builder();
-        searchClusterB.name("cluster1");
-        searchClusterB.name("cluster2");
-        searchClusterB.name("cluster3");
+        for (String cluster : clusters) {
+            searchClusterB.name(cluster);
+        }
         b.searchcluster(searchClusterB);
-        VipStatus v = new VipStatus(b.build());
+        return b.build();
+    }
+    private static VipStatus getVipStatus(StateMonitor.Status startState) {
+        return new VipStatus(getSearchersCfg(), new ClustersStatus(), new StateMonitor(1000, startState, new SystemTimer(), runnable -> {
+            Thread thread = new Thread(runnable, "StateMonitor");
+            thread.setDaemon(true);
+            return thread;
+        }));
+    }
 
-        String cluster1 = "cluster1";
-        String cluster2 = "cluster2";
-        String cluster3 = "cluster3";
+    private static void removeAll(VipStatus v) {
+        for (String s : clusters) {
+            v.removeFromRotation(s);
+        }
+    }
+    private static void addAll(VipStatus v) {
+        for (String s : clusters) {
+            v.addToRotation(s);
+        }
+    }
 
+    private static void verifyUpOrDown(StateMonitor.Status status) {
+        VipStatus v = getVipStatus(status);
+        removeAll(v);
         // initial state
         assertFalse(v.isInRotation());
-
-        // one cluster becomes up
-        v.addToRotation(cluster1);
+        v.addToRotation(clusters[0]);
+        assertFalse(v.isInRotation());
+        v.addToRotation(clusters[1]);
+        assertFalse(v.isInRotation());
+        v.addToRotation(clusters[2]);
         assertTrue(v.isInRotation());
+    }
 
-        // all clusters down
-        v.removeFromRotation(cluster1);
-        v.removeFromRotation(cluster2);
-        v.removeFromRotation(cluster3);
+    @Test
+    public void testInitializingOrDownRequireAllUp() {
+        verifyUpOrDown(StateMonitor.Status.initializing);
+        verifyUpOrDown(StateMonitor.Status.down);
+    }
+
+    @Test
+    public void testUpRequireAllDown() {
+        VipStatus v = getVipStatus(StateMonitor.Status.initializing);
         assertFalse(v.isInRotation());
-        // some clusters down
-        v.addToRotation(cluster2);
+        addAll(v);
         assertTrue(v.isInRotation());
-        // all clusters up
-        v.addToRotation(cluster1);
-        v.addToRotation(cluster3);
+
+        v.removeFromRotation(clusters[0]);
+        assertTrue(v.isInRotation());
+        v.removeFromRotation(clusters[1]);
         assertTrue(v.isInRotation());
-        // and down again
-        v.removeFromRotation(cluster1);
-        v.removeFromRotation(cluster2);
-        v.removeFromRotation(cluster3);
+        v.removeFromRotation(clusters[2]);
+        assertFalse(v.isInRotation());  // All down
+        v.addToRotation(clusters[1]);
         assertFalse(v.isInRotation());
+        v.addToRotation(clusters[0]);
+        v.addToRotation(clusters[2]);
+        assertTrue(v.isInRotation());   // All up
+        v.removeFromRotation(clusters[0]);
+        v.removeFromRotation(clusters[2]);
+        assertTrue(v.isInRotation());
+        v.addToRotation(clusters[0]);
+        v.addToRotation(clusters[2]);
+        assertTrue(v.isInRotation());
     }
 
 }