Set healt status to down when we decide to go out of rotation

4 files changed, 42 insertions, 13 deletions

diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
index 16c2677e1b6..aab13a1cc7b 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
@@ -9,7 +9,9 @@ import java.util.Map;
 
 /**
  * A component which tracks the up/down status of any clusters which should influence
- * the up down status of this container itself, as well as the separate fact that such clusters are present.
+ * the up down status of this container itself, as well as the separate fact (from config)
+ * that such clusters are present. This is a separate fact because we might know we have clusters configured
+ * but we don't have positive information that they are up yet, and in this case we should be down.
  *
  * This is a separate component which has <b>no dependencies</b> such that the status tracked in this
  * will survive reconfiguration events and inform other components even immediately after a reconfiguration
diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
index 555dc81057d..a0b959a1c26 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
@@ -4,6 +4,7 @@ package com.yahoo.container.handler;
 import com.google.inject.Inject;
 import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.container.core.VipStatusConfig;
+import com.yahoo.container.jdisc.state.StateMonitor;
 
 /**
  * API for programmatically removing the container from VIP rotation.
@@ -17,6 +18,8 @@ public class VipStatus {
 
     private final ClustersStatus clustersStatus;
 
+    private final StateMonitor healthState;
+
     /** If this is non-null, its value decides whether this container is in rotation */
     private Boolean rotationOverride = null;
 
@@ -25,23 +28,30 @@ public class VipStatus {
 
     private final Object mutex = new Object();
 
+    /** For testing */
     public VipStatus() {
-        this(new QrSearchersConfig(new QrSearchersConfig.Builder()),
-             new VipStatusConfig(new VipStatusConfig.Builder()),
-             new ClustersStatus());
+        this(new ClustersStatus());
     }
 
+    /** For testing */
     public VipStatus(QrSearchersConfig dispatchers) {
         this(dispatchers, new ClustersStatus());
     }
 
+    /** For testing */
     public VipStatus(ClustersStatus clustersStatus) {
-        this.clustersStatus = clustersStatus;
+        this(new QrSearchersConfig.Builder().build(), clustersStatus);
     }
 
-    @Inject
     public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus) {
+        this(dispatchers, clustersStatus, new StateMonitor());
+    }
+
+    @Inject
+    public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) {
         this.clustersStatus = clustersStatus;
+        this.healthState = healthState;
+        healthState.status(StateMonitor.Status.initializing);
         clustersStatus.setContainerHasClusters(! dispatchers.searchcluster().isEmpty());
         updateCurrentlyInRotation();
     }
@@ -95,6 +105,12 @@ public class VipStatus {
                 currentlyInRotation = rotationOverride;
             else
                 currentlyInRotation = clustersStatus.containerShouldReceiveTraffic();
+
+            // Change to/from 'up' when appropriate but don't change 'initializing' to 'down'
+            if (currentlyInRotation)
+                healthState.status(StateMonitor.Status.up);
+            else if (healthState.status() == StateMonitor.Status.up)
+                healthState.status(StateMonitor.Status.down);
         }
     }
 
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
index 9d558a9c2a2..f690c240537 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
@@ -6,6 +6,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.jdisc.Timer;
 import com.yahoo.jdisc.application.MetricConsumer;
+import com.yahoo.jdisc.core.SystemTimer;
 import com.yahoo.log.LogLevel;
 
 import java.util.Map;
@@ -37,6 +38,11 @@ public class StateMonitor extends AbstractComponent {
     private volatile Status status;
     private final TreeSet<String> valueNames = new TreeSet<>();
 
+    /** For testing */
+    public StateMonitor() {
+        this(new HealthMonitorConfig.Builder().build(), new SystemTimer());
+    }
+
     @Inject
     public StateMonitor(HealthMonitorConfig config, Timer timer) {
         this(config, timer, runnable -> {
diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
index 4c1c1622140..52679c15957 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
@@ -3,6 +3,7 @@ package com.yahoo.container.handler;
 
 import static org.junit.Assert.*;
 
+import com.yahoo.container.QrSearchersConfig;
 import org.junit.Test;
 
 /**
@@ -14,13 +15,17 @@ public class VipStatusTestCase {
 
     @Test
     public void testVipStatusWorksWithClusters() {
-        ClustersStatus clustersStatus = new ClustersStatus();
-        clustersStatus.setContainerHasClusters(true);
-        VipStatus v = new VipStatus(clustersStatus);
-
-        String cluster1 = new String("a");
-        String cluster2 = new String("b");
-        String cluster3 = new String("c");
+        var b = new QrSearchersConfig.Builder();
+        var searchClusterB = new QrSearchersConfig.Searchcluster.Builder();
+        searchClusterB.name("cluster1");
+        searchClusterB.name("cluster2");
+        searchClusterB.name("cluster3");
+        b.searchcluster(searchClusterB);
+        VipStatus v = new VipStatus(b.build());
+
+        String cluster1 = "cluster1";
+        String cluster2 = "cluster2";
+        String cluster3 = "cluster3";
 
         // initial state
         assertFalse(v.isInRotation());