Transition from down to up initially

- Use tri-state logic for working/failing/unknown
- Be initially down in test and verify we come up

2 files changed, 35 insertions, 19 deletions

diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java
index 84c10991293..79e96a7c5a2 100644
--- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java
@@ -114,7 +114,8 @@ public class ClusteredConnectionTestCase {
         connection0.setInService(false);
         forcePing(myBackend);
         r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0));
-        assertEquals("No backends in service. Try later",r.hits().getError().getMessage());
+        System.out.println(r.hits().getError().getDetailedMessage());
+        assertEquals("No backends in service. Try later", r.hits().getError().getMessage());
 
         connection2.setInService(true);
         connection1.setInService(true);
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
index bbaf512534a..df1049f499d 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
@@ -19,13 +19,16 @@ import java.util.concurrent.atomic.AtomicInteger;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+/**
+ * @author baldersheim
+ */
 public class SearchClusterTest {
 
     static class State {
         final String clusterId;
         final int nodesPerGroup;
         final VipStatus vipStatus;
-        final SearchCluster sc;
+        final SearchCluster searchCluster;
         final List<AtomicInteger> numDocsPerNode;
         List<AtomicInteger> pingCounts;
         State(String clusterId, int nodesPergroup, String ... nodeNames) {
@@ -35,9 +38,6 @@ public class SearchClusterTest {
             this.clusterId = clusterId;
             this.nodesPerGroup = nodesPergroup;
             vipStatus = new VipStatus(new QrSearchersConfig.Builder().searchcluster(new QrSearchersConfig.Searchcluster.Builder().name(clusterId)).build(), new ClustersStatus());
-            assertFalse(vipStatus.isInRotation());
-            vipStatus.addToRotation(clusterId);
-            assertTrue(vipStatus.isInRotation());
             numDocsPerNode = new ArrayList<>(nodeNames.size());
             pingCounts = new ArrayList<>(nodeNames.size());
             List<Node> nodes = new ArrayList<>(nodeNames.size());
@@ -49,10 +49,10 @@ public class SearchClusterTest {
                 numDocsPerNode.add(new AtomicInteger(1));
                 pingCounts.add(new AtomicInteger(0));
             }
-            sc = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes),nodes.size() / nodesPergroup, vipStatus);
+            searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPergroup, vipStatus);
         }
         void startMonitoring() {
-            sc.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts));
+            searchCluster.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts));
         }
         private static int getMaxValue(List<AtomicInteger> list) {
             int max = list.get(0).get();
@@ -118,19 +118,25 @@ public class SearchClusterTest {
     }
 
     @Test
-    public void requireThatVipStatusIsDefaultUp() {
+    public void requireThatVipStatusIsDefaultDownButComesUpAfterPinging() {
         State test = new State("cluster.1", 2, "a", "b");
+        assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
+
+        assertFalse(test.vipStatus.isInRotation());
+        test.startMonitoring();
+        test.waitOneFullPingRound();
         assertTrue(test.vipStatus.isInRotation());
-        assertTrue(test.sc.localCorpusDispatchTarget().isEmpty());
     }
 
     @Test
     public void requireThatZeroDocsAreFine() {
         State test = new State("cluster.1", 2,"a", "b");
+        test.startMonitoring();
+        test.waitOneFullPingRound();
+
         assertTrue(test.vipStatus.isInRotation());
-        assertTrue(test.sc.localCorpusDispatchTarget().isEmpty());
+        assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
 
-        test.startMonitoring();
         test.numDocsPerNode.get(0).set(-1);
         test.numDocsPerNode.get(1).set(-1);
         test.waitOneFullPingRound();
@@ -141,21 +147,27 @@ public class SearchClusterTest {
     }
 
     @Test
-    public void requireThatVipStatusIsDefaultUpWithLocalDispatch() {
+    public void requireThatVipStatusIsDefaultDownWithLocalDispatch() {
         State test = new State("cluster.1", 1, HostName.getLocalhost(), "b");
+        assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
+
+        assertFalse(test.vipStatus.isInRotation());
+        test.startMonitoring();
+        test.waitOneFullPingRound();
         assertTrue(test.vipStatus.isInRotation());
-        assertTrue(test.sc.localCorpusDispatchTarget().isPresent());
     }
 
     @Test
     public void requireThatVipStatusDownWhenLocalIsDown() {
         State test = new State("cluster.1",1,HostName.getLocalhost(), "b");
-        assertTrue(test.vipStatus.isInRotation());
-        assertTrue(test.sc.localCorpusDispatchTarget().isPresent());
 
         test.startMonitoring();
         test.waitOneFullPingRound();
         assertTrue(test.vipStatus.isInRotation());
+        assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
+
+        test.waitOneFullPingRound();
+        assertTrue(test.vipStatus.isInRotation());
         test.numDocsPerNode.get(0).set(-1);
         test.waitOneFullPingRound();
         assertFalse(test.vipStatus.isInRotation());
@@ -183,10 +195,11 @@ public class SearchClusterTest {
     private void verifyThatVipStatusDownRequireAllNodesDown(int numGroups, int nodesPerGroup) {
         List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup);
         State test = new State("cluster.1", nodesPerGroup, nodeNames);
+        test.startMonitoring();
+        test.waitOneFullPingRound();
         assertTrue(test.vipStatus.isInRotation());
-        assertTrue(test.sc.localCorpusDispatchTarget().isEmpty());
+        assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
 
-        test.startMonitoring();
         test.waitOneFullPingRound();
         assertTrue(test.vipStatus.isInRotation());
 
@@ -214,13 +227,15 @@ public class SearchClusterTest {
         }
         return nodeNames;
     }
+
     private void verifyThatVipStatusUpRequireOnlyOneOnlineNode(int numGroups, int nodesPerGroup) {
         List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup);
         State test = new State("cluster.1", nodesPerGroup, nodeNames);
+        test.startMonitoring();
+        test.waitOneFullPingRound();
         assertTrue(test.vipStatus.isInRotation());
-        assertTrue(test.sc.localCorpusDispatchTarget().isEmpty());
+        assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
 
-        test.startMonitoring();
         for (int i=0; i < test.numDocsPerNode.size()-1; i++) {
             test.numDocsPerNode.get(i).set(-1);
         }