Protect against interference from ping threads

2 files changed, 18 insertions, 6 deletions

diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java
index 3cc7a07cbf7..1f92738b632 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java
@@ -86,30 +86,30 @@ public class DirectSearchTestCase {
         tester.search("?query=test&dispatch.direct=true&nocache");
         assertEquals("Still 1 ping request, 0 search requests because the default coverage is 97%, and we only have 96% locally",
                      1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
-        assertFalse(tester.vipStatus().isInRotation());
+        tester.waitForInRotationIs(false);
 
         tester.setActiveDocuments(FastSearcherTester.selfHostname, (long) (99 * k));
         assertEquals("2 ping request, 0 search requests", 2, tester.requestCount(FastSearcherTester.selfHostname, 9999));
         tester.search("?query=test&dispatch.direct=true&nocache");
         assertEquals("2 ping request, 1 search requests because we now have 99% locally",
                      3, tester.requestCount(FastSearcherTester.selfHostname, 9999));
-        assertTrue(tester.vipStatus().isInRotation());
+        tester.waitForInRotationIs(true);
 
         tester.setActiveDocuments("host1", (long) (104 * k));
         assertEquals("2 ping request, 1 search requests", 3, tester.requestCount(FastSearcherTester.selfHostname, 9999));
         tester.search("?query=test&dispatch.direct=true&nocache");
         assertEquals("2 ping request, 2 search requests because 99/((104+100)/2) > 0.97",
                      4, tester.requestCount(FastSearcherTester.selfHostname, 9999));
-        assertTrue(tester.vipStatus().isInRotation());
+        tester.waitForInRotationIs(true);
 
         tester.setActiveDocuments("host2", (long) (102 * k));
         assertEquals("2 ping request, 2 search requests", 4, tester.requestCount(FastSearcherTester.selfHostname, 9999));
         tester.search("?query=test&dispatch.direct=true&nocache");
         assertEquals("Still 2 ping request, 2 search requests because 99/((104+102)/2) < 0.97",
                      4, tester.requestCount(FastSearcherTester.selfHostname, 9999));
-        assertFalse(tester.vipStatus().isInRotation());
+        tester.waitForInRotationIs(false);
     }
-
+    
     @Test
     public void testCoverageWithSingleGroup() {
         FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0");
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java
index d2638be4bc7..c9e36865689 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java
@@ -21,6 +21,7 @@ import java.util.Collections;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 /**
  * @author bratseth
@@ -93,7 +94,18 @@ class FastSearcherTester {
         mockDispatcher.searchCluster().ping(node, MoreExecutors.directExecutor());
         mockDispatcher.searchCluster().pingIterationCompleted();
     }
-    
+
     public VipStatus vipStatus() { return vipStatus; }
 
+    /** Retrying is needed because earlier pings from the monitoring thread may interfere with the testing thread */
+    public void waitForInRotationIs(boolean expectedRotationStatus) {
+        int triesLeft = 9000;
+        while (vipStatus.isInRotation() != expectedRotationStatus && triesLeft > 0) {
+            triesLeft--;
+            try { Thread.sleep(10); } catch (InterruptedException e) {}
+        }
+        if (triesLeft == 0)
+            fail("Did not reach VIP in rotation status = " + expectedRotationStatus + " after trying for 90 seconds");
+    }
+
 }