10 files changed, 549 insertions, 133 deletions
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java
new file mode 100644
index 00000000000..eb6f92d928c
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/CancellableImplTest.java
@@ -0,0 +1,79 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author hakonhall
+ */
+public class CancellableImplTest {
+    private final TestExecutor executor = new TestExecutor();
+    private final TestRunlet runlet = new TestRunlet();
+    private final Cancellable cancellable = executor.scheduleWithFixedDelay(runlet, Duration.ofSeconds(1));
+
+    @After
+    public void tearDown() {
+        executor.close();
+    }
+
+    @Before
+    public void setUp() {
+        assertEquals(0, runlet.getRunsStarted());
+        executor.runToCompletion(1);
+        assertEquals(1, runlet.getRunsStarted());
+        executor.runToCompletion(2);
+        assertEquals(2, runlet.getRunsStarted());
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+    }
+
+    @Test
+    public void testCancelWhileIdle() {
+        // Cancel while runlet is not running and verify closure and executor cancellation
+        cancellable.cancel();
+        assertFalse(executor.isExecutionRunning());
+        assertTrue(runlet.isClosed());
+
+        // Ensure a spurious run is ignored.
+        executor.runAsync();
+        executor.runToCompletion(3);
+        assertEquals(2, runlet.getRunsStarted());
+    }
+
+    @Test
+    public void testCancelWhileRunning() {
+        // halt execution in runlet
+        runlet.shouldWaitInRun(true);
+        executor.runAsync();
+        runlet.waitUntilInRun();
+        assertEquals(3, runlet.getRunsStarted());
+        assertEquals(2, runlet.getRunsCompleted());
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+
+        // Cancel now
+        cancellable.cancel();
+        assertTrue(executor.isExecutionRunning());
+        assertFalse(runlet.isClosed());
+
+        // Complete the runlet.run(), and verify the close and executor cancellation takes effect
+        runlet.shouldWaitInRun(false);
+        executor.waitUntilRunCompleted(3);
+        assertFalse(executor.isExecutionRunning());
+        assertTrue(runlet.isClosed());
+
+        // Ensure a spurious run is ignored.
+        executor.runToCompletion(4);
+        assertEquals(3, runlet.getRunsStarted());
+    }
+}
+\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java
new file mode 100644
index 00000000000..9828d6300ed
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/RunletExecutorImplTest.java
@@ -0,0 +1,71 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import org.junit.After;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author hakonhall
+ */
+public class RunletExecutorImplTest {
+    private final RunletExecutorImpl executor = new RunletExecutorImpl(2);
+
+    @After
+    public void tearDown() {
+        executor.close();
+    }
+
+    @Test
+    public void testAFewCancellations() {
+        for (int i = 0; i < 10; ++i) {
+            TestRunlet runlet = new TestRunlet();
+            Cancellable cancellable = schedule(runlet);
+            runlet.waitUntilCompleted(5);
+            cancellable.cancel();
+            runlet.waitUntilClosed();
+        }
+    }
+
+    @Test
+    public void testCongestedThreadPool() {
+        TestRunlet runlet1 = new TestRunlet();
+        runlet1.shouldWaitInRun(true);
+        Cancellable cancellable1 = schedule(runlet1);
+        runlet1.waitUntilInRun();
+
+        TestRunlet runlet2 = new TestRunlet();
+        runlet2.shouldWaitInRun(true);
+        Cancellable cancellable2 = schedule(runlet2);
+        runlet2.waitUntilInRun();
+
+        TestRunlet runlet3 = new TestRunlet();
+        Cancellable cancellable3 = schedule(runlet3);
+        try { Thread.sleep(10); } catch (InterruptedException ignored) { }
+        assertEquals(0, runlet3.getRunsStarted());
+
+        cancellable3.cancel();
+        assertTrue(runlet3.isClosed());
+        assertEquals(0, runlet3.getRunsStarted());
+
+        runlet1.shouldWaitInRun(false);
+        runlet2.shouldWaitInRun(false);
+        cancellable1.cancel();
+        cancellable2.cancel();
+    }
+
+    @Test
+    public void testWithoutCancellation() {
+        TestRunlet runlet = new TestRunlet();
+        Cancellable toBeIgnored = schedule(runlet);
+        runlet.waitUntilCompleted(2);
+    }
+
+    private Cancellable schedule(Runlet runlet) {
+        return executor.scheduleWithFixedDelay(runlet, Duration.ofMillis(20));
+    }
+}
+\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java
new file mode 100644
index 00000000000..c40fc03ea00
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestExecutor.java
@@ -0,0 +1,105 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author hakonhall
+ */
+public class TestExecutor implements RunletExecutor {
+    private List<Thread> threads = new ArrayList<>();
+
+    private Runlet runlet;
+    private CancellableImpl cancellable;
+
+    private final Object monitor = new Object();
+    private boolean afterRun = false;
+    private boolean waitAfterRun = false;
+    private int runsCompleted = 0;
+
+    private final Runnable cancelExecution = () -> executionRunning = false;
+    private volatile boolean executionRunning = true;
+
+    @Override
+    public Cancellable scheduleWithFixedDelay(Runlet runlet, Duration delay) {
+        if (this.runlet != null) {
+            throw new IllegalStateException("TestExecutor only supports execution of one runlet");
+        }
+
+        this.runlet = runlet;
+        this.cancellable = new CancellableImpl(runlet);
+        this.cancellable.setPeriodicExecutionCancellationCallback(cancelExecution);
+        return this::cancel;
+    }
+
+    private void cancel() {
+        cancellable.cancel();
+    }
+
+    boolean isExecutionRunning() {
+        return executionRunning;
+    }
+
+    void runAsync() {
+        Thread thread = new Thread(this::threadMain);
+        thread.start();
+        threads.add(thread);
+    }
+
+    void runToCompletion(int run) {
+        runAsync();
+        waitUntilRunCompleted(run);
+    }
+
+    private void threadMain() {
+        cancellable.run();
+
+        synchronized (monitor) {
+            ++runsCompleted;
+            afterRun = true;
+            monitor.notifyAll();
+
+            while (waitAfterRun) {
+                monitor.notifyAll();
+            }
+            afterRun = false;
+        }
+    }
+
+    void setWaitAfterRun(boolean waitAfterRun) {
+        synchronized (monitor) {
+            this.waitAfterRun = waitAfterRun;
+        }
+    }
+
+    void waitUntilAfterRun() {
+        synchronized (monitor) {
+            while (!afterRun) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void waitUntilRunCompleted(int run) {
+        synchronized (monitor) {
+            while (runsCompleted < run) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void uncheckedWait() {
+        try {
+            monitor.wait();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void close() {
+        threads.forEach(thread -> { try { thread.join(); } catch (InterruptedException ignored) {} });
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java
new file mode 100644
index 00000000000..7e671dccd96
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/executor/TestRunlet.java
@@ -0,0 +1,98 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.executor;
+
+/**
+ * @author hakonhall
+ */
+public class TestRunlet implements Runlet {
+    private final Object monitor = new Object();
+    private boolean running = false;
+    private boolean shouldWaitInRun = false;
+    private boolean closed = false;
+    private int runsStarted = 0;
+    private int runsCompleted = 0;
+
+    int getRunsStarted() {
+        synchronized (monitor) {
+            return runsStarted;
+        }
+    }
+
+    int getRunsCompleted() {
+        return runsCompleted;
+    }
+
+    boolean isClosed() {
+        synchronized (monitor) {
+            return closed;
+        }
+    }
+
+    void shouldWaitInRun(boolean value) {
+        synchronized (monitor) {
+            shouldWaitInRun = value;
+            monitor.notifyAll();
+        }
+    }
+
+    void waitUntilInRun() {
+        synchronized (monitor) {
+            while (!running) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void waitUntilCompleted(int runsCompleted) {
+        synchronized (monitor) {
+            while (this.runsCompleted < runsCompleted) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    void waitUntilClosed() {
+        synchronized (monitor) {
+            while (!closed) {
+                uncheckedWait();
+            }
+        }
+    }
+
+    @Override
+    public void run() {
+        synchronized (monitor) {
+            if (closed) {
+                throw new IllegalStateException("run after close");
+            }
+
+            ++runsStarted;
+            running = true;
+            monitor.notifyAll();
+
+            while (shouldWaitInRun) {
+                uncheckedWait();
+            }
+
+            ++runsCompleted;
+            running = false;
+            monitor.notifyAll();
+        }
+    }
+
+    @Override
+    public void close() {
+        synchronized (monitor) {
+            closed = true;
+            monitor.notifyAll();
+        }
+    }
+
+    private void uncheckedWait() {
+        try {
+            monitor.wait();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
index 0dfca12099e..821f5282998 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java
@@ -1,47 +1,92 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
+import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
+import com.yahoo.vespa.service.model.ServiceId;
 import com.yahoo.vespa.service.monitor.ConfigserverUtil;
 import org.junit.Test;
 
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
-import java.util.function.Function;
 
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class ApplicationHealthMonitorTest {
     private final ConfigServerApplication configServerApplication = new ConfigServerApplication();
 
     @Test
-    public void sanityCheck() {
-        MonitorFactory monitorFactory = new MonitorFactory();
-
+    public void activationAndRemoval() {
         HealthMonitor monitor1 = mock(HealthMonitor.class);
         HealthMonitor monitor2 = mock(HealthMonitor.class);
         HealthMonitor monitor3 = mock(HealthMonitor.class);
 
-        monitorFactory.expectEndpoint("http://cfg1:19071/state/v1/health", monitor1);
-        monitorFactory.expectEndpoint("http://cfg2:19071/state/v1/health", monitor2);
-        monitorFactory.expectEndpoint("http://cfg3:19071/state/v1/health", monitor3);
+        ApplicationInfo configServer = ConfigserverUtil.makeExampleConfigServer();
+        StateV1HealthModel model = mock(StateV1HealthModel.class);
+        ApplicationHealthMonitor applicationMonitor = new ApplicationHealthMonitor(configServer.getApplicationId(), model);
+
+        // Activate with cfg1-2
+        HealthEndpoint endpoint1 = mock(HealthEndpoint.class);
+        HealthEndpoint endpoint2 = mock(HealthEndpoint.class);
+        Map<ServiceId, HealthEndpoint> initialEndpoints = new HashMap<>();
+        initialEndpoints.put(serviceIdOf("cfg1"), endpoint1);
+        initialEndpoints.put(serviceIdOf("cfg2"), endpoint2);
+
+        when(model.extractHealthEndpoints(configServer)).thenReturn(initialEndpoints);
+        when(endpoint1.startMonitoring()).thenReturn(monitor1);
+        when(endpoint2.startMonitoring()).thenReturn(monitor2);
+        applicationMonitor.monitor(configServer);
+
+        verify(endpoint1, times(1)).startMonitoring();
+        verify(endpoint2, times(1)).startMonitoring();
 
         when(monitor1.getStatus()).thenReturn(ServiceStatus.UP);
         when(monitor2.getStatus()).thenReturn(ServiceStatus.DOWN);
-        when(monitor3.getStatus()).thenReturn(ServiceStatus.NOT_CHECKED);
-
-        ApplicationHealthMonitor applicationMonitor = ApplicationHealthMonitor.startMonitoring(
-                ConfigserverUtil.makeExampleConfigServer(),
-                monitorFactory);
+        when(monitor3.getStatus()).thenReturn(ServiceStatus.UP);
 
         assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg1"));
         assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2"));
         assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg3"));
+
+        // Update application to contain cfg2-3
+        HealthEndpoint endpoint3 = mock(HealthEndpoint.class);
+        when(endpoint3.startMonitoring()).thenReturn(monitor3);
+        Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>();
+        endpoints.put(serviceIdOf("cfg2"), endpoint2);
+        endpoints.put(serviceIdOf("cfg3"), endpoint3);
+        when(model.extractHealthEndpoints(configServer)).thenReturn(endpoints);
+        applicationMonitor.monitor(configServer);
+
+        // Only monitor1 has been removed and had its close called
+        verify(monitor1, times(1)).close();
+        verify(monitor2, never()).close();
+        verify(monitor3, never()).close();
+
+        // Only endpoint3 started monitoring from last monitor()
+        verify(endpoint1, times(1)).startMonitoring();
+        verify(endpoint2, times(1)).startMonitoring();
+        verify(endpoint3, times(1)).startMonitoring();
+
+        // Now cfg1 will be NOT_CHECKED, while cfg3 should be UP.
+        assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg1"));
+        assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2"));
+        assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg3"));
+
+        applicationMonitor.close();
+    }
+
+    private ServiceId serviceIdOf(String hostname) {
+        return new ServiceId(configServerApplication.getApplicationId(),
+                configServerApplication.getClusterId(),
+                configServerApplication.getServiceType(),
+                configServerApplication.configIdFor(HostName.from(hostname)));
     }
 
     private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) {
@@ -51,70 +96,4 @@ public class ApplicationHealthMonitorTest {
                 configServerApplication.getServiceType(),
                 configServerApplication.configIdFor(HostName.from(hostname)));
     }
-
-    private static class MonitorFactory implements Function<HealthEndpoint, HealthMonitor> {
-        private Map<String, EndpointInfo> endpointMonitors = new HashMap<>();
-
-        public void expectEndpoint(String url, HealthMonitor monitorToReturn) {
-            endpointMonitors.put(url, new EndpointInfo(url, monitorToReturn));
-        }
-
-        @Override
-        public HealthMonitor apply(HealthEndpoint endpoint) {
-            String url = endpoint.getStateV1HealthUrl().toString();
-            EndpointInfo info = endpointMonitors.get(url);
-            if (info == null) {
-                throw new IllegalArgumentException("Endpoint not expected: " + url);
-            }
-
-            if (info.isEndpointDiscovered()) {
-                throw new IllegalArgumentException("A HealthMonitor has already been created to " + url);
-            }
-
-            info.setEndpointDiscovered(true);
-
-            return info.getMonitorToReturn();
-        }
-    }
-
-    private static class EndpointInfo {
-        private final String url;
-        private final HealthMonitor monitorToReturn;
-
-        private boolean endpointDiscovered = false;
-
-        private EndpointInfo(String url, HealthMonitor monitorToReturn) {
-            this.url = url;
-            this.monitorToReturn = monitorToReturn;
-        }
-
-        public String getUrl() {
-            return url;
-        }
-
-        public boolean isEndpointDiscovered() {
-            return endpointDiscovered;
-        }
-
-        public void setEndpointDiscovered(boolean endpointDiscovered) {
-            this.endpointDiscovered = endpointDiscovered;
-        }
-
-        public HealthMonitor getMonitorToReturn() {
-            return monitorToReturn;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            EndpointInfo that = (EndpointInfo) o;
-            return Objects.equals(url, that.url);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(url);
-        }
-    }
 }
 \ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
index f420f5c1284..86b0ee4a8f3 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
@@ -49,7 +49,9 @@ public class HealthMonitorManagerTest {
         when(monitorInfra.value()).thenReturn(false);
         ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer();
         manager.applicationActivated(applicationInfo);
+        verify(monitor, times(1)).monitor(applicationInfo);
         manager.applicationRemoved(applicationInfo.getApplicationId());
+        verify(monitor, times(1)).close();
     }
 
     @Test
@@ -73,7 +75,7 @@ public class HealthMonitorManagerTest {
         ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
 
         manager.applicationActivated(proxyHostApplicationInfo);
-        verify(monitorFactory, never()).create(proxyHostApplicationInfo);
+        verify(monitorFactory, never()).create(proxyHostApplicationInfo.getApplicationId());
 
         assertStatus(ServiceStatus.NOT_CHECKED, 0, proxyHostApplication, "proxyhost1");
     }
@@ -88,7 +90,7 @@ public class HealthMonitorManagerTest {
         ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
 
         manager.applicationActivated(proxyHostApplicationInfo);
-        verify(monitorFactory, times(1)).create(proxyHostApplicationInfo);
+        verify(monitorFactory, times(1)).create(proxyHostApplicationInfo.getApplicationId());
 
         when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.UP);
         assertStatus(ServiceStatus.UP, 1, proxyHostApplication, "proxyhost1");
@@ -98,6 +100,11 @@ public class HealthMonitorManagerTest {
         assertStatus(ServiceStatus.NOT_CHECKED, 0, controllerHostApplication, "controllerhost1");
     }
 
+    @Test
+    public void threadPoolSize() {
+        assertEquals(9, HealthMonitorManager.THREAD_POOL_SIZE);
+    }
+
     private void assertStatus(ServiceStatus expected, int verifyTimes, InfraApplication infraApplication, String hostname) {
         ServiceStatus actual = manager.getStatus(
                 infraApplication.getApplicationId(),
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java
deleted file mode 100644
index 94ba4726ad0..00000000000
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.health;
-
-import com.yahoo.vespa.applicationmodel.ServiceStatus;
-import org.junit.Test;
-
-import java.time.Duration;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class HealthMonitorTest {
-    @Test
-    public void initiallyDown() {
-        HealthClient healthClient = mock(HealthClient.class);
-        try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofHours(12))) {
-            monitor.startMonitoring();
-            assertEquals(ServiceStatus.DOWN, monitor.getStatus());
-        }
-    }
-
-    @Test
-    public void eventuallyUp() {
-        HealthClient healthClient = mock(HealthClient.class);
-        when(healthClient.getHealthInfo()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE));
-        try (HealthMonitor monitor = new HealthMonitor(healthClient, Duration.ofMillis(10))) {
-            monitor.startMonitoring();
-
-            while (monitor.getStatus() != ServiceStatus.UP) {
-                try {
-                    Thread.sleep(1);
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-        }
-    }
-}
-\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
new file mode 100644
index 00000000000..480691772bb
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java
@@ -0,0 +1,66 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.service.duper.ProxyHostApplication;
+import com.yahoo.vespa.service.executor.Cancellable;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.model.ServiceId;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+public class StateV1HealthModelTest {
+    private RunletExecutor executor = mock(RunletExecutor.class);
+    private Duration healthStaleness = Duration.ofSeconds(1);
+    private Duration requestTimeout = Duration.ofSeconds(2);
+    private Duration keepAlive = Duration.ofSeconds(3);
+    private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor);
+    private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
+    private final List<HostName> hostnames = Stream.of("host1", "host2").map(HostName::from).collect(Collectors.toList());
+    private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
+    private final Map<ServiceId, HealthEndpoint> endpoints = model.extractHealthEndpoints(proxyHostApplicationInfo);
+
+    @Test
+    public void test() {
+        assertEquals(2, endpoints.size());
+
+        ApplicationId applicationId = ApplicationId.from("hosted-vespa", "proxy-host", "default");
+        ClusterId clusterId = new ClusterId("proxy-host");
+        ServiceId hostAdmin1 = new ServiceId(applicationId, clusterId, ServiceType.HOST_ADMIN, new ConfigId("proxy-host/host1"));
+        ServiceId hostAdmin2 = new ServiceId(applicationId, clusterId, ServiceType.HOST_ADMIN, new ConfigId("proxy-host/host2"));
+
+        HealthEndpoint endpoint1 = endpoints.get(hostAdmin1);
+        assertNotNull(endpoint1);
+        assertEquals("http://host1:8080/state/v1/health", endpoint1.description());
+
+        HealthEndpoint endpoint2 = endpoints.get(hostAdmin2);
+        assertNotNull(endpoint2);
+        assertEquals("http://host2:8080/state/v1/health", endpoint2.description());
+
+        Cancellable cancellable = mock(Cancellable.class);
+        when(executor.scheduleWithFixedDelay(any(), any())).thenReturn(cancellable);
+        try (HealthMonitor healthMonitor = endpoint1.startMonitoring()) {
+            assertEquals(ServiceStatus.DOWN, healthMonitor.getStatus());
+        }
+    }
+}
+\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java
new file mode 100644
index 00000000000..c892118990f
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthMonitorTest.java
@@ -0,0 +1,37 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.vespa.applicationmodel.ServiceStatus;
+import com.yahoo.vespa.service.executor.RunletExecutor;
+import com.yahoo.vespa.service.executor.RunletExecutorImpl;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class StateV1HealthMonitorTest {
+    @Test
+    public void downThenUpThenDown() throws Exception {
+        StateV1HealthClient client = mock(StateV1HealthClient.class);
+        when(client.get()).thenReturn(HealthInfo.empty());
+
+        StateV1HealthUpdater updater = new StateV1HealthUpdater(client);
+        RunletExecutor executor = new RunletExecutorImpl(2);
+        try (StateV1HealthMonitor monitor = new StateV1HealthMonitor(updater, executor, Duration.ofMillis(10))) {
+            assertEquals(ServiceStatus.DOWN, monitor.getStatus());
+
+            when(client.get()).thenReturn(HealthInfo.fromHealthStatusCode(HealthInfo.UP_STATUS_CODE));
+            while (monitor.getStatus() != ServiceStatus.UP) {
+                try { Thread.sleep(2); } catch (InterruptedException ignored) { }
+            }
+
+            when(client.get()).thenReturn(HealthInfo.fromException(new IllegalStateException("foo")));
+            while (monitor.getStatus() != ServiceStatus.DOWN) {
+                try { Thread.sleep(2); } catch (InterruptedException ignored) { }
+            }
+        }
+    }
+}
+\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthClientTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java
index 157b5565071..e7b7a829dac 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthClientTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthUpdaterTest.java
@@ -1,16 +1,18 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.health;
 
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import org.apache.http.HttpEntity;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.net.URL;
+import java.util.function.Function;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -19,7 +21,14 @@ import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class HealthClientTest {
+public class StateV1HealthUpdaterTest {
+    private URL url;
+
+    @Before
+    public void setUp() throws Exception{
+        url = new URL("http://host.com:19071");
+    }
+
     @Test
     public void successfulRequestResponse() throws IOException {
         HealthInfo info = getHealthInfoFromJsonResponse("{\n" +
@@ -96,7 +105,6 @@ public class HealthClientTest {
 
     private HealthInfo getHealthInfoFromJsonResponse(String content)
             throws IOException {
-        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
         CloseableHttpClient client = mock(CloseableHttpClient.class);
 
         CloseableHttpResponse response = mock(CloseableHttpResponse.class);
@@ -110,22 +118,22 @@ public class HealthClientTest {
         HttpEntity httpEntity = mock(HttpEntity.class);
         when(response.getEntity()).thenReturn(httpEntity);
 
-        try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> content)) {
-
+        try (StateV1HealthUpdater updater = makeUpdater(client, entry -> content)) {
             when(httpEntity.getContentLength()).thenReturn((long) content.length());
-            return healthClient.getHealthInfo();
+            updater.run();
+            return updater.getLatestHealthInfo();
         }
     }
 
     @Test
     public void testRequestException() throws IOException {
-        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
         CloseableHttpClient client = mock(CloseableHttpClient.class);
 
         when(client.execute(any())).thenThrow(new ConnectTimeoutException("exception string"));
 
-        try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> "")) {
-            HealthInfo info = healthClient.getHealthInfo();
+        try (StateV1HealthUpdater updater = makeUpdater(client, entry -> "")) {
+            updater.run();
+            HealthInfo info = updater.getLatestHealthInfo();
             assertFalse(info.isHealthy());
             assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
             assertEquals("Exception: exception string", info.toString());
@@ -135,7 +143,6 @@ public class HealthClientTest {
     @Test
     public void testBadHttpResponseCode()
             throws IOException {
-        HealthEndpoint endpoint = HealthEndpoint.forHttp(HostName.from("host.com"), 19071);
         CloseableHttpClient client = mock(CloseableHttpClient.class);
 
         CloseableHttpResponse response = mock(CloseableHttpResponse.class);
@@ -150,13 +157,19 @@ public class HealthClientTest {
         when(response.getEntity()).thenReturn(httpEntity);
 
         String content = "{}";
-        try (HealthClient healthClient = new HealthClient(endpoint, client, entry -> content)) {
-
+        try (HealthUpdater updater = makeUpdater(client, entry -> content)) {
             when(httpEntity.getContentLength()).thenReturn((long) content.length());
-            HealthInfo info = healthClient.getHealthInfo();
+            updater.run();
+            HealthInfo info = updater.getLatestHealthInfo();
             assertFalse(info.isHealthy());
             assertEquals(ServiceStatus.DOWN, info.toServiceStatus());
             assertEquals("Bad HTTP response status code 500", info.toString());
         }
     }
+
+    private StateV1HealthUpdater makeUpdater(CloseableHttpClient client, Function<HttpEntity, String> getContentFunction) {
+        ApacheHttpClient apacheHttpClient = new ApacheHttpClient(url, client);
+        StateV1HealthClient healthClient = new StateV1HealthClient(apacheHttpClient, getContentFunction);
+        return new StateV1HealthUpdater(healthClient);
+    }
 }
 \ No newline at end of file