Monitor health of host infra applications

7 files changed, 135 insertions, 26 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
index cc8dc7f8df7..e0e2b64bdae 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
@@ -32,13 +32,14 @@ public class DuperModelManager implements DuperModelInfraApi {
     private static Logger logger = Logger.getLogger(DuperModelManager.class.getName());
 
     // Infrastructure applications
+    private final ConfigServerHostApplication configServerHostApplication = new ConfigServerHostApplication();
+    private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
+    private final ControllerApplication controllerApplication = new ControllerApplication();
+    private final ControllerHostApplication controllerHostApplication = new ControllerHostApplication();
+    // this must be static to be referenced in this(). Remove static once legacy config server from config is gone.
     private static final ConfigServerApplication configServerApplication = new ConfigServerApplication();
-    private static final ConfigServerHostApplication configServerHostApplication = new ConfigServerHostApplication();
-    private static final ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
-    private static final ControllerApplication controllerApplication = new ControllerApplication();
-    private static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication();
 
-    private static final Map<ApplicationId, InfraApplication> supportedInfraApplications = Stream.of(
+    private final Map<ApplicationId, InfraApplication> supportedInfraApplications = Stream.of(
             configServerApplication,
             configServerHostApplication,
             proxyHostApplication,
@@ -55,16 +56,6 @@ public class DuperModelManager implements DuperModelInfraApi {
     // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons.
     private final Set<ApplicationId> activeInfraInfos = new HashSet<>(2 * supportedInfraApplications.size());
 
-    /**
-     * Returns true if application is considered an infrastructure application by the DuperModel.
-     *
-     * <p>Note: The tenant host "application" is NOT considered an infrastructure application: It is just a
-     * cluster in the {@link ZoneApplication zone application}.
-     */
-    public static boolean isInfrastructureApplication(ApplicationId applicationId) {
-        return supportedInfraApplications.containsKey(applicationId);
-    }
-
     @Inject
     public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) {
         this(
@@ -148,6 +139,16 @@ public class DuperModelManager implements DuperModelInfraApi {
         return new ArrayList<>(supportedInfraApplications.values());
     }
 
+    /**
+     * Returns true if application is considered an infrastructure application by the DuperModel.
+     *
+     * <p>Note: The tenant host "application" is NOT considered an infrastructure application: It is just a
+     * cluster in the {@link ZoneApplication zone application}.
+     */
+    public boolean isSupportedInfraApplication(ApplicationId applicationId) {
+        return supportedInfraApplications.containsKey(applicationId);
+    }
+
     @Override
     public boolean infraApplicationIsActive(ApplicationId applicationId) {
         synchronized (monitor) {
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
new file mode 100644
index 00000000000..43be236268c
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java
@@ -0,0 +1,12 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.health;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+
+/**
+ * @author hakonhall
+ */
+@FunctionalInterface
+interface ApplicationHealthMonitorFactory {
+    ApplicationHealthMonitor create(ApplicationInfo applicationInfo);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index 91f9f313fe4..e9a5ec314f6 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -8,6 +8,8 @@ import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.flags.FeatureFlag;
+import com.yahoo.vespa.flags.FileFlagSource;
 import com.yahoo.vespa.service.duper.DuperModelManager;
 import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.manager.MonitorManager;
@@ -22,16 +24,28 @@ import java.util.concurrent.ConcurrentHashMap;
 public class HealthMonitorManager implements MonitorManager {
     private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = new ConcurrentHashMap<>();
     private final DuperModelManager duperModel;
+    private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory;
+    private final FeatureFlag monitorInfra;
 
     @Inject
-    public HealthMonitorManager(DuperModelManager duperModel) {
+    public HealthMonitorManager(DuperModelManager duperModel, FileFlagSource flagSource) {
+        this(duperModel,
+                new FeatureFlag("healthmonitor-monitorinfra", true, flagSource),
+                ApplicationHealthMonitor::startMonitoring);
+    }
+
+    HealthMonitorManager(DuperModelManager duperModel,
+                         FeatureFlag monitorInfra,
+                         ApplicationHealthMonitorFactory applicationHealthMonitorFactory) {
         this.duperModel = duperModel;
+        this.applicationHealthMonitorFactory = applicationHealthMonitorFactory;
+        this.monitorInfra = monitorInfra;
     }
 
     @Override
     public void applicationActivated(ApplicationInfo application) {
         if (wouldMonitor(application.getApplicationId())) {
-            ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring(application);
+            ApplicationHealthMonitor monitor = applicationHealthMonitorFactory.create(application);
             healthMonitors.put(application.getApplicationId(), monitor);
         }
     }
@@ -65,6 +79,10 @@ public class HealthMonitorManager implements MonitorManager {
 
     @Override
     public boolean wouldMonitor(ApplicationId id) {
+        if (duperModel.isSupportedInfraApplication(id) && monitorInfra.value()) {
+            return true;
+        }
+
         if (id.equals(duperModel.getConfigServerApplication().getApplicationId())) {
             return true;
         }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java
index b695b0b19f8..c0bcff7284c 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java
@@ -22,9 +22,9 @@ public class ServiceId {
     private final ConfigId configId;
 
     public ServiceId(ApplicationId applicationId,
-              ClusterId clusterId,
-              ServiceType serviceType,
-              ConfigId configId) {
+                     ClusterId clusterId,
+                     ServiceType serviceType,
+                     ConfigId configId) {
         this.applicationId = applicationId;
         this.clusterId = clusterId;
         this.serviceType = serviceType;
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
index 39da5040ca0..13f24bc3694 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
@@ -28,14 +28,16 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager {
 
     private final Object monitor = new Object();
     private final HashMap<ApplicationId, SlobrokMonitor> slobrokMonitors = new HashMap<>();
+    private final DuperModelManager duperModel;
 
     @Inject
-    public SlobrokMonitorManagerImpl() {
-        this(SlobrokMonitor::new);
+    public SlobrokMonitorManagerImpl(DuperModelManager duperModel) {
+        this(SlobrokMonitor::new, duperModel);
     }
 
-    SlobrokMonitorManagerImpl(Supplier<SlobrokMonitor> slobrokMonitorFactory) {
+    SlobrokMonitorManagerImpl(Supplier<SlobrokMonitor> slobrokMonitorFactory, DuperModelManager duperModel) {
         this.slobrokMonitorFactory = slobrokMonitorFactory;
+        this.duperModel = duperModel;
     }
 
     @Override
@@ -108,7 +110,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager {
 
     @Override
     public boolean wouldMonitor(ApplicationId applicationId) {
-        if (DuperModelManager.isInfrastructureApplication(applicationId)) {
+        if (duperModel.isSupportedInfraApplication(applicationId)) {
             return false;
         }
 
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
index 5195bf218b6..f420f5c1284 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java
@@ -2,33 +2,51 @@
 package com.yahoo.vespa.service.health;
 
 import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.flags.FeatureFlag;
 import com.yahoo.vespa.service.duper.ConfigServerApplication;
+import com.yahoo.vespa.service.duper.ControllerHostApplication;
 import com.yahoo.vespa.service.duper.DuperModelManager;
+import com.yahoo.vespa.service.duper.InfraApplication;
+import com.yahoo.vespa.service.duper.ProxyHostApplication;
 import com.yahoo.vespa.service.duper.ZoneApplication;
 import com.yahoo.vespa.service.monitor.ConfigserverUtil;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class HealthMonitorManagerTest {
     private final ConfigServerApplication configServerApplication = new ConfigServerApplication();
     private final DuperModelManager duperModel = mock(DuperModelManager.class);
-    private final HealthMonitorManager manager = new HealthMonitorManager(duperModel);
+    private final FeatureFlag monitorInfra = mock(FeatureFlag.class);
+    private final ApplicationHealthMonitor monitor = mock(ApplicationHealthMonitor.class);
+    private final ApplicationHealthMonitorFactory monitorFactory = mock(ApplicationHealthMonitorFactory.class);
+    private final HealthMonitorManager manager = new HealthMonitorManager(duperModel, monitorInfra, monitorFactory);
 
     @Before
     public void setUp() {
         when(duperModel.getConfigServerApplication()).thenReturn(configServerApplication);
+        when(monitorFactory.create(any())).thenReturn(monitor);
     }
 
     @Test
     public void addRemove() {
+        when(monitorInfra.value()).thenReturn(false);
         ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer();
         manager.applicationActivated(applicationInfo);
         manager.applicationRemoved(applicationInfo.getApplicationId());
@@ -36,6 +54,7 @@ public class HealthMonitorManagerTest {
 
     @Test
     public void withHostAdmin() {
+        when(monitorInfra.value()).thenReturn(false);
         ServiceStatus status = manager.getStatus(
                 ZoneApplication.ZONE_APPLICATION_ID,
                 ClusterId.NODE_ADMIN,
@@ -43,4 +62,56 @@ public class HealthMonitorManagerTest {
                 new ConfigId("config-id-1"));
         assertEquals(ServiceStatus.UP, status);
     }
+
+    @Test
+    public void infrastructureApplication() {
+        when(monitorInfra.value()).thenReturn(false);
+
+        ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
+        when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true);
+        List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList());
+        ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
+
+        manager.applicationActivated(proxyHostApplicationInfo);
+        verify(monitorFactory, never()).create(proxyHostApplicationInfo);
+
+        assertStatus(ServiceStatus.NOT_CHECKED, 0, proxyHostApplication, "proxyhost1");
+    }
+
+    @Test
+    public void infrastructureApplicationWithMonitoring() {
+        when(monitorInfra.value()).thenReturn(true);
+
+        ProxyHostApplication proxyHostApplication = new ProxyHostApplication();
+        when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true);
+        List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList());
+        ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames);
+
+        manager.applicationActivated(proxyHostApplicationInfo);
+        verify(monitorFactory, times(1)).create(proxyHostApplicationInfo);
+
+        when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.UP);
+        assertStatus(ServiceStatus.UP, 1, proxyHostApplication, "proxyhost1");
+
+        ControllerHostApplication controllerHostApplication = new ControllerHostApplication();
+        when(duperModel.isSupportedInfraApplication(controllerHostApplication.getApplicationId())).thenReturn(true);
+        assertStatus(ServiceStatus.NOT_CHECKED, 0, controllerHostApplication, "controllerhost1");
+    }
+
+    private void assertStatus(ServiceStatus expected, int verifyTimes, InfraApplication infraApplication, String hostname) {
+        ServiceStatus actual = manager.getStatus(
+                infraApplication.getApplicationId(),
+                infraApplication.getClusterId(),
+                infraApplication.getServiceType(),
+                infraApplication.configIdFor(HostName.from(hostname)));
+
+        assertEquals(expected, actual);
+
+        verify(monitor, times(verifyTimes)).getStatus(
+                infraApplication.getApplicationId(),
+                infraApplication.getClusterId(),
+                infraApplication.getServiceType(),
+                infraApplication.configIdFor(HostName.from(hostname)));
+
+    }
 }
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java
index c3ee1d078bf..bc6bff6fba8 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java
@@ -7,6 +7,8 @@ import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.service.duper.ConfigServerApplication;
+import com.yahoo.vespa.service.duper.DuperModelManager;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -25,8 +27,10 @@ public class SlobrokMonitorManagerImplTest {
     @SuppressWarnings("unchecked")
     private final Supplier<SlobrokMonitor> slobrokMonitorFactory = mock(Supplier.class);
 
+    private final ConfigServerApplication configServerApplication = new ConfigServerApplication();
+    private final DuperModelManager duperModelManager = mock(DuperModelManager.class);
     private final SlobrokMonitorManagerImpl slobrokMonitorManager =
-            new SlobrokMonitorManagerImpl(slobrokMonitorFactory);
+            new SlobrokMonitorManagerImpl(slobrokMonitorFactory, duperModelManager);
     private final SlobrokMonitor slobrokMonitor = mock(SlobrokMonitor.class);
     private final ApplicationId applicationId = ApplicationId.from("tenant", "app", "instance");
     private final ApplicationInfo application = mock(ApplicationInfo.class);
@@ -34,6 +38,7 @@ public class SlobrokMonitorManagerImplTest {
 
     @Before
     public void setup() {
+        when(duperModelManager.getConfigServerApplication()).thenReturn(configServerApplication);
         when(slobrokMonitorFactory.get()).thenReturn(slobrokMonitor);
         when(application.getApplicationId()).thenReturn(applicationId);
     }