One SlobrokMonitor2 per application

11 files changed, 513 insertions, 200 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java
index bee34524b20..ed40dcc675a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java
@@ -5,6 +5,7 @@ import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.model.api.SuperModel;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
@@ -40,14 +41,16 @@ public class ModelGenerator {
             SuperModel superModel,
             Zone zone,
             List<String> configServerHosts,
-            SlobrokMonitor2 slobrokMonitor) {
-        Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> applicationInstances = new HashMap<>();
+            SlobrokMonitorManager slobrokMonitorManager) {
+        Map<ApplicationInstanceReference,
+                ApplicationInstance<ServiceMonitorStatus>> applicationInstances = new HashMap<>();
 
         for (ApplicationInfo applicationInfo : superModel.getAllApplicationInfos()) {
+
             ApplicationInstance<ServiceMonitorStatus> applicationInstance = toApplicationInstance(
                     applicationInfo,
                     zone,
-                    slobrokMonitor);
+                    slobrokMonitorManager);
             applicationInstances.put(applicationInstance.reference(), applicationInstance);
         }
 
@@ -65,7 +68,7 @@ public class ModelGenerator {
     ApplicationInstance<ServiceMonitorStatus> toApplicationInstance(
             ApplicationInfo applicationInfo,
             Zone zone,
-            SlobrokMonitor2 slobrokMonitor) {
+            SlobrokMonitorManager slobrokMonitorManager) {
         Map<ServiceClusterKey, Set<ServiceInstance<ServiceMonitorStatus>>> groupedServiceInstances = new HashMap<>();
 
         for (HostInfo host : applicationInfo.getModel().getHosts()) {
@@ -73,7 +76,11 @@ public class ModelGenerator {
             for (ServiceInfo serviceInfo : host.getServices()) {
                 ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo);
                 ServiceInstance<ServiceMonitorStatus> serviceInstance =
-                        toServiceInstance(serviceInfo, hostName, slobrokMonitor);
+                        toServiceInstance(
+                                applicationInfo.getApplicationId(),
+                                serviceInfo,
+                                hostName,
+                                slobrokMonitorManager);
 
                 if (!groupedServiceInstances.containsKey(serviceClusterKey)) {
                     groupedServiceInstances.put(serviceClusterKey, new HashSet<>());
@@ -104,12 +111,18 @@ public class ModelGenerator {
     }
 
     ServiceInstance<ServiceMonitorStatus> toServiceInstance(
+            ApplicationId applicationId,
             ServiceInfo serviceInfo,
             HostName hostName,
-            SlobrokMonitor2 slobrokMonitor) {
+            SlobrokMonitorManager slobrokMonitorManager) {
         ConfigId configId = new ConfigId(serviceInfo.getConfigId());
-        ServiceMonitorStatus serviceStatus = slobrokMonitor.getStatus(toServiceType(serviceInfo), configId);
-        return new ServiceInstance<>(configId, hostName,serviceStatus);
+
+        ServiceMonitorStatus status = slobrokMonitorManager.getStatus(
+                applicationId,
+                toServiceType(serviceInfo),
+                configId);
+
+        return new ServiceInstance<>(configId, hostName, status);
     }
 
     ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) {
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java
index c3ce129e89a..f1d8e31d26c 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitorImpl.java
@@ -18,16 +18,16 @@ public class ServiceMonitorImpl implements ServiceMonitor {
     private static final Logger logger = Logger.getLogger(ServiceMonitorImpl.class.getName());
 
     private final Zone zone;
-    private final List<String> configServerHostnames;
-    private final SlobrokMonitor2 slobrokMonitor = new SlobrokMonitor2();
+    private final List<String> configServerHosts;
+    private final SlobrokMonitorManager slobrokMonitorManager = new SlobrokMonitorManager();
     private final SuperModelListenerImpl superModelListener =
-            new SuperModelListenerImpl(slobrokMonitor);
+            new SuperModelListenerImpl(slobrokMonitorManager);
 
     @Inject
     public ServiceMonitorImpl(SuperModelProvider superModelProvider,
                               ConfigserverConfig configserverConfig) {
         this.zone = superModelProvider.getZone();
-        this.configServerHostnames = toConfigServerList(configserverConfig);
+        this.configServerHosts = toConfigServerList(configserverConfig);
         superModelListener.start(superModelProvider);
     }
 
@@ -47,7 +47,7 @@ public class ServiceMonitorImpl implements ServiceMonitor {
         // If we ever need to optimize this method, then consider reusing ServiceModel snapshots
         // for up to X ms.
         ServiceModel serviceModel =
-                superModelListener.createServiceModelSnapshot(zone, configServerHostnames);
+                superModelListener.createServiceModelSnapshot(zone, configServerHosts);
         return serviceModel.getAllApplicationInstances();
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java
index b463a50089a..8737d915bcf 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java
@@ -5,28 +5,23 @@ import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.PortInfo;
 import com.yahoo.config.model.api.ServiceInfo;
-import com.yahoo.config.model.api.SuperModel;
 import com.yahoo.jrt.Spec;
 import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Transport;
 import com.yahoo.jrt.slobrok.api.Mirror;
 import com.yahoo.jrt.slobrok.api.SlobrokList;
-import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.applicationmodel.ConfigId;
-import com.yahoo.vespa.applicationmodel.ServiceType;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
-import java.util.logging.Logger;
 
+/**
+ * Class to manage Slobrok
+ */
 public class SlobrokMonitor2 implements AutoCloseable {
     public static final String SLOBROK_SERVICE_TYPE = "slobrok";
     public static final String SLOBROK_RPC_PORT_TAG = "rpc";
 
-    private static final Logger log = Logger.getLogger(SlobrokMonitor2.class.getName());
-
     private final SlobrokList slobrokList;
     private final Mirror mirror;
 
@@ -44,84 +39,39 @@ public class SlobrokMonitor2 implements AutoCloseable {
         this(slobrokList, new Mirror(new Supervisor(new Transport()), slobrokList));
     }
 
-    void updateSlobrokList(SuperModel superModel) {
-        // If we ever need to optimize this method, then we should make this class
-        // have a Map<ApplicationId, List<String>>, mapping each application to
-        // its list of specs. Then, whenever a single application is activated or removed,
-        // only modify that List<String>.
+    void updateSlobrokList(ApplicationInfo application) {
+        List<String> slobrokSpecs = getSlobrokSpecs(application);
+        slobrokList.setup(slobrokSpecs.toArray(new String[0]));
+    }
 
+    List<String> getSlobrokSpecs(ApplicationInfo applicationInfo) {
         List<String> slobrokSpecs = new ArrayList<>();
 
-        for (ApplicationInfo application : superModel.getAllApplicationInfos()) {
-            for (HostInfo host : application.getModel().getHosts()) {
-                for (ServiceInfo service : host.getServices()) {
-                    if (!Objects.equals(service.getServiceType(), SLOBROK_SERVICE_TYPE)) {
-                        continue;
-                    }
+        for (HostInfo host : applicationInfo.getModel().getHosts()) {
+            for (ServiceInfo service : host.getServices()) {
+                if (!Objects.equals(service.getServiceType(), SLOBROK_SERVICE_TYPE)) {
+                    continue;
+                }
 
-                    for (PortInfo port : service.getPorts()) {
-                        if (port.getTags().contains(SLOBROK_RPC_PORT_TAG)) {
-                            Spec spec = new Spec(host.getHostname(), port.getPort());
-                            slobrokSpecs.add(spec.toString());
-                        }
+                for (PortInfo port : service.getPorts()) {
+                    if (port.getTags().contains(SLOBROK_RPC_PORT_TAG)) {
+                        Spec spec = new Spec(host.getHostname(), port.getPort());
+                        slobrokSpecs.add(spec.toString());
                     }
                 }
             }
         }
 
-        slobrokList.setup(slobrokSpecs.toArray(new String[0]));
-    }
-
-    ServiceMonitorStatus getStatus(ServiceType serviceType, ConfigId configId) {
-        Optional<String> slobrokServiceName = lookup(serviceType, configId);
-        if (slobrokServiceName.isPresent()) {
-            if (mirror.lookup(slobrokServiceName.get()).length != 0) {
-                return ServiceMonitorStatus.UP;
-            } else {
-                return ServiceMonitorStatus.DOWN;
-            }
-        } else {
-            return ServiceMonitorStatus.NOT_CHECKED;
-        }
+        return slobrokSpecs;
     }
 
     @Override
     public void close() {
+        // TODO: Make sure registeredInSlobrok returns DOWN from now on (concurrently)
         mirror.shutdown();
     }
 
-    // Package-private for testing
-    Optional<String> lookup(ServiceType serviceType, ConfigId configId) {
-        switch (serviceType.s()) {
-            case "adminserver":
-            case "config-sentinel":
-            case "configproxy":
-            case "configserver":
-            case "filedistributorservice":
-            case "logd":
-            case "logserver":
-            case "metricsproxy":
-            case "slobrok":
-            case "transactionlogserver":
-                return Optional.empty();
-
-            case "topleveldispatch":
-                return Optional.of(configId.s());
-
-            case "qrserver":
-            case "container":
-            case "docprocservice":
-            case "container-clustercontroller":
-                return Optional.of("vespa/service/" + configId.s());
-
-            case "searchnode": //TODO: handle only as storagenode instead of both as searchnode/storagenode
-                return Optional.of(configId.s() + "/realtimecontroller");
-            case "distributor":
-            case "storagenode":
-                return Optional.of("storage/cluster." + configId.s());
-            default:
-                log.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() + " with config id " + configId.s());
-                return Optional.empty();
-        }
+    boolean registeredInSlobrok(String slobrokServiceName) {
+        return mirror.lookup(slobrokServiceName).length > 0;
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitorManager.java
new file mode 100644
index 00000000000..95baa9013d1
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitorManager.java
@@ -0,0 +1,114 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.SuperModel;
+import com.yahoo.config.model.api.SuperModelListener;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+
+import java.util.HashMap;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+public class SlobrokMonitorManager implements SuperModelListener {
+    private static final Logger logger =
+            Logger.getLogger(SlobrokMonitorManager.class.getName());
+
+    private final Supplier<SlobrokMonitor2> slobrokMonitorFactory;
+
+    private final Object monitor = new Object();
+    private final HashMap<ApplicationId, SlobrokMonitor2> slobrokMonitors = new HashMap<>();
+
+    SlobrokMonitorManager() {
+        this(() -> new SlobrokMonitor2());
+    }
+
+    SlobrokMonitorManager(Supplier<SlobrokMonitor2> slobrokMonitorFactory) {
+        this.slobrokMonitorFactory = slobrokMonitorFactory;
+    }
+
+    @Override
+    public void applicationActivated(SuperModel superModel, ApplicationInfo application) {
+        synchronized (monitor) {
+            SlobrokMonitor2 slobrokMonitor = slobrokMonitors.computeIfAbsent(
+                    application.getApplicationId(),
+                    id -> slobrokMonitorFactory.get());
+            slobrokMonitor.updateSlobrokList(application);
+        }
+    }
+
+    @Override
+    public void applicationRemoved(SuperModel superModel, ApplicationId id) {
+        synchronized (monitor) {
+            SlobrokMonitor2 slobrokMonitor = slobrokMonitors.remove(id);
+            if (slobrokMonitor == null) {
+                logger.log(LogLevel.WARNING, "Removed application " + id +
+                        ", but it was never registered");
+            } else {
+                slobrokMonitor.close();
+            }
+        }
+    }
+
+    ServiceMonitorStatus getStatus(ApplicationId applicationId,
+                                   ServiceType serviceType,
+                                   ConfigId configId) {
+        Optional<String> slobrokServiceName = findSlobrokServiceName(serviceType, configId);
+        if (slobrokServiceName.isPresent()) {
+            synchronized (monitor) {
+                SlobrokMonitor2 slobrokMonitor = slobrokMonitors.get(applicationId);
+                if (slobrokMonitor != null &&
+                        slobrokMonitor.registeredInSlobrok(slobrokServiceName.get())) {
+                    return ServiceMonitorStatus.UP;
+                } else {
+                    return ServiceMonitorStatus.DOWN;
+                }
+            }
+        } else {
+            return ServiceMonitorStatus.NOT_CHECKED;
+        }
+    }
+
+    /**
+     * Get the Slobrok service name of the service, or empty if the service
+     * is not registered with Slobrok.
+     */
+    Optional<String> findSlobrokServiceName(ServiceType serviceType, ConfigId configId) {
+        switch (serviceType.s()) {
+            case "adminserver":
+            case "config-sentinel":
+            case "configproxy":
+            case "configserver":
+            case "filedistributorservice":
+            case "logd":
+            case "logserver":
+            case "metricsproxy":
+            case "slobrok":
+            case "transactionlogserver":
+                return Optional.empty();
+
+            case "topleveldispatch":
+                return Optional.of(configId.s());
+
+            case "qrserver":
+            case "container":
+            case "docprocservice":
+            case "container-clustercontroller":
+                return Optional.of("vespa/service/" + configId.s());
+
+            case "searchnode": //TODO: handle only as storagenode instead of both as searchnode/storagenode
+                return Optional.of(configId.s() + "/realtimecontroller");
+            case "distributor":
+            case "storagenode":
+                return Optional.of("storage/cluster." + configId.s());
+            default:
+                logger.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() +
+                        " with config id " + configId.s());
+                return Optional.empty();
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java
index 8beb90c382a..18f1e3ede6d 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SuperModelListenerImpl.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.monitor;
 
+import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.model.api.SuperModel;
 import com.yahoo.config.model.api.SuperModelListener;
 import com.yahoo.config.model.api.SuperModelProvider;
@@ -13,15 +14,14 @@ import java.util.logging.Logger;
 public class SuperModelListenerImpl implements SuperModelListener {
     private static final Logger logger = Logger.getLogger(SuperModelListenerImpl.class.getName());
 
-    // Guard for updating superModel and slobrokMonitor exclusively and atomically:
-    //  - superModel and slobrokMonitor must be updated in combination (exclusively and atomically)
-    //  - Anyone may take a snapshot of superModel for reading purposes, hence volatile.
+    // superModel and slobrokMonitorManager are always updated together
+    // and atomically using this monitor.
     private final Object monitor = new Object();
-    private final SlobrokMonitor2 slobrokMonitor;
-    private volatile SuperModel superModel;
+    private final SlobrokMonitorManager slobrokMonitorManager;
+    private SuperModel superModel;
 
-    SuperModelListenerImpl(SlobrokMonitor2 slobrokMonitor) {
-        this.slobrokMonitor = slobrokMonitor;
+    SuperModelListenerImpl(SlobrokMonitorManager slobrokMonitorManager) {
+        this.slobrokMonitorManager = slobrokMonitorManager;
     }
 
     void start(SuperModelProvider superModelProvider) {
@@ -30,38 +30,39 @@ public class SuperModelListenerImpl implements SuperModelListener {
             // since applicationActivated()/applicationRemoved() may be called
             // asynchronously even before snapshot() returns.
             SuperModel snapshot = superModelProvider.snapshot(this);
-            exclusiveUpdate(snapshot);
+
+            snapshot.getAllApplicationInfos().stream().forEach(application ->
+                    applicationActivated(superModel, application));
         }
     }
 
     @Override
-    public void applicationActivated(SuperModel superModel, ApplicationId applicationId) {
+    public void applicationActivated(SuperModel superModel, ApplicationInfo application) {
         synchronized (monitor) {
-            exclusiveUpdate(superModel);
+            this.superModel = superModel;
+            slobrokMonitorManager.applicationActivated(superModel, application);
         }
     }
 
     @Override
     public void applicationRemoved(SuperModel superModel, ApplicationId id) {
         synchronized (monitor) {
-            exclusiveUpdate(superModel);
+            this.superModel = superModel;
+            slobrokMonitorManager.applicationRemoved(superModel, id);
         }
     }
 
     ServiceModel createServiceModelSnapshot(Zone zone, List<String> configServerHostnames) {
-        // Save a snapshot of volatile this.superModel outside of synchronized block.
-        SuperModel superModelSnapshot = this.superModel;
-
         ModelGenerator modelGenerator = new ModelGenerator();
+
+        // TODO: Add latency and calls-per-second metrics
+        // If calculating the service model is too expensive per call, then
+        // cache the generated snapshot, invalidate after X seconds.
+        // WARNING: The slobrok monitor manager may be out-of-sync with super model (no locking)
         return modelGenerator.toServiceModel(
-                superModelSnapshot,
+                superModel,
                 zone,
                 configServerHostnames,
-                slobrokMonitor);
-    }
-
-    private void exclusiveUpdate(SuperModel superModel) {
-        this.superModel = superModel;
-        slobrokMonitor.updateSlobrokList(superModel);
+                slobrokMonitorManager);
     }
 }
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModel.java
index 771f139402e..5d7d955f15e 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModel.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModel.java
@@ -8,14 +8,15 @@ import com.yahoo.config.model.api.PortInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.model.api.SuperModel;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -27,39 +28,137 @@ public class ExampleModel {
     static final String CLUSTER_ID = "cluster-id";
     static final String SERVICE_NAME = "service-name";
     static final String SERVICE_TYPE = SlobrokMonitor2.SLOBROK_SERVICE_TYPE;
-    static final String CONFIG_ID = "config-id";
+    static final String CONFIG_ID = "configid/1";
     static final String TENANT = "tenant";
     static final String APPLICATION_NAME = "application";
     public static final String INSTANCE_NAME = "default";
 
     static SuperModel createExampleSuperModelWithOneRpcPort(String hostname, int rpcPort) {
-        Set<String> tags = Stream.of(SlobrokMonitor2.SLOBROK_RPC_PORT_TAG, "footag")
-                .collect(Collectors.toSet());
-        Map<String, String> properties = new HashMap<>();
-        properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, CLUSTER_ID);
-        Set<PortInfo> portInfos = Stream.of(new PortInfo(rpcPort, tags)).collect(Collectors.toSet());
-        ServiceInfo serviceInfo = new ServiceInfo(
-                SERVICE_NAME,
-                SERVICE_TYPE,
-                portInfos,
-                properties,
-                CONFIG_ID,
-                hostname);
-        List<ServiceInfo> serviceInfos = Stream.of(serviceInfo).collect(Collectors.toList());
-        HostInfo hostInfo = new HostInfo(hostname, serviceInfos);
-        List<HostInfo> hostInfos = Stream.of(hostInfo).collect(Collectors.toList());
-
-        TenantName tenantName = TenantName.from(TENANT);
-        ApplicationName applicationName = ApplicationName.from(APPLICATION_NAME);
-        InstanceName instanceName = InstanceName.from(INSTANCE_NAME);
-        ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName);
-        Model model = mock(Model.class);
-        when(model.getHosts()).thenReturn(hostInfos);
-        ApplicationInfo applicationInfo = new ApplicationInfo(applicationId, 1l, model);
+        List<String> hosts = Stream.of(hostname).collect(Collectors.toList());
+
+        ApplicationInfo applicationInfo = ExampleModel
+                .createApplication(TENANT, APPLICATION_NAME)
+                .addServiceCluster(CLUSTER_ID, SERVICE_NAME, SERVICE_TYPE, hosts)
+                .addPort(rpcPort, "footag", SlobrokMonitor2.SLOBROK_RPC_PORT_TAG)
+                .addPort(rpcPort + 1, "bartag")
+                .then()
+                .build();
 
         Map<TenantName, Map<ApplicationId, ApplicationInfo>> applicationInfos = new HashMap<>();
-        applicationInfos.put(tenantName, new HashMap<>());
-        applicationInfos.get(tenantName).put(applicationId, applicationInfo);
+        applicationInfos.put(applicationInfo.getApplicationId().tenant(), new HashMap<>());
+        applicationInfos.get(applicationInfo.getApplicationId().tenant())
+                .put(applicationInfo.getApplicationId(), applicationInfo);
         return new SuperModel(applicationInfos);
     }
+
+    static ApplicationBuilder createApplication(String tenant,
+                                                String applicationName) {
+        return new ApplicationBuilder(tenant, applicationName);
+    }
+
+
+    static class ApplicationBuilder {
+        private final String tenant;
+        private final String applicationName;
+        private final List<ClusterBuilder> clusters = new ArrayList<>();
+
+        ApplicationBuilder(String tenant, String applicationName) {
+            this.tenant = tenant;
+            this.applicationName = applicationName;
+        }
+
+        ClusterBuilder addServiceCluster(
+                String clusterId,
+                String serviceName,
+                String serviceType,
+                List<String> hosts) {
+            return new ClusterBuilder(
+                    this,
+                    clusterId,
+                    serviceName,
+                    serviceType,
+                    hosts);
+        }
+
+        ApplicationInfo build() {
+            List<String> allHosts = clusters.stream()
+                    .flatMap(clusterBuilder -> clusterBuilder.hosts.stream())
+                    .distinct()
+                    .collect(Collectors.toList());
+
+            List<HostInfo> hostInfos = new ArrayList<>();
+            for (String hostname : allHosts) {
+                List<ServiceInfo> serviceInfos = new ArrayList<>();
+                for (ClusterBuilder cluster : clusters) {
+                    buildServiceInfo(hostname, cluster).ifPresent(serviceInfos::add);
+                }
+
+                HostInfo hostInfo = new HostInfo(hostname, serviceInfos);
+                hostInfos.add(hostInfo);
+            }
+
+            ApplicationId id = ApplicationId.from(
+                    tenant,
+                    applicationName,
+                    InstanceName.defaultName().toString());
+
+            Model model = mock(Model.class);
+            when(model.getHosts()).thenReturn(hostInfos);
+
+            return new ApplicationInfo(id, 1, model);
+        }
+
+        private Optional<ServiceInfo> buildServiceInfo(
+                String hostname,
+                ClusterBuilder cluster) {
+            int hostIndex = cluster.hosts.indexOf(hostname);
+            if (hostIndex < 0) {
+                return Optional.empty();
+            }
+
+            Map<String, String> properties = new HashMap<>();
+            properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, cluster.clusterId);
+            return Optional.of(new ServiceInfo(
+                    cluster.serviceName,
+                    cluster.serviceType,
+                    cluster.portInfos,
+                    properties,
+                    "configid/" + (hostIndex + 1),
+                    hostname));
+        }
+    }
+
+    static class ClusterBuilder {
+        private final ApplicationBuilder applicationBuilder;
+        private final String clusterId;
+        private final String serviceName;
+        private final String serviceType;
+        private final List<String> hosts;
+        private final List<PortInfo> portInfos = new ArrayList<>();
+
+        ClusterBuilder(ApplicationBuilder applicationBuilder,
+                       String clusterId,
+                       String serviceName,
+                       String serviceType,
+                       List<String> hosts) {
+            this.applicationBuilder = applicationBuilder;
+            this.clusterId = clusterId;
+            this.serviceName = serviceName;
+            this.serviceType = serviceType;
+            this.hosts = hosts;
+        }
+
+        /**
+         * A bit unrealistic, but the port is the same on all hosts.
+         */
+        ClusterBuilder addPort(int port, String... tags) {
+            portInfos.add(new PortInfo(port, Arrays.asList(tags)));
+            return this;
+        }
+
+        ApplicationBuilder then() {
+            applicationBuilder.clusters.add(this);
+            return applicationBuilder;
+        }
+    }
 }
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModelTest.java
new file mode 100644
index 00000000000..bcd4eae1b4e
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ExampleModelTest.java
@@ -0,0 +1,93 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import org.junit.Test;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+
+public class ExampleModelTest {
+    @Test
+    public void testEmptyApplication() {
+        ApplicationInfo application = ExampleModel
+                .createApplication(
+                        "tenant",
+                        "app")
+                .build();
+
+        assertEquals("tenant.app", application.getApplicationId().toString());
+        assertEquals(1, application.getGeneration());
+        assertEquals(0, application.getModel().getHosts().size());
+    }
+
+    @Test
+    public void test() {
+        List<String> contentNodes = Stream.of("host1", "host2").collect(Collectors.toList());
+        List<String> containerNodes = Stream.of("host3", "host4").collect(Collectors.toList());
+
+        ApplicationInfo application = ExampleModel
+                .createApplication(
+                        "tenant",
+                        "app")
+                .addServiceCluster(
+                        "product-controllers",
+                        "container-clustercontroller.1",
+                        "container-clustercontroller",
+                        contentNodes)
+                .then()
+                .addServiceCluster(
+                        "product",
+                        "searchnode.1",
+                        "searchnode",
+                        contentNodes)
+                .then()
+                .addServiceCluster(
+                        "admin",
+                        "slobrok.1",
+                        "slobrok",
+                        containerNodes)
+                .then()
+                .addServiceCluster(
+                        "default",
+                        "container.1",
+                        "container",
+                        containerNodes)
+                .then()
+                .build();
+
+        assertEquals("tenant.app", application.getApplicationId().toString());
+
+        Collection<HostInfo> hostInfos = application.getModel().getHosts();
+        assertEquals(containerNodes.size() + contentNodes.size(), hostInfos.size());
+
+        HostInfo host1 = hostInfos.stream()
+                .filter(hostInfo -> hostInfo.getHostname().equals("host1"))
+                .findAny()
+                .orElseThrow(() -> new RuntimeException());
+        ServiceInfo controller1 = host1.getServices().stream()
+                .filter(i -> i.getServiceType().equals("container-clustercontroller"))
+                .findAny()
+                .orElseThrow(() -> new RuntimeException());
+
+        assertEquals("container-clustercontroller", controller1.getServiceType());
+        assertEquals("configid/1", controller1.getConfigId());
+
+        HostInfo host4 = hostInfos.stream()
+                .filter(hostInfo -> hostInfo.getHostname().equals("host4"))
+                .findAny()
+                .orElseThrow(() -> new RuntimeException());
+        ServiceInfo slobrok2 = host4.getServices().stream()
+                .filter(i -> i.getServiceType().equals("slobrok"))
+                .findAny()
+                .orElseThrow(() -> new RuntimeException());
+        assertEquals("configid/2", slobrok2.getConfigId());
+    }
+}
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ModelGeneratorTest.java
index 87cbb7950b6..4b82286235e 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ModelGeneratorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ModelGeneratorTest.java
@@ -41,15 +41,16 @@ public class ModelGeneratorTest {
         List<String> configServerHosts = Stream.of("cfg1", "cfg2", "cfg3")
                 .collect(Collectors.toList());
 
-        SlobrokMonitor2 slobrokMonitor = mock(SlobrokMonitor2.class);
-        when(slobrokMonitor.getStatus(any(), any())).thenReturn(ServiceMonitorStatus.UP);
+        SlobrokMonitorManager slobrokMonitorManager = mock(SlobrokMonitorManager.class);
+        when(slobrokMonitorManager.getStatus(any(), any(), any()))
+                .thenReturn(ServiceMonitorStatus.UP);
 
         ServiceModel serviceModel =
                 modelGenerator.toServiceModel(
                         superModel,
                         zone,
                         configServerHosts,
-                        slobrokMonitor);
+                        slobrokMonitorManager);
 
         Map<ApplicationInstanceReference,
                 ApplicationInstance<ServiceMonitorStatus>> applicationInstances =
@@ -84,15 +85,16 @@ public class ModelGeneratorTest {
 
         List<String> configServerHosts = Collections.emptyList();
 
-        SlobrokMonitor2 slobrokMonitor = mock(SlobrokMonitor2.class);
-        when(slobrokMonitor.getStatus(any(), any())).thenReturn(ServiceMonitorStatus.UP);
+        SlobrokMonitorManager slobrokMonitorManager = mock(SlobrokMonitorManager.class);
+        when(slobrokMonitorManager.getStatus(any(), any(), any()))
+                .thenReturn(ServiceMonitorStatus.UP);
 
         ServiceModel serviceModel =
                 modelGenerator.toServiceModel(
                         superModel,
                         zone,
                         configServerHosts,
-                        slobrokMonitor);
+                        slobrokMonitorManager);
 
         Map<ApplicationInstanceReference,
                 ApplicationInstance<ServiceMonitorStatus>> applicationInstances =
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2Test.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2Test.java
index 242112694eb..7be63e527cf 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2Test.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2Test.java
@@ -1,19 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.monitor;
 
-import com.yahoo.config.model.api.SuperModel;
+import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.jrt.slobrok.api.Mirror;
 import com.yahoo.jrt.slobrok.api.SlobrokList;
-import com.yahoo.vespa.applicationmodel.ConfigId;
-import com.yahoo.vespa.applicationmodel.ServiceType;
 import org.junit.Test;
 
-import java.util.Optional;
-
-import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
 public class SlobrokMonitor2Test {
     private final SlobrokList slobrokList = mock(SlobrokList.class);
@@ -21,38 +14,23 @@ public class SlobrokMonitor2Test {
     private SlobrokMonitor2 slobrokMonitor = new SlobrokMonitor2(slobrokList, mirror);
 
     @Test
-    public void testLookup() {
-        assertEquals(
-                Optional.of("config.id"),
-                lookup("topleveldispatch", "config.id"));
-
-        assertEquals(
-                Optional.empty(),
-                lookup("adminserver", "config.id"));
-    }
-
-    private Optional<String> lookup(String serviceType, String configId) {
-        return slobrokMonitor.lookup(new ServiceType(serviceType), new ConfigId(configId));
-    }
-
-    @Test
-    public void testGetStatus() {
-        ServiceType serviceType = new ServiceType("topleveldispatch");
-        ConfigId configId = new ConfigId("config.id");
-        when(mirror.lookup("config.id")).thenReturn(new Mirror.Entry[1]);
-        assertEquals(ServiceMonitorStatus.UP, slobrokMonitor.getStatus(serviceType, configId));
+    public void testUpdateSlobrokList() {
+        ApplicationInfo applicationInfo = ExampleModel.createApplication(
+                "tenant",
+                "application-name")
+                .build();
     }
 
     @Test
-    public void testUpdateSlobrokList() {
+    public void testUpdateSlobrokList2() {
+        /*
         final String hostname = "hostname";
         final int port = 1;
 
         SuperModel superModel = ExampleModel.createExampleSuperModelWithOneRpcPort(hostname, port);
-        slobrokMonitor.updateSlobrokList(superModel);
+        slobrokMonitor.updateSlobrokList(superModel.getApplicationInfo());
 
         String[] expectedSpecs = new String[] {"tcp/" + hostname + ":" + port};
-        verify(slobrokList).setup(expectedSpecs);
+        verify(slobrokList).setup(expectedSpecs); */
     }
-
 }
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitorManagerTest.java
new file mode 100644
index 00000000000..f34dd91181c
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SlobrokMonitorManagerTest.java
@@ -0,0 +1,92 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.SuperModel;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class SlobrokMonitorManagerTest {
+    // IntelliJ complains if parametrized type is specified, Maven complains if not specified.
+    @SuppressWarnings("unchecked")
+    private final Supplier<SlobrokMonitor2> slobrokMonitorFactory = mock(Supplier.class);
+
+    private final SlobrokMonitorManager slobrokMonitorManager =
+            new SlobrokMonitorManager(slobrokMonitorFactory);
+    private final SlobrokMonitor2 slobrokMonitor = mock(SlobrokMonitor2.class);
+    private final SuperModel superModel = mock(SuperModel.class);
+    private final ApplicationInfo application = mock(ApplicationInfo.class);
+
+    @Before
+    public void setup() {
+        when(slobrokMonitorFactory.get()).thenReturn(slobrokMonitor);
+    }
+
+    @Test
+    public void testActivationOfApplication() {
+        slobrokMonitorManager.applicationActivated(superModel, application);
+        verify(slobrokMonitorFactory, times(1)).get();
+    }
+
+    @Test
+    public void testGetStatus_ApplicationNotInSlobrok() {
+        when(slobrokMonitor.registeredInSlobrok("config.id")).thenReturn(true);
+        assertEquals(ServiceMonitorStatus.DOWN, getStatus("topleveldispatch"));
+    }
+
+    @Test
+    public void testGetStatus_ApplicationInSlobrok() {
+        slobrokMonitorManager.applicationActivated(superModel, application);
+        when(slobrokMonitor.registeredInSlobrok("config.id")).thenReturn(true);
+        assertEquals(ServiceMonitorStatus.UP, getStatus("topleveldispatch"));
+    }
+
+    @Test
+    public void testGetStatus_ServiceNotInSlobrok() {
+        slobrokMonitorManager.applicationActivated(superModel, application);
+        when(slobrokMonitor.registeredInSlobrok("config.id")).thenReturn(false);
+        assertEquals(ServiceMonitorStatus.DOWN, getStatus("topleveldispatch"));
+    }
+
+    @Test
+    public void testGetStatus_NotChecked() {
+        assertEquals(ServiceMonitorStatus.NOT_CHECKED, getStatus("slobrok"));
+        verify(slobrokMonitor, times(0)).registeredInSlobrok(any());
+    }
+
+    private ServiceMonitorStatus getStatus(String serviceType) {
+        return slobrokMonitorManager.getStatus(
+                application.getApplicationId(),
+                new ServiceType(serviceType),
+                new ConfigId("config.id"));
+    }
+
+    @Test
+    public void testLookup() {
+        assertEquals(
+                Optional.of("config.id"),
+                findSlobrokServiceName("topleveldispatch", "config.id"));
+
+        assertEquals(
+                Optional.empty(),
+                findSlobrokServiceName("adminserver", "config.id"));
+    }
+
+    private Optional<String> findSlobrokServiceName(String serviceType, String configId) {
+        return slobrokMonitorManager.findSlobrokServiceName(
+                new ServiceType(serviceType),
+                new ConfigId(configId));
+    }
+}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SuperModelListenerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SuperModelListenerImplTest.java
deleted file mode 100644
index 07e84b22d72..00000000000
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/SuperModelListenerImplTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-package com.yahoo.vespa.service.monitor;
-
-import com.yahoo.config.model.api.SuperModel;
-import com.yahoo.config.provision.ApplicationId;
-import org.junit.Test;
-
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-
-public class SuperModelListenerImplTest {
-    private final SlobrokMonitor2 slobrokMonitor = mock(SlobrokMonitor2.class);
-    private final SuperModel superModel = mock(SuperModel.class);
-    private final ApplicationId applicationId = ApplicationId.defaultId();
-    private final SuperModelListenerImpl listener = new SuperModelListenerImpl(slobrokMonitor);
-
-    @Test
-    public void testActivateApplication() {
-        listener.applicationActivated(superModel, applicationId);
-        doNothing().when(slobrokMonitor).updateSlobrokList(superModel);
-    }
-
-    @Test
-    public void testRemoveApplication() {
-        listener.applicationRemoved(superModel, applicationId);
-        doNothing().when(slobrokMonitor).updateSlobrokList(superModel);
-    }
-}
\ No newline at end of file