diff options
author | HÃ¥kon Hallingstad <hakon@oath.com> | 2018-12-08 12:28:45 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-12-08 12:28:45 +0100 |
commit | b3afc75e66c909660b7c72b6f649981f31134d87 (patch) | |
tree | 8dbb00e6c5f982a6001c9d66de600bd462dd85a7 | |
parent | 836a4369d382611d338d11ed589db884e01cbb6c (diff) | |
parent | 4023a30a3dfd3fe231693cd1a300af9725bd008e (diff) |
Merge pull request #7912 from vespa-engine/hakonhall/simplify-infrastructure-applications
Monitor health of host-admin infrastructure applications
21 files changed, 302 insertions, 181 deletions
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a3c7759779a..54f5403ca73 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -312,7 +312,7 @@ public class ProvisioningTester { nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName()); nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName()); - ConfigServerApplication application = ConfigServerApplication.CONFIG_SERVER_APPLICATION; + ConfigServerApplication application = new ConfigServerApplication(); List<HostSpec> hosts = prepare( application.getApplicationId(), application.getClusterSpecWithVersion(configServersVersion), diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java index f93f25a4eb3..88c2c0d4469 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerApplication.java @@ -3,76 +3,30 @@ package com.yahoo.vespa.service.duper; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.model.api.HostInfo; -import com.yahoo.config.model.api.PortInfo; -import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; -import com.yahoo.vespa.applicationmodel.ClusterId; -import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.applicationmodel.TenantId; -import com.yahoo.vespa.service.model.ModelGenerator; -import com.yahoo.vespa.service.health.ApplicationHealthMonitor; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.stream.Collectors; /** * A service/application model of the config server with health status. */ -public class ConfigServerApplication extends InfraApplication { +public class ConfigServerApplication extends ConfigServerLikeApplication { public static final ConfigServerApplication CONFIG_SERVER_APPLICATION = new ConfigServerApplication(); - public static final TenantId TENANT_ID = new TenantId(CONFIG_SERVER_APPLICATION.getApplicationId().tenant().value()); - public static final ApplicationInstanceId APPLICATION_INSTANCE_ID = - new ApplicationInstanceId(CONFIG_SERVER_APPLICATION.getApplicationId().application().value()); - public static final ClusterId CLUSTER_ID = new ClusterId(CONFIG_SERVER_APPLICATION.getClusterId().value()); - public static final ServiceType SERVICE_TYPE = new ServiceType("configserver"); - public static final String CONFIG_ID_PREFIX = "configid."; - - public static ConfigId configIdFrom(int index) { - return new ConfigId(CONFIG_ID_PREFIX + index); - } public ConfigServerApplication() { - super("zone-config-servers", NodeType.config, - ClusterSpec.Type.admin, ClusterSpec.Id.from("zone-config-servers"), ServiceType.CONFIG_SERVER); + super("zone-config-servers", NodeType.config, ClusterSpec.Type.admin, ServiceType.CONFIG_SERVER); } public ApplicationInfo makeApplicationInfoFromConfig(ConfigserverConfig config) { - List<HostInfo> hostInfos = new ArrayList<>(); - List<ConfigserverConfig.Zookeeperserver> zooKeeperServers = config.zookeeperserver(); - for (int index = 0; index < zooKeeperServers.size(); ++index) { - String hostname = zooKeeperServers.get(index).hostname(); - hostInfos.add(makeHostInfo(hostname, config.httpport(), index)); - } - - return new ApplicationInfo( - CONFIG_SERVER_APPLICATION.getApplicationId(), - 0, - new HostsModel(hostInfos)); - } - - private static HostInfo makeHostInfo(String hostname, int port, int configIndex) { - PortInfo portInfo = new PortInfo(port, ApplicationHealthMonitor.PORT_TAGS_HEALTH); - - Map<String, String> properties = new HashMap<>(); - properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, CLUSTER_ID.s()); - - ServiceInfo serviceInfo = new ServiceInfo( - // service name == service type for the first service of each type on each host - SERVICE_TYPE.s(), - SERVICE_TYPE.s(), - Collections.singletonList(portInfo), - properties, - configIdFrom(configIndex).s(), - hostname); - - return new HostInfo(hostname, Collections.singletonList(serviceInfo)); + List<HostName> hostnames = config.zookeeperserver().stream() + .map(ConfigserverConfig.Zookeeperserver::hostname) + .map(HostName::from) + .collect(Collectors.toList()); + return makeApplicationInfo(hostnames); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java index f29cc89e987..4273bccbf45 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerHostApplication.java @@ -1,14 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ServiceType; -public class ConfigServerHostApplication extends InfraApplication { +public class ConfigServerHostApplication extends HostAdminApplication { public ConfigServerHostApplication() { - super("configserver-host", NodeType.confighost, - ClusterSpec.Type.container, ClusterSpec.Id.from("configserver-host"), - ServiceType.HOST_ADMIN); + super("configserver-host", NodeType.confighost); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerLikeApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerLikeApplication.java new file mode 100644 index 00000000000..ac5f13e2394 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ConfigServerLikeApplication.java @@ -0,0 +1,17 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.duper; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.applicationmodel.ServiceType; + +/** + * Base class for config server and controller infrastructure applications. + * + * @author hakonhall + */ +public abstract class ConfigServerLikeApplication extends InfraApplication { + protected ConfigServerLikeApplication(String applicationName, NodeType nodeType, ClusterSpec.Type clusterType, ServiceType serviceType) { + super(applicationName, nodeType, clusterType, ClusterSpec.Id.from(applicationName), serviceType, 19071); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java index 1fb01ce0897..3d69b1c9925 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerApplication.java @@ -8,10 +8,8 @@ import com.yahoo.vespa.applicationmodel.ServiceType; /** * @author mpolden */ -public class ControllerApplication extends InfraApplication { +public class ControllerApplication extends ConfigServerLikeApplication { public ControllerApplication() { - super("controller", NodeType.controller, ClusterSpec.Type.container, - ClusterSpec.Id.from("controller"), ServiceType.CONTROLLER); + super("controller", NodeType.controller, ClusterSpec.Type.container, ServiceType.CONTROLLER); } - } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java index b8e8ead5b7e..7a0c952780d 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ControllerHostApplication.java @@ -1,16 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ServiceType; /** * @author mpolden */ -public class ControllerHostApplication extends InfraApplication { +public class ControllerHostApplication extends HostAdminApplication { public ControllerHostApplication() { - super("controller-host", NodeType.controllerhost, ClusterSpec.Type.container, - ClusterSpec.Id.from("controller-host"), ServiceType.HOST_ADMIN); + super("controller-host", NodeType.controllerhost); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index ed99d2e3166..e0e2b64bdae 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -32,13 +32,14 @@ public class DuperModelManager implements DuperModelInfraApi { private static Logger logger = Logger.getLogger(DuperModelManager.class.getName()); // Infrastructure applications + private final ConfigServerHostApplication configServerHostApplication = new ConfigServerHostApplication(); + private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); + private final ControllerApplication controllerApplication = new ControllerApplication(); + private final ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); + // this must be static to be referenced in this(). Remove static once legacy config server from config is gone. private static final ConfigServerApplication configServerApplication = new ConfigServerApplication(); - private static final ConfigServerHostApplication configServerHostApplication = new ConfigServerHostApplication(); - private static final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); - private static final ControllerApplication controllerApplication = new ControllerApplication(); - private static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); - private static final Map<ApplicationId, InfraApplication> supportedInfraApplications = Stream.of( + private final Map<ApplicationId, InfraApplication> supportedInfraApplications = Stream.of( configServerApplication, configServerHostApplication, proxyHostApplication, @@ -52,19 +53,9 @@ public class DuperModelManager implements DuperModelInfraApi { private final Object monitor = new Object(); private final DuperModel duperModel; - // The set of active infrastructure ApplicationInfo. Not all is necessarily in the DuperModel for historical reasons. + // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons. private final Set<ApplicationId> activeInfraInfos = new HashSet<>(2 * supportedInfraApplications.size()); - /** - * Returns true if application is considered an infrastructure application by the DuperModel. - * - * <p>Note: The tenant host "application" is NOT considered an infrastructure application: It is just a - * cluster in the {@link ZoneApplication zone application}. - */ - public static boolean isInfrastructureApplication(ApplicationId applicationId) { - return supportedInfraApplications.containsKey(applicationId); - } - @Inject public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) { this( @@ -148,6 +139,16 @@ public class DuperModelManager implements DuperModelInfraApi { return new ArrayList<>(supportedInfraApplications.values()); } + /** + * Returns true if application is considered an infrastructure application by the DuperModel. + * + * <p>Note: The tenant host "application" is NOT considered an infrastructure application: It is just a + * cluster in the {@link ZoneApplication zone application}. + */ + public boolean isSupportedInfraApplication(ApplicationId applicationId) { + return supportedInfraApplications.containsKey(applicationId); + } + @Override public boolean infraApplicationIsActive(ApplicationId applicationId) { synchronized (monitor) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java new file mode 100644 index 00000000000..5e6cb23e9c1 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/HostAdminApplication.java @@ -0,0 +1,16 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.duper; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.applicationmodel.ServiceType; + +/** + * @author hakonhall + */ +public abstract class HostAdminApplication extends InfraApplication { + protected HostAdminApplication(String applicationName, NodeType nodeType) { + super(applicationName, nodeType, ClusterSpec.Type.container, ClusterSpec.Id.from(applicationName), + ServiceType.HOST_ADMIN, 8080); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index 89aabbfe2b0..8c74fe0396e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import com.yahoo.component.Version; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; @@ -11,34 +12,34 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.component.Version; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.monitor.InfraApplicationApi; -import com.yahoo.vespa.service.model.ModelGenerator; +import com.yahoo.vespa.applicationmodel.TenantId; import com.yahoo.vespa.service.health.ApplicationHealthMonitor; +import com.yahoo.vespa.service.model.ModelGenerator; +import com.yahoo.vespa.service.monitor.InfraApplicationApi; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; /** * @author freva */ public abstract class InfraApplication implements InfraApplicationApi { - static final int HEALTH_PORT = 8080; - private static final TenantName TENANT_NAME = TenantName.from("hosted-vespa"); - private static final String CONFIG_ID_PREFIX = "configid."; private final ApplicationId applicationId; private final Capacity capacity; - private final ClusterSpec.Type clusterType; - private final ClusterSpec.Id clusterId; + private final ClusterSpec.Type clusterSpecType; + private final ClusterSpec.Id clusterSpecId; private final ServiceType serviceType; + private final int healthPort; public static ApplicationId createHostedVespaApplicationId(String applicationName) { return new ApplicationId.Builder() @@ -49,14 +50,16 @@ public abstract class InfraApplication implements InfraApplicationApi { protected InfraApplication(String applicationName, NodeType nodeType, - ClusterSpec.Type clusterType, - ClusterSpec.Id clusterId, - ServiceType serviceType) { + ClusterSpec.Type clusterSpecType, + ClusterSpec.Id clusterSpecId, + ServiceType serviceType, + int healthPort) { this.applicationId = createHostedVespaApplicationId(applicationName); this.capacity = Capacity.fromRequiredNodeType(nodeType); - this.clusterType = clusterType; - this.clusterId = clusterId; + this.clusterSpecType = clusterSpecType; + this.clusterSpecId = clusterSpecId; this.serviceType = serviceType; + this.healthPort = healthPort; } @Override @@ -71,31 +74,43 @@ public abstract class InfraApplication implements InfraApplicationApi { @Override public ClusterSpec getClusterSpecWithVersion(Version version) { - return ClusterSpec.request(clusterType, clusterId, version, true); + return ClusterSpec.request(clusterSpecType, clusterSpecId, version, true); } - public ClusterSpec.Type getClusterType() { - return clusterType; + public ClusterSpec.Type getClusterSpecType() { + return clusterSpecType; } - public ClusterSpec.Id getClusterId() { - return clusterId; + public ClusterSpec.Id getClusterSpecId() { + return clusterSpecId; } - public ApplicationInfo makeApplicationInfo(List<HostName> hostnames) { - List<HostInfo> hostInfos = new ArrayList<>(); - for (int index = 0; index < hostnames.size(); ++index) { - hostInfos.add(makeHostInfo(hostnames.get(index), HEALTH_PORT, index, serviceType, clusterId)); - } + public ClusterId getClusterId() { + return new ClusterId(clusterSpecId.value()); + } + public ServiceType getServiceType() { + return serviceType; + } + + public ApplicationInstanceId getApplicationInstanceId() { + return new ApplicationInstanceId(applicationId.application().value()); + } + + public TenantId getTenantId() { + return new TenantId(applicationId.tenant().value()); + } + + public ApplicationInfo makeApplicationInfo(List<HostName> hostnames) { + List<HostInfo> hostInfos = hostnames.stream().map(this::makeHostInfo).collect(Collectors.toList()); return new ApplicationInfo(applicationId, 0, new HostsModel(hostInfos)); } - private static HostInfo makeHostInfo(HostName hostname, int port, int configIndex, ServiceType serviceType, ClusterSpec.Id clusterId) { - PortInfo portInfo = new PortInfo(port, ApplicationHealthMonitor.PORT_TAGS_HEALTH); + private HostInfo makeHostInfo(HostName hostname) { + PortInfo portInfo = new PortInfo(healthPort, ApplicationHealthMonitor.PORT_TAGS_HEALTH); Map<String, String> properties = new HashMap<>(); - properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, clusterId.value()); + properties.put(ModelGenerator.CLUSTER_ID_PROPERTY_NAME, getClusterId().s()); ServiceInfo serviceInfo = new ServiceInfo( // service name == service type for the first service of each type on each host @@ -103,14 +118,24 @@ public abstract class InfraApplication implements InfraApplicationApi { serviceType.s(), Collections.singletonList(portInfo), properties, - configIdFrom(configIndex).s(), + configIdFor(hostname).s(), hostname.value()); return new HostInfo(hostname.value(), Collections.singletonList(serviceInfo)); } - private static ConfigId configIdFrom(int index) { - return new ConfigId(CONFIG_ID_PREFIX + index); + public ConfigId configIdFor(HostName hostname) { + // Not necessarily unique, but service monitor doesn't require it to be unique. + return new ConfigId(String.format("%s/%s", clusterSpecId.value(), prefixTo(hostname.value(), '.'))); + } + + private static String prefixTo(String string, char sentinel) { + int offset = string.indexOf(sentinel); + if (offset == -1) { + return string; + } else { + return string.substring(0, offset); + } } @Override @@ -118,16 +143,17 @@ public abstract class InfraApplication implements InfraApplicationApi { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; InfraApplication that = (InfraApplication) o; - return Objects.equals(applicationId, that.applicationId) && + return healthPort == that.healthPort && + Objects.equals(applicationId, that.applicationId) && Objects.equals(capacity, that.capacity) && - clusterType == that.clusterType && - Objects.equals(clusterId, that.clusterId) && + clusterSpecType == that.clusterSpecType && + Objects.equals(clusterSpecId, that.clusterSpecId) && Objects.equals(serviceType, that.serviceType); } @Override public int hashCode() { - return Objects.hash(applicationId, capacity, clusterType, clusterId, serviceType); + return Objects.hash(applicationId, capacity, clusterSpecType, clusterSpecId, serviceType, healthPort); } @Override @@ -135,9 +161,10 @@ public abstract class InfraApplication implements InfraApplicationApi { return "InfraApplication{" + "applicationId=" + applicationId + ", capacity=" + capacity + - ", clusterType=" + clusterType + - ", clusterId=" + clusterId + + ", clusterSpecType=" + clusterSpecType + + ", clusterSpecId=" + clusterSpecId + ", serviceType=" + serviceType + + ", healthPort=" + healthPort + '}'; } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java index b6b0964a82d..24230609e60 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ProxyHostApplication.java @@ -1,13 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.applicationmodel.ServiceType; -public class ProxyHostApplication extends InfraApplication { +public class ProxyHostApplication extends HostAdminApplication { public ProxyHostApplication() { - super("proxy-host", NodeType.proxyhost, ClusterSpec.Type.container, - ClusterSpec.Id.from("proxy-host"), ServiceType.HOST_ADMIN); + super("proxy-host", NodeType.proxyhost); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java new file mode 100644 index 00000000000..43be236268c --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorFactory.java @@ -0,0 +1,12 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.health; + +import com.yahoo.config.model.api.ApplicationInfo; + +/** + * @author hakonhall + */ +@FunctionalInterface +interface ApplicationHealthMonitorFactory { + ApplicationHealthMonitor create(ApplicationInfo applicationInfo); +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java index daf9fd1931b..e9a5ec314f6 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java @@ -8,7 +8,9 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.flags.FeatureFlag; +import com.yahoo.vespa.flags.FileFlagSource; +import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.manager.MonitorManager; @@ -21,14 +23,29 @@ import java.util.concurrent.ConcurrentHashMap; */ public class HealthMonitorManager implements MonitorManager { private final ConcurrentHashMap<ApplicationId, ApplicationHealthMonitor> healthMonitors = new ConcurrentHashMap<>(); + private final DuperModelManager duperModel; + private final ApplicationHealthMonitorFactory applicationHealthMonitorFactory; + private final FeatureFlag monitorInfra; @Inject - public HealthMonitorManager() { } + public HealthMonitorManager(DuperModelManager duperModel, FileFlagSource flagSource) { + this(duperModel, + new FeatureFlag("healthmonitor-monitorinfra", true, flagSource), + ApplicationHealthMonitor::startMonitoring); + } + + HealthMonitorManager(DuperModelManager duperModel, + FeatureFlag monitorInfra, + ApplicationHealthMonitorFactory applicationHealthMonitorFactory) { + this.duperModel = duperModel; + this.applicationHealthMonitorFactory = applicationHealthMonitorFactory; + this.monitorInfra = monitorInfra; + } @Override public void applicationActivated(ApplicationInfo application) { if (wouldMonitor(application.getApplicationId())) { - ApplicationHealthMonitor monitor = ApplicationHealthMonitor.startMonitoring(application); + ApplicationHealthMonitor monitor = applicationHealthMonitorFactory.create(application); healthMonitors.put(application.getApplicationId(), monitor); } } @@ -62,7 +79,11 @@ public class HealthMonitorManager implements MonitorManager { @Override public boolean wouldMonitor(ApplicationId id) { - if (id.equals(ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId())) { + if (duperModel.isSupportedInfraApplication(id) && monitorInfra.value()) { + return true; + } + + if (id.equals(duperModel.getConfigServerApplication().getApplicationId())) { return true; } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index 9bcb3f07cf6..f15101d4439 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; import com.yahoo.vespa.applicationmodel.TenantId; +import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.HashMap; @@ -25,8 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.vespa.service.duper.ConfigServerApplication.CONFIG_SERVER_APPLICATION; - /** * Class to generate an ApplicationInstance given service status for a standard (deployed) application. * @@ -37,10 +36,15 @@ public class ApplicationInstanceGenerator { private final ApplicationInfo applicationInfo; private final Zone zone; + private ApplicationId configServerApplicationId; public ApplicationInstanceGenerator(ApplicationInfo applicationInfo, Zone zone) { this.applicationInfo = applicationInfo; this.zone = zone; + + // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. + // We do this to avoid passing through the ID through multiple levels. + this.configServerApplicationId = new ConfigServerApplication().getApplicationId(); } public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) { @@ -105,7 +109,7 @@ public class ApplicationInstanceGenerator { } private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { - if (applicationInfo.getApplicationId().equals(CONFIG_SERVER_APPLICATION.getApplicationId())) { + if (applicationInfo.getApplicationId().equals(configServerApplicationId)) { // Removing this historical discrepancy would break orchestration during rollout. // An alternative may be to use a feature flag and flip it between releases, // once that's available. diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java index b695b0b19f8..c0bcff7284c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceId.java @@ -22,9 +22,9 @@ public class ServiceId { private final ConfigId configId; public ServiceId(ApplicationId applicationId, - ClusterId clusterId, - ServiceType serviceType, - ConfigId configId) { + ClusterId clusterId, + ServiceType serviceType, + ConfigId configId) { this.applicationId = applicationId; this.clusterId = clusterId; this.serviceType = serviceType; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index 39da5040ca0..13f24bc3694 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -28,14 +28,16 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { private final Object monitor = new Object(); private final HashMap<ApplicationId, SlobrokMonitor> slobrokMonitors = new HashMap<>(); + private final DuperModelManager duperModel; @Inject - public SlobrokMonitorManagerImpl() { - this(SlobrokMonitor::new); + public SlobrokMonitorManagerImpl(DuperModelManager duperModel) { + this(SlobrokMonitor::new, duperModel); } - SlobrokMonitorManagerImpl(Supplier<SlobrokMonitor> slobrokMonitorFactory) { + SlobrokMonitorManagerImpl(Supplier<SlobrokMonitor> slobrokMonitorFactory, DuperModelManager duperModel) { this.slobrokMonitorFactory = slobrokMonitorFactory; + this.duperModel = duperModel; } @Override @@ -108,7 +110,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { @Override public boolean wouldMonitor(ApplicationId applicationId) { - if (DuperModelManager.isInfrastructureApplication(applicationId)) { + if (duperModel.isSupportedInfraApplication(applicationId)) { return false; } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java index 078f34b84d1..0dfca12099e 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.service.duper.ConfigServerApplication; import com.yahoo.vespa.service.monitor.ConfigserverUtil; @@ -16,6 +17,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ApplicationHealthMonitorTest { + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); + @Test public void sanityCheck() { MonitorFactory monitorFactory = new MonitorFactory(); @@ -36,28 +39,17 @@ public class ApplicationHealthMonitorTest { ConfigserverUtil.makeExampleConfigServer(), monitorFactory); - ConfigServerApplication configServerApplication = new ConfigServerApplication(); - - ServiceStatus status1 = applicationMonitor.getStatus( - ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), - ConfigServerApplication.CLUSTER_ID, - ConfigServerApplication.SERVICE_TYPE, - ConfigServerApplication.configIdFrom(0)); - assertEquals(ServiceStatus.UP, status1); - - ServiceStatus status2 = applicationMonitor.getStatus( - ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), - ConfigServerApplication.CLUSTER_ID, - ConfigServerApplication.SERVICE_TYPE, - ConfigServerApplication.configIdFrom(1)); - assertEquals(ServiceStatus.DOWN, status2); - - ServiceStatus status3 = applicationMonitor.getStatus( - ConfigServerApplication.CONFIG_SERVER_APPLICATION.getApplicationId(), - ConfigServerApplication.CLUSTER_ID, - ConfigServerApplication.SERVICE_TYPE, - ConfigServerApplication.configIdFrom(2)); - assertEquals(ServiceStatus.NOT_CHECKED, status3); + assertEquals(ServiceStatus.UP, getStatus(applicationMonitor, "cfg1")); + assertEquals(ServiceStatus.DOWN, getStatus(applicationMonitor, "cfg2")); + assertEquals(ServiceStatus.NOT_CHECKED, getStatus(applicationMonitor, "cfg3")); + } + + private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) { + return monitor.getStatus( + configServerApplication.getApplicationId(), + configServerApplication.getClusterId(), + configServerApplication.getServiceType(), + configServerApplication.configIdFor(HostName.from(hostname))); } private static class MonitorFactory implements Function<HealthEndpoint, HealthMonitor> { diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index 0b9d7e123b5..f420f5c1284 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -2,20 +2,51 @@ package com.yahoo.vespa.service.health; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.flags.FeatureFlag; +import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.duper.ControllerHostApplication; +import com.yahoo.vespa.service.duper.DuperModelManager; +import com.yahoo.vespa.service.duper.InfraApplication; +import com.yahoo.vespa.service.duper.ProxyHostApplication; import com.yahoo.vespa.service.duper.ZoneApplication; import com.yahoo.vespa.service.monitor.ConfigserverUtil; +import org.junit.Before; import org.junit.Test; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class HealthMonitorManagerTest { + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); + private final DuperModelManager duperModel = mock(DuperModelManager.class); + private final FeatureFlag monitorInfra = mock(FeatureFlag.class); + private final ApplicationHealthMonitor monitor = mock(ApplicationHealthMonitor.class); + private final ApplicationHealthMonitorFactory monitorFactory = mock(ApplicationHealthMonitorFactory.class); + private final HealthMonitorManager manager = new HealthMonitorManager(duperModel, monitorInfra, monitorFactory); + + @Before + public void setUp() { + when(duperModel.getConfigServerApplication()).thenReturn(configServerApplication); + when(monitorFactory.create(any())).thenReturn(monitor); + } + @Test public void addRemove() { - HealthMonitorManager manager = new HealthMonitorManager(); + when(monitorInfra.value()).thenReturn(false); ApplicationInfo applicationInfo = ConfigserverUtil.makeExampleConfigServer(); manager.applicationActivated(applicationInfo); manager.applicationRemoved(applicationInfo.getApplicationId()); @@ -23,7 +54,7 @@ public class HealthMonitorManagerTest { @Test public void withHostAdmin() { - HealthMonitorManager manager = new HealthMonitorManager(); + when(monitorInfra.value()).thenReturn(false); ServiceStatus status = manager.getStatus( ZoneApplication.ZONE_APPLICATION_ID, ClusterId.NODE_ADMIN, @@ -31,4 +62,56 @@ public class HealthMonitorManagerTest { new ConfigId("config-id-1")); assertEquals(ServiceStatus.UP, status); } + + @Test + public void infrastructureApplication() { + when(monitorInfra.value()).thenReturn(false); + + ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); + when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); + List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList()); + ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); + + manager.applicationActivated(proxyHostApplicationInfo); + verify(monitorFactory, never()).create(proxyHostApplicationInfo); + + assertStatus(ServiceStatus.NOT_CHECKED, 0, proxyHostApplication, "proxyhost1"); + } + + @Test + public void infrastructureApplicationWithMonitoring() { + when(monitorInfra.value()).thenReturn(true); + + ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); + when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); + List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::from).collect(Collectors.toList()); + ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); + + manager.applicationActivated(proxyHostApplicationInfo); + verify(monitorFactory, times(1)).create(proxyHostApplicationInfo); + + when(monitor.getStatus(any(), any(), any(), any())).thenReturn(ServiceStatus.UP); + assertStatus(ServiceStatus.UP, 1, proxyHostApplication, "proxyhost1"); + + ControllerHostApplication controllerHostApplication = new ControllerHostApplication(); + when(duperModel.isSupportedInfraApplication(controllerHostApplication.getApplicationId())).thenReturn(true); + assertStatus(ServiceStatus.NOT_CHECKED, 0, controllerHostApplication, "controllerhost1"); + } + + private void assertStatus(ServiceStatus expected, int verifyTimes, InfraApplication infraApplication, String hostname) { + ServiceStatus actual = manager.getStatus( + infraApplication.getApplicationId(), + infraApplication.getClusterId(), + infraApplication.getServiceType(), + infraApplication.configIdFor(HostName.from(hostname))); + + assertEquals(expected, actual); + + verify(monitor, times(verifyTimes)).getStatus( + infraApplication.getApplicationId(), + infraApplication.getClusterId(), + infraApplication.getServiceType(), + infraApplication.configIdFor(HostName.from(hostname))); + + } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java index a59206d14e2..bf3f7017b01 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java @@ -6,16 +6,15 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; -import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.monitor.ConfigserverUtil; import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.monitor.ConfigserverUtil; +import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.service.duper.ConfigServerApplication.CONFIG_SERVER_APPLICATION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -30,6 +29,7 @@ public class ApplicationInstanceGeneratorTest { configServer1, configServer2, configServer3).collect(Collectors.toList()); + private static final ConfigServerApplication configServerApplication = new ConfigServerApplication(); private final ServiceStatusProvider statusProvider = mock(ServiceStatusProvider.class); @@ -43,24 +43,24 @@ public class ApplicationInstanceGeneratorTest { configServer2, configServer3); Zone zone = mock(Zone.class); - ApplicationInfo configServer = CONFIG_SERVER_APPLICATION.makeApplicationInfoFromConfig(config); + ApplicationInfo configServer = configServerApplication.makeApplicationInfoFromConfig(config); ApplicationInstance applicationInstance = new ApplicationInstanceGenerator(configServer, zone) .makeApplicationInstance(statusProvider); assertEquals( - ConfigServerApplication.APPLICATION_INSTANCE_ID, + configServerApplication.getApplicationInstanceId(), applicationInstance.applicationInstanceId()); assertEquals( - ConfigServerApplication.TENANT_ID, + configServerApplication.getTenantId(), applicationInstance.tenantId()); assertEquals( - ConfigServerApplication.TENANT_ID.toString() + - ":" + ConfigServerApplication.APPLICATION_INSTANCE_ID, + configServerApplication.getTenantId().toString() + + ":" + configServerApplication.getApplicationInstanceId(), applicationInstance.reference().toString()); assertEquals( - ConfigServerApplication.CLUSTER_ID, + configServerApplication.getClusterId(), applicationInstance.serviceClusters().iterator().next().clusterId()); assertEquals( diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java index bcc136f80e3..30a49835f03 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java @@ -33,6 +33,7 @@ public class ModelGeneratorTest { private final String REGION = "us-west-1"; private final String HOSTNAME = "hostname"; private final int PORT = 2; + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); @Test public void toApplicationModel() throws Exception { @@ -63,8 +64,7 @@ public class ModelGeneratorTest { ApplicationInstance applicationInstance1 = iterator.next().getValue(); ApplicationInstance applicationInstance2 = iterator.next().getValue(); - if (applicationInstance1.applicationInstanceId().equals( - ConfigServerApplication.APPLICATION_INSTANCE_ID)) { + if (applicationInstance1.applicationInstanceId().equals(configServerApplication.getApplicationInstanceId())) { verifyConfigServerApplication(applicationInstance1); verifyOtherApplication(applicationInstance2); } else { @@ -108,7 +108,6 @@ public class ModelGeneratorTest { private void verifyConfigServerApplication( ApplicationInstance applicationInstance) { - assertEquals(ConfigServerApplication.APPLICATION_INSTANCE_ID, - applicationInstance.applicationInstanceId()); + assertEquals(configServerApplication.getApplicationInstanceId(), applicationInstance.applicationInstanceId()); } } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java index c334fad2334..7f817a0f1e6 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java @@ -33,7 +33,7 @@ public class ConfigserverUtil { String configServerHostname1, String configServerHostname2, String configServerHostname3) { - return ConfigServerApplication.CONFIG_SERVER_APPLICATION.makeApplicationInfoFromConfig(create( + return new ConfigServerApplication().makeApplicationInfoFromConfig(create( true, true, configServerHostname1, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java index c3ee1d078bf..bc6bff6fba8 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImplTest.java @@ -7,6 +7,8 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.service.duper.ConfigServerApplication; +import com.yahoo.vespa.service.duper.DuperModelManager; import org.junit.Before; import org.junit.Test; @@ -25,8 +27,10 @@ public class SlobrokMonitorManagerImplTest { @SuppressWarnings("unchecked") private final Supplier<SlobrokMonitor> slobrokMonitorFactory = mock(Supplier.class); + private final ConfigServerApplication configServerApplication = new ConfigServerApplication(); + private final DuperModelManager duperModelManager = mock(DuperModelManager.class); private final SlobrokMonitorManagerImpl slobrokMonitorManager = - new SlobrokMonitorManagerImpl(slobrokMonitorFactory); + new SlobrokMonitorManagerImpl(slobrokMonitorFactory, duperModelManager); private final SlobrokMonitor slobrokMonitor = mock(SlobrokMonitor.class); private final ApplicationId applicationId = ApplicationId.from("tenant", "app", "instance"); private final ApplicationInfo application = mock(ApplicationInfo.class); @@ -34,6 +38,7 @@ public class SlobrokMonitorManagerImplTest { @Before public void setup() { + when(duperModelManager.getConfigServerApplication()).thenReturn(configServerApplication); when(slobrokMonitorFactory.get()).thenReturn(slobrokMonitor); when(application.getApplicationId()).thenReturn(applicationId); } |