diff options
23 files changed, 366 insertions, 116 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java index 3363ddb040f..1d230d6cb47 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java @@ -11,24 +11,28 @@ import java.util.Set; */ public class ApplicationInstance { - private final TenantId tenantId; - private final ApplicationInstanceId applicationInstanceId; + private final ApplicationInstanceReference reference; private final Set<ServiceCluster> serviceClusters; - public ApplicationInstance(TenantId tenantId, ApplicationInstanceId applicationInstanceId, Set<ServiceCluster> serviceClusters) { - this.tenantId = tenantId; - this.applicationInstanceId = applicationInstanceId; + public ApplicationInstance(TenantId tenantId, + ApplicationInstanceId applicationInstanceId, + Set<ServiceCluster> serviceClusters) { + this(new ApplicationInstanceReference(tenantId, applicationInstanceId), serviceClusters); + } + + public ApplicationInstance(ApplicationInstanceReference reference, Set<ServiceCluster> serviceClusters) { + this.reference = reference; this.serviceClusters = serviceClusters; } @JsonProperty("tenantId") public TenantId tenantId() { - return tenantId; + return reference.tenantId(); } @JsonProperty("applicationInstanceId") public ApplicationInstanceId applicationInstanceId() { - return applicationInstanceId; + return reference.applicationInstanceId(); } @JsonProperty("serviceClusters") @@ -38,7 +42,7 @@ public class ApplicationInstance { @JsonProperty("reference") public ApplicationInstanceReference reference() { - return new ApplicationInstanceReference(tenantId, applicationInstanceId); + return reference; } @Override @@ -46,21 +50,19 @@ public class ApplicationInstance { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ApplicationInstance that = (ApplicationInstance) o; - return Objects.equals(tenantId, that.tenantId) && - Objects.equals(applicationInstanceId, that.applicationInstanceId) && + return Objects.equals(reference, that.reference) && Objects.equals(serviceClusters, that.serviceClusters); } @Override public int hashCode() { - return Objects.hash(tenantId, applicationInstanceId, serviceClusters); + return Objects.hash(reference, serviceClusters); } @Override public String toString() { return "ApplicationInstance{" + - "tenantId=" + tenantId + - ", applicationInstanceId=" + applicationInstanceId + + "reference=" + reference + ", serviceClusters=" + serviceClusters + '}'; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index ed6e7cc71ef..e65b7273b9e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -60,8 +60,7 @@ public class MetricsReporter extends Maintainer { @Override public void maintain() { NodeList nodes = nodeRepository().list(); - Map<HostName, List<ServiceInstance>> servicesByHost = - serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName(); + Map<HostName, List<ServiceInstance>> servicesByHost = serviceMonitor.getServicesByHostname(); nodes.forEach(node -> updateNodeMetrics(node, servicesByHost)); updateStateMetrics(nodes); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index ca53b215237..8a5a119f6a7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; -import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.yolean.Exceptions; @@ -187,7 +186,7 @@ public class NodeFailer extends Maintainer { Map<String, Node> activeNodesByHostname = nodeRepository().getNodes(Node.State.active).stream() .collect(Collectors.toMap(Node::hostname, node -> node)); - serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName() + serviceMonitor.getServicesByHostname() .forEach((hostName, serviceInstances) -> { Node node = activeNodesByHostname.get(hostName.s()); if (node == null) return; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 321812497bd..8daec1d641e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -26,7 +26,6 @@ import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; -import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; import org.junit.Test; @@ -97,9 +96,7 @@ public class MetricsReporterTest { when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1))) ); - ServiceModel serviceModel = mock(ServiceModel.class); - when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); + when(serviceMonitor.getServicesByHostname()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); MetricsReporter metricsReporter = new MetricsReporter( @@ -146,9 +143,7 @@ public class MetricsReporterTest { Orchestrator orchestrator = mock(Orchestrator.class); ServiceMonitor serviceMonitor = mock(ServiceMonitor.class); when(orchestrator.getHostResolver()).thenReturn(hostName -> Optional.of(HostInfo.createNoRemarks())); - ServiceModel serviceModel = mock(ServiceModel.class); - when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel); - when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of()); + when(serviceMonitor.getServicesByHostname()).thenReturn(Map.of()); TestMetric metric = new TestMetric(); ManualClock clock = new ManualClock(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java index 554b6e11501..acf3fe7f9b2 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java @@ -1,10 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; -import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -16,5 +18,5 @@ public interface InstanceLookupService { Optional<ApplicationInstance> findInstanceById(ApplicationInstanceReference applicationInstanceReference); Optional<ApplicationInstance> findInstanceByHost(HostName hostName); Set<ApplicationInstanceReference> knownInstances(); - + List<ServiceInstance> findServicesOnHost(HostName hostName); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index d0062966a6d..6639a3032dc 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -106,6 +106,7 @@ public class OrchestratorImpl implements Orchestrator { @Override public Host getHost(HostName hostName) throws HostNameNotFoundException { ApplicationInstance applicationInstance = getApplicationInstance(hostName); + instanceLookupService.findServicesOnHost(hostName); List<ServiceInstance> serviceInstances = applicationInstance .serviceClusters().stream() .flatMap(cluster -> cluster.serviceInstances().stream()) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java index 1a859cfacc5..30f5233d0c6 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java @@ -5,13 +5,12 @@ import com.google.inject.Inject; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * Uses slobrok data (a.k.a. heartbeat) to implement {@link InstanceLookupService}. @@ -29,17 +28,21 @@ public class ServiceMonitorInstanceLookupService implements InstanceLookupServic @Override public Optional<ApplicationInstance> findInstanceById(ApplicationInstanceReference applicationInstanceReference) { - return serviceMonitor.getServiceModelSnapshot().getApplicationInstance(applicationInstanceReference); + return serviceMonitor.getApplication(applicationInstanceReference); } @Override - public Optional<ApplicationInstance> findInstanceByHost(HostName hostName) { - return Optional.ofNullable(serviceMonitor.getServiceModelSnapshot().getApplicationsByHostName().get(hostName)); + public Optional<ApplicationInstance> findInstanceByHost(HostName hostname) { + return serviceMonitor.getApplication(hostname); } @Override public Set<ApplicationInstanceReference> knownInstances() { - return serviceMonitor.getServiceModelSnapshot().getAllApplicationInstances().keySet(); + return serviceMonitor.getAllApplicationInstanceReferences(); } + @Override + public List<ServiceInstance> findServicesOnHost(HostName hostName) { + return serviceMonitor.getServiceInstancesOn(hostName); + } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java index fbb8f445db0..cfdda3250f6 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java @@ -33,7 +33,6 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.time.Instant; import java.util.List; -import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -70,8 +69,8 @@ public class InstanceResource { @GET @Produces(MediaType.APPLICATION_JSON) - public Set<ApplicationInstanceReference> getAllInstances() { - return instanceLookupService.knownInstances(); + public List<ApplicationInstanceReference> getAllInstances() { + return instanceLookupService.knownInstances().stream().sorted().collect(Collectors.toList()); } @GET diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java index a54f5284ee0..07acc0dad81 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.orchestrator.model.NodeGroup; import com.yahoo.vespa.orchestrator.model.VespaModelUtil; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -154,6 +155,16 @@ public class DummyInstanceLookupService implements InstanceLookupService { } + @Override + public List<ServiceInstance> findServicesOnHost(HostName hostName) { + return apps.stream() + .flatMap(application -> application.serviceClusters().stream()) + .flatMap(cluster -> cluster.serviceInstances().stream()) + .filter(service -> service.hostName().equals(hostName)) + .sorted() + .collect(Collectors.toList()); + } + public static Set<HostName> getContentHosts(ApplicationInstanceReference appRef) { Set<HostName> hosts = apps.stream() .filter(application -> application.reference().equals(appRef)) diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java index bfe4b523e4a..0869fde3422 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService; +import com.yahoo.vespa.service.model.ServiceModelCache; import com.yahoo.vespa.service.monitor.ServiceModel; import org.junit.Before; import org.junit.Test; @@ -416,7 +417,7 @@ public class OrchestratorImplTest { ServiceStatus.NOT_CHECKED))))); InstanceLookupService lookupService = new ServiceMonitorInstanceLookupService( - () -> new ServiceModel(Map.of(reference, applicationInstance))); + new ServiceModelCache(() -> new ServiceModel(Map.of(reference, applicationInstance)), new TestTimer())); orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory), clusterControllerClientFactory, diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java index eff222bc074..991b5837111 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java @@ -32,7 +32,9 @@ import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; import com.yahoo.vespa.orchestrator.status.StatusService; import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService; +import com.yahoo.vespa.service.model.ServiceModelCache; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.yolean.Exceptions; import java.time.Clock; @@ -58,10 +60,12 @@ class ModelTestUtils { private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock(); private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>(); private final StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()); + private final TestTimer timer = new TestTimer(); + private final ServiceMonitor serviceMonitor = new ServiceModelCache(() -> new ServiceModel(applications), timer); private final Orchestrator orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory()), clusterControllerClientFactory, statusService, - new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)), + new ServiceMonitorInstanceLookupService(serviceMonitor), 0, new ManualClock(), applicationApiFactory(), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java index bfa68145828..a84f99dff34 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java @@ -110,6 +110,11 @@ public class HostResourceTest { public Set<ApplicationInstanceReference> knownInstances() { return Collections.emptySet(); } + + @Override + public List<ServiceInstance> findServicesOnHost(HostName hostName) { + return List.of(); + } }; private static class AlwaysAllowPolicy implements Policy { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 1cfb70560b8..aa0b9e81c44 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.service.monitor.DuperModelListener; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; import java.util.logging.Logger; @@ -49,6 +50,10 @@ public class DuperModel { } } + public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) { + return Optional.ofNullable(applications.get(applicationId)); + } + public List<ApplicationInfo> getApplicationInfos() { logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size()); return List.copyOf(applications.values()); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 15c461c7f59..9c819d89fb2 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -173,6 +173,12 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi } } + public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) { + synchronized (monitor) { + return duperModel.getApplicationInfo(applicationId); + } + } + public List<ApplicationInfo> getApplicationInfos() { synchronized (monitor) { return duperModel.getApplicationInfos(); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index 5cc2d538c24..db15c2789d5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -5,9 +5,13 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.HostName; @@ -24,7 +28,9 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -37,23 +43,100 @@ public class ApplicationInstanceGenerator { private final ApplicationInfo applicationInfo; private final Zone zone; - private ApplicationId configServerApplicationId; + + // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. + // We do this to avoid passing through the ID through multiple levels. + private static ApplicationId configServerApplicationId = new ConfigServerApplication().getApplicationId(); public ApplicationInstanceGenerator(ApplicationInfo applicationInfo, Zone zone) { this.applicationInfo = applicationInfo; this.zone = zone; - - // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. - // We do this to avoid passing through the ID through multiple levels. - this.configServerApplicationId = new ConfigServerApplication().getApplicationId(); } public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) { + return makeApplicationInstanceLimitedToHosts(serviceStatusProvider, hostname -> true); + } + + public static ServiceId getServiceId(ApplicationInfo applicationInfo, ServiceInfo serviceInfo) { + return new ServiceId( + applicationInfo.getApplicationId(), + getClusterId(serviceInfo), + toServiceType(serviceInfo), + toConfigId(serviceInfo)); + } + + public ApplicationInstanceReference toApplicationInstanceReference() { + TenantId tenantId = new TenantId(applicationInfo.getApplicationId().tenant().toString()); + ApplicationInstanceId applicationInstanceId = toApplicationInstanceId(applicationInfo.getApplicationId(), zone); + return new ApplicationInstanceReference(tenantId, applicationInstanceId); + } + + public boolean containsHostname(HostName hostname) { + return applicationInfo.getModel().getHosts().stream() + .map(HostInfo::getHostname) + .anyMatch(hostnameString -> Objects.equals(hostnameString, hostname.s())); + } + + public ApplicationInstance makeApplicationInstanceLimitedTo( + HostName hostname, ServiceStatusProvider serviceStatusProvider) { + return makeApplicationInstanceLimitedToHosts( + serviceStatusProvider, candidateHostname -> candidateHostname.equals(hostname)); + } + + /** Reverse of toApplicationInstanceId, put in this file because it its inverse is. */ + public static ApplicationId toApplicationId(ApplicationInstanceReference reference) { + + String appNameStr = reference.asString(); + String[] appNameParts = appNameStr.split(":"); + + // Env, region and instance seems to be optional due to the hardcoded config server app + // Assume here that first two are tenant and application name. + if (appNameParts.length == 2) { + return ApplicationId.from(TenantName.from(appNameParts[0]), + ApplicationName.from(appNameParts[1]), + InstanceName.defaultName()); + } + + // Other normal application should have 5 parts. + if (appNameParts.length != 5) { + throw new IllegalArgumentException("Application reference not valid (not 5 parts): " + reference); + } + + return ApplicationId.from(TenantName.from(appNameParts[0]), + ApplicationName.from(appNameParts[1]), + InstanceName.from(appNameParts[4])); + } + + private static ApplicationInstanceId toApplicationInstanceId(ApplicationId applicationId, Zone zone) { + if (applicationId.equals(configServerApplicationId)) { + // Removing this historical discrepancy would break orchestration during rollout. + // An alternative may be to use a feature flag and flip it between releases, + // once that's available. + return new ApplicationInstanceId(applicationId.application().value()); + } else { + return new ApplicationInstanceId(String.format("%s:%s:%s:%s", + applicationId.application().value(), + zone.environment().value(), + zone.region().value(), + applicationId.instance().value())); + } + } + + private static ClusterId getClusterId(ServiceInfo serviceInfo) { + return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); + } + + private ApplicationInstance makeApplicationInstanceLimitedToHosts(ServiceStatusProvider serviceStatusProvider, + Predicate<HostName> includeHostPredicate) { Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>(); for (HostInfo host : applicationInfo.getModel().getHosts()) { HostName hostName = new HostName(host.getHostname()); + if (!includeHostPredicate.test(hostName)) { + continue; + } + for (ServiceInfo serviceInfo : host.getServices()) { ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); @@ -77,10 +160,8 @@ public class ApplicationInstanceGenerator { entry.getValue())) .collect(Collectors.toSet()); - ApplicationInstance applicationInstance = new ApplicationInstance( - new TenantId(applicationInfo.getApplicationId().tenant().toString()), - toApplicationInstanceId(applicationInfo, zone), - serviceClusters); + ApplicationInstanceReference reference = toApplicationInstanceReference(); + ApplicationInstance applicationInstance = new ApplicationInstance(reference, serviceClusters); // Fill back-references for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) { @@ -106,33 +187,6 @@ public class ApplicationInstanceGenerator { return new ServiceInstance(configId, hostName, status); } - private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { - if (applicationInfo.getApplicationId().equals(configServerApplicationId)) { - // Removing this historical discrepancy would break orchestration during rollout. - // An alternative may be to use a feature flag and flip it between releases, - // once that's available. - return new ApplicationInstanceId(applicationInfo.getApplicationId().application().value()); - } else { - return new ApplicationInstanceId(String.format("%s:%s:%s:%s", - applicationInfo.getApplicationId().application().value(), - zone.environment().value(), - zone.region().value(), - applicationInfo.getApplicationId().instance().value())); - } - } - - public static ServiceId getServiceId(ApplicationInfo applicationInfo, ServiceInfo serviceInfo) { - return new ServiceId( - applicationInfo.getApplicationId(), - getClusterId(serviceInfo), - toServiceType(serviceInfo), - toConfigId(serviceInfo)); - } - - public static ClusterId getClusterId(ServiceInfo serviceInfo) { - return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); - } - private static ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) { ClusterId clusterId = getClusterId(serviceInfo); ServiceType serviceType = toServiceType(serviceInfo); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java index a6bf41d6c9b..45a9d1c77c5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java @@ -5,11 +5,15 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -18,6 +22,11 @@ import java.util.stream.Collectors; */ public class ModelGenerator { public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + private final Zone zone; + + public ModelGenerator(Zone zone) { + this.zone = zone; + } /** * Create service model based primarily on super model. @@ -25,7 +34,6 @@ public class ModelGenerator { * If the configServerhosts is non-empty, a config server application is added. */ public ServiceModel toServiceModel(List<ApplicationInfo> allApplicationInfos, - Zone zone, ServiceStatusProvider serviceStatusProvider) { Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances = allApplicationInfos.stream() @@ -36,4 +44,48 @@ public class ModelGenerator { return new ServiceModel(applicationInstances); } + public Set<ApplicationInstanceReference> toApplicationInstanceReferenceSet(List<ApplicationInfo> infos) { + return infos.stream() + .map(info -> new ApplicationInstanceGenerator(info, zone).toApplicationInstanceReference()) + .collect(Collectors.toSet()); + } + + public Optional<ApplicationInstance> toApplication(List<ApplicationInfo> applicationInfos, + HostName hostname, + ServiceStatusProvider serviceStatusProvider) { + for (var applicationInfo : applicationInfos) { + var generator = new ApplicationInstanceGenerator(applicationInfo, zone); + if (generator.containsHostname(hostname)) { + return Optional.of(generator.makeApplicationInstance(serviceStatusProvider)); + } + } + + return Optional.empty(); + } + + public ApplicationInstance toApplication(ApplicationInfo applicationInfo, + ServiceStatusProvider serviceStatusProvider) { + var generator = new ApplicationInstanceGenerator(applicationInfo, zone); + return generator.makeApplicationInstance(serviceStatusProvider); + } + + public List<ServiceInstance> toServices(List<ApplicationInfo> applicationInfos, + HostName hostname, + ServiceStatusProvider serviceStatusProvider) { + for (var applicationInfo : applicationInfos) { + var generator = new ApplicationInstanceGenerator(applicationInfo, zone); + ApplicationInstance applicationInstance = generator.makeApplicationInstanceLimitedTo( + hostname, serviceStatusProvider); + + List<ServiceInstance> serviceInstances = applicationInstance.serviceClusters().stream() + .flatMap(cluster -> cluster.serviceInstances().stream()) + .collect(Collectors.toList()); + + if (serviceInstances.size() > 0) { + return serviceInstances; + } + } + + return List.of(); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java index 1b37555a554..29b2832efd0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.service.model; import com.yahoo.jdisc.Timer; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.util.function.Supplier; @@ -12,12 +13,11 @@ import java.util.function.Supplier; * * @author hakonhall */ -public class ServiceModelCache implements Supplier<ServiceModel> { +public class ServiceModelCache implements ServiceMonitor { public static final long EXPIRY_MILLIS = 10000; private final Supplier<ServiceModel> expensiveSupplier; private final Timer timer; - private final boolean useCache; private volatile ServiceModel snapshot; private boolean updatePossiblyInProgress = false; @@ -25,18 +25,13 @@ public class ServiceModelCache implements Supplier<ServiceModel> { private final Object updateMonitor = new Object(); private long snapshotMillis; - public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer, boolean useCache) { + public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer) { this.expensiveSupplier = expensiveSupplier; this.timer = timer; - this.useCache = useCache; } @Override - public ServiceModel get() { - if (!useCache) { - return expensiveSupplier.get(); - } - + public ServiceModel getServiceModelSnapshot() { if (snapshot == null) { synchronized (updateMonitor) { if (snapshot == null) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java index c6947810fa0..641bc5ca7ff 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java @@ -2,33 +2,40 @@ package com.yahoo.vespa.service.model; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.manager.MonitorManager; -import com.yahoo.vespa.service.duper.DuperModelManager; import java.util.List; -import java.util.function.Supplier; +import java.util.Map; +import java.util.Optional; +import java.util.Set; /** * An uncached supplier of ServiceModel based on the DuperModel and MonitorManager. * * @author hakonhall */ -public class ServiceModelProvider implements Supplier<ServiceModel> { +public class ServiceModelProvider implements ServiceMonitor { private final ServiceMonitorMetrics metrics; private final DuperModelManager duperModelManager; private final ModelGenerator modelGenerator; private final Zone zone; - private final MonitorManager monitorManager; + private final ServiceStatusProvider serviceStatusProvider; - public ServiceModelProvider(MonitorManager monitorManager, + public ServiceModelProvider(ServiceStatusProvider serviceStatusProvider, ServiceMonitorMetrics metrics, DuperModelManager duperModelManager, ModelGenerator modelGenerator, Zone zone) { - this.monitorManager = monitorManager; + this.serviceStatusProvider = serviceStatusProvider; this.metrics = metrics; this.duperModelManager = duperModelManager; this.modelGenerator = modelGenerator; @@ -36,13 +43,46 @@ public class ServiceModelProvider implements Supplier<ServiceModel> { } @Override - public ServiceModel get() { + public ServiceModel getServiceModelSnapshot() { try (LatencyMeasurement measurement = metrics.startServiceModelSnapshotLatencyMeasurement()) { - // WARNING: The monitor manager may be out-of-sync with duper model (no locking) - List<ApplicationInfo> applicationInfos = duperModelManager.getApplicationInfos(); - - return modelGenerator.toServiceModel(applicationInfos, zone, (ServiceStatusProvider) monitorManager); + return modelGenerator.toServiceModel(applicationInfos(), serviceStatusProvider); } } + @Override + public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return modelGenerator.toApplicationInstanceReferenceSet(applicationInfos()); + } + + @Override + public Optional<ApplicationInstance> getApplication(HostName hostname) { + // TODO(hakonhall): Lookup the applicationInfo from the hostname. + return modelGenerator.toApplication(applicationInfos(), hostname, serviceStatusProvider); + } + + @Override + public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return getApplicationInfo(reference) + .map(applicationInfo -> modelGenerator.toApplication(applicationInfo, serviceStatusProvider)); + } + + @Override + public List<ServiceInstance> getServiceInstancesOn(HostName hostname) { + // TODO(hakonhall): Lookup the applicationInfo from the hostname. + return modelGenerator.toServices(applicationInfos(), hostname, serviceStatusProvider); + } + + @Override + public Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return getServiceModelSnapshot().getServiceInstancesByHostName(); + } + + private Optional<ApplicationInfo> getApplicationInfo(ApplicationInstanceReference reference) { + ApplicationId applicationId = ApplicationInstanceGenerator.toApplicationId(reference); + return duperModelManager.getApplicationInfo(applicationId); + } + + private List<ApplicationInfo> applicationInfos() { + return duperModelManager.getApplicationInfos(); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 67b4e890c29..a37a52f775e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -5,6 +5,10 @@ import com.google.inject.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; @@ -12,9 +16,14 @@ import com.yahoo.vespa.service.manager.UnionMonitorManager; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + public class ServiceMonitorImpl implements ServiceMonitor { - private final ServiceModelCache serviceModelProvider; + private final ServiceMonitor delegate; @Inject public ServiceMonitorImpl(DuperModelManager duperModelManager, @@ -25,19 +34,47 @@ public class ServiceMonitorImpl implements ServiceMonitor { FlagSource flagSource) { duperModelManager.registerListener(monitorManager); - ServiceModelProvider uncachedServiceModelProvider = new ServiceModelProvider( + ServiceMonitor serviceMonitor = new ServiceModelProvider( monitorManager, new ServiceMonitorMetrics(metric, timer), duperModelManager, - new ModelGenerator(), + new ModelGenerator(zone), zone); - boolean cache = Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value(); - serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer, cache); + + if (Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value()) { + delegate = new ServiceModelCache(serviceMonitor::getServiceModelSnapshot, timer); + } else { + delegate = serviceMonitor; + } } @Override public ServiceModel getServiceModelSnapshot() { - return serviceModelProvider.get(); + return delegate.getServiceModelSnapshot(); + } + + @Override + public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return delegate.getAllApplicationInstanceReferences(); + } + + @Override + public Optional<ApplicationInstance> getApplication(HostName hostname) { + return delegate.getApplication(hostname); } + @Override + public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return delegate.getApplication(reference); + } + + @Override + public List<ServiceInstance> getServiceInstancesOn(HostName hostname) { + return delegate.getServiceInstancesOn(hostname); + } + + @Override + public Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return delegate.getServicesByHostname(); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java index 49539c61e5d..8ad77213bc1 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java @@ -1,6 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + /** * The service monitor interface. A service monitor provides up to date information about the liveness status * (up, down or not known) of each service instance in a Vespa zone @@ -12,7 +23,37 @@ public interface ServiceMonitor { /** * Returns a ServiceModel which contains the current liveness status (up, down or unknown) of all instances * of all services of all clusters of all applications in a zone. + * Deprecated: Please use the more specific methods below. */ ServiceModel getServiceModelSnapshot(); + default Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return getServiceModelSnapshot().getAllApplicationInstances().keySet(); + } + + default Optional<ApplicationInstance> getApplication(HostName hostname) { + return Optional.ofNullable(getServiceModelSnapshot().getApplicationsByHostName().get(hostname)); + } + + default Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return getServiceModelSnapshot().getApplicationInstance(reference); + } + + default List<ServiceInstance> getServiceInstancesOn(HostName hostname) { + ApplicationInstance application = getServiceModelSnapshot().getApplicationsByHostName().get(hostname); + if (application == null) { + return List.of(); + } + + return application + .serviceClusters().stream() + .flatMap(cluster -> cluster.serviceInstances().stream()) + .filter(serviceInstance -> hostname.equals(serviceInstance.hostName())) + .collect(Collectors.toList()); + } + + default Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return getServiceModelSnapshot().getServiceInstancesByHostName(); + } + } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java index ce13cf60082..7be4ff38a7f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java @@ -38,9 +38,9 @@ public class ModelGeneratorTest { @Test public void toApplicationModel() throws Exception { - ModelGenerator modelGenerator = new ModelGenerator(); - Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); + ModelGenerator modelGenerator = new ModelGenerator(zone); + SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) @@ -49,7 +49,6 @@ public class ModelGeneratorTest { ServiceModel serviceModel = modelGenerator.toServiceModel( getExampleApplicationInfos(), - zone, slobrokMonitorManager); Map<ApplicationInstanceReference, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java index c2314be1e0f..21c6a49cc18 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java @@ -16,27 +16,27 @@ import static org.mockito.Mockito.when; public class ServiceModelCacheTest { @SuppressWarnings("unchecked") - private final Supplier<ServiceModel> rawSupplier = mock(Supplier.class); + private final Supplier<ServiceModel> expensiveServiceMonitor = mock(Supplier.class); private final Timer timer = mock(Timer.class); - private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer, true); + private final ServiceModelCache cache = new ServiceModelCache(expensiveServiceMonitor, timer); @Test public void sanityCheck() { ServiceModel serviceModel = mock(ServiceModel.class); - when(rawSupplier.get()).thenReturn(serviceModel); + when(expensiveServiceMonitor.get()).thenReturn(serviceModel); long timeMillis = 0; when(timer.currentTimeMillis()).thenReturn(timeMillis); // Will always populate cache the first time - ServiceModel actualServiceModel = cache.get(); + ServiceModel actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel); - verify(rawSupplier, times(1)).get(); + verify(expensiveServiceMonitor, times(1)).get(); // Cache hit timeMillis += ServiceModelCache.EXPIRY_MILLIS / 2; when(timer.currentTimeMillis()).thenReturn(timeMillis); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel); // Cache expired @@ -44,17 +44,17 @@ public class ServiceModelCacheTest { when(timer.currentTimeMillis()).thenReturn(timeMillis); ServiceModel serviceModel2 = mock(ServiceModel.class); - when(rawSupplier.get()).thenReturn(serviceModel2); + when(expensiveServiceMonitor.get()).thenReturn(serviceModel2); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel2); // '2' because it's cumulative with '1' from the first times(1). - verify(rawSupplier, times(2)).get(); + verify(expensiveServiceMonitor, times(2)).get(); // Cache hit #2 timeMillis += 1; when(timer.currentTimeMillis()).thenReturn(timeMillis); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel2); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java index 13f6da1534d..f34d61970ef 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java @@ -37,8 +37,8 @@ public class ServiceModelProviderTest { .collect(Collectors.toList()); when(duperModelManager.getApplicationInfos()).thenReturn(applications); - ServiceModel serviceModel = provider.get(); + ServiceModel serviceModel = provider.getServiceModelSnapshot(); verify(duperModelManager, times(1)).getApplicationInfos(); - verify(modelGenerator).toServiceModel(applications, zone, slobrokMonitorManager); + verify(modelGenerator).toServiceModel(applications, slobrokMonitorManager); } }
\ No newline at end of file |