diff options
Diffstat (limited to 'service-monitor/src')
13 files changed, 418 insertions, 82 deletions
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 1cfb70560b8..1065ed29dc4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -2,14 +2,19 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.log.LogLevel; import com.yahoo.vespa.service.monitor.DuperModelListener; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.TreeMap; +import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; /** @@ -20,37 +25,106 @@ import java.util.logging.Logger; public class DuperModel { private static Logger logger = Logger.getLogger(DuperModel.class.getName()); - private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>(); + private final Map<ApplicationId, ApplicationInfo> applicationsById = new HashMap<>(); + private final Map<HostName, ApplicationId> idsByHostname = new HashMap<>(); + private final Map<ApplicationId, Set<HostName>> hostnamesById = new HashMap<>(); + private final List<DuperModelListener> listeners = new ArrayList<>(); private boolean isComplete = false; public void registerListener(DuperModelListener listener) { - applications.values().forEach(listener::applicationActivated); + applicationsById.values().forEach(listener::applicationActivated); listeners.add(listener); } - public void setCompleteness(boolean isComplete) { this.isComplete = isComplete; } + void setComplete() { + if (!isComplete) { + logger.log(LogLevel.INFO, "Bootstrap done - duper model is complete"); + isComplete = true; + + listeners.forEach(DuperModelListener::bootstrapComplete); + } + } + public boolean isComplete() { return isComplete; } + public int numberOfApplications() { + return applicationsById.size(); + } + + public int numberOfHosts() { + return idsByHostname.size(); + } + public boolean contains(ApplicationId applicationId) { - return applications.containsKey(applicationId); + return applicationsById.containsKey(applicationId); + } + + public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) { + return Optional.ofNullable(applicationsById.get(applicationId)); + } + + public Optional<ApplicationInfo> getApplicationInfo(HostName hostName) { + return Optional.ofNullable(idsByHostname.get(hostName)).map(applicationsById::get); + } + + public List<ApplicationInfo> getApplicationInfos() { + return List.copyOf(applicationsById.values()); } public void add(ApplicationInfo applicationInfo) { - applications.put(applicationInfo.getApplicationId(), applicationInfo); - logger.log(LogLevel.DEBUG, "Added " + applicationInfo.getApplicationId()); + ApplicationId id = applicationInfo.getApplicationId(); + ApplicationInfo oldApplicationInfo = applicationsById.put(id, applicationInfo); + + final String logPrefix; + if (oldApplicationInfo == null) { + logPrefix = isComplete ? "New application " : "Bootstrapped application "; + } else { + logPrefix = isComplete ? "Reactivated application " : "Rebootstrapped application "; + } + logger.log(LogLevel.INFO, logPrefix + id); + + Set<HostName> hostnames = hostnamesById.computeIfAbsent(id, k -> new HashSet<>()); + Set<HostName> removedHosts = new HashSet<>(hostnames); + + applicationInfo.getModel().getHosts().stream() + .map(HostInfo::getHostname) + .map(HostName::from) + .forEach(hostname -> { + if (!removedHosts.remove(hostname)) { + hostnames.add(hostname); + ApplicationId previousId = idsByHostname.put(hostname, id); + + if (previousId != null && !previousId.equals(id)) { + // If an activation contains a host that is currently assigned to a + // different application we will patch up our data structures to remain + // internally consistent. But listeners may be fooled. + logger.log(LogLevel.WARNING, hostname + " has been reassigned from " + + previousId + " to " + id); + + Set<HostName> previousHostnames = hostnamesById.get(previousId); + if (previousHostnames != null) { + previousHostnames.remove(hostname); + } + } + } + }); + + removedHosts.forEach(idsByHostname::remove); + listeners.forEach(listener -> listener.applicationActivated(applicationInfo)); } public void remove(ApplicationId applicationId) { - if (applications.remove(applicationId) != null) { - logger.log(LogLevel.DEBUG, "Removed " + applicationId); - listeners.forEach(listener -> listener.applicationRemoved(applicationId)); + Set<HostName> hostnames = hostnamesById.remove(applicationId); + if (hostnames != null) { + hostnames.forEach(idsByHostname::remove); } - } - public List<ApplicationInfo> getApplicationInfos() { - logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size()); - return List.copyOf(applications.values()); + ApplicationInfo application = applicationsById.remove(applicationId); + if (application != null) { + logger.log(LogLevel.INFO, "Removed application " + applicationId); + listeners.forEach(listener -> listener.applicationRemoved(applicationId)); + } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index 15c461c7f59..9c93bc1d390 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -173,6 +173,18 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi } } + public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) { + synchronized (monitor) { + return duperModel.getApplicationInfo(applicationId); + } + } + + public Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { + synchronized (monitor) { + return duperModel.getApplicationInfo(hostname); + } + } + public List<ApplicationInfo> getApplicationInfos() { synchronized (monitor) { return duperModel.getApplicationInfos(); @@ -181,7 +193,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi private void maybeSetDuperModelAsComplete() { if (superModelIsComplete && infraApplicationsIsComplete) { - duperModel.setCompleteness(true); + duperModel.setComplete(); } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java index 5cc2d538c24..0116b992e23 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java @@ -5,9 +5,13 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.HostName; @@ -24,7 +28,9 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -37,23 +43,73 @@ public class ApplicationInstanceGenerator { private final ApplicationInfo applicationInfo; private final Zone zone; - private ApplicationId configServerApplicationId; + + // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. + // We do this to avoid passing through the ID through multiple levels. + private static final ApplicationId configServerApplicationId = new ConfigServerApplication().getApplicationId(); public ApplicationInstanceGenerator(ApplicationInfo applicationInfo, Zone zone) { this.applicationInfo = applicationInfo; this.zone = zone; - - // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different. - // We do this to avoid passing through the ID through multiple levels. - this.configServerApplicationId = new ConfigServerApplication().getApplicationId(); } public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) { + return makeApplicationInstanceLimitedToHosts(serviceStatusProvider, hostname -> true); + } + + public ApplicationInstanceReference toApplicationInstanceReference() { + TenantId tenantId = new TenantId(applicationInfo.getApplicationId().tenant().toString()); + ApplicationInstanceId applicationInstanceId = toApplicationInstanceId(applicationInfo.getApplicationId(), zone); + return new ApplicationInstanceReference(tenantId, applicationInstanceId); + } + + public boolean containsHostname(HostName hostname) { + return applicationInfo.getModel().getHosts().stream() + .map(HostInfo::getHostname) + .anyMatch(hostnameString -> Objects.equals(hostnameString, hostname.s())); + } + + public ApplicationInstance makeApplicationInstanceLimitedTo( + HostName hostname, ServiceStatusProvider serviceStatusProvider) { + return makeApplicationInstanceLimitedToHosts( + serviceStatusProvider, candidateHostname -> candidateHostname.equals(hostname)); + } + + /** Reverse of toApplicationInstanceId, put in this file because it its inverse is. */ + public static ApplicationId toApplicationId(ApplicationInstanceReference reference) { + + String appNameStr = reference.asString(); + String[] appNameParts = appNameStr.split(":"); + + // Env, region and instance seems to be optional due to the hardcoded config server app + // Assume here that first two are tenant and application name. + if (appNameParts.length == 2) { + return ApplicationId.from(TenantName.from(appNameParts[0]), + ApplicationName.from(appNameParts[1]), + InstanceName.defaultName()); + } + + // Other normal application should have 5 parts. + if (appNameParts.length != 5) { + throw new IllegalArgumentException("Application reference not valid (not 5 parts): " + reference); + } + + return ApplicationId.from(TenantName.from(appNameParts[0]), + ApplicationName.from(appNameParts[1]), + InstanceName.from(appNameParts[4])); + } + + private ApplicationInstance makeApplicationInstanceLimitedToHosts(ServiceStatusProvider serviceStatusProvider, + Predicate<HostName> includeHostPredicate) { Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>(); for (HostInfo host : applicationInfo.getModel().getHosts()) { HostName hostName = new HostName(host.getHostname()); + if (!includeHostPredicate.test(hostName)) { + continue; + } + for (ServiceInfo serviceInfo : host.getServices()) { ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); @@ -77,10 +133,8 @@ public class ApplicationInstanceGenerator { entry.getValue())) .collect(Collectors.toSet()); - ApplicationInstance applicationInstance = new ApplicationInstance( - new TenantId(applicationInfo.getApplicationId().tenant().toString()), - toApplicationInstanceId(applicationInfo, zone), - serviceClusters); + ApplicationInstanceReference reference = toApplicationInstanceReference(); + ApplicationInstance applicationInstance = new ApplicationInstance(reference, serviceClusters); // Fill back-references for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) { @@ -106,18 +160,18 @@ public class ApplicationInstanceGenerator { return new ServiceInstance(configId, hostName, status); } - private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { - if (applicationInfo.getApplicationId().equals(configServerApplicationId)) { + private static ApplicationInstanceId toApplicationInstanceId(ApplicationId applicationId, Zone zone) { + if (applicationId.equals(configServerApplicationId)) { // Removing this historical discrepancy would break orchestration during rollout. // An alternative may be to use a feature flag and flip it between releases, // once that's available. - return new ApplicationInstanceId(applicationInfo.getApplicationId().application().value()); + return new ApplicationInstanceId(applicationId.application().value()); } else { return new ApplicationInstanceId(String.format("%s:%s:%s:%s", - applicationInfo.getApplicationId().application().value(), + applicationId.application().value(), zone.environment().value(), zone.region().value(), - applicationInfo.getApplicationId().instance().value())); + applicationId.instance().value())); } } @@ -129,7 +183,7 @@ public class ApplicationInstanceGenerator { toConfigId(serviceInfo)); } - public static ClusterId getClusterId(ServiceInfo serviceInfo) { + private static ClusterId getClusterId(ServiceInfo serviceInfo) { return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java index a6bf41d6c9b..d9378da957e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java @@ -5,11 +5,13 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -19,13 +21,18 @@ import java.util.stream.Collectors; public class ModelGenerator { public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + private final Zone zone; + + public ModelGenerator(Zone zone) { + this.zone = zone; + } + /** * Create service model based primarily on super model. * * If the configServerhosts is non-empty, a config server application is added. */ public ServiceModel toServiceModel(List<ApplicationInfo> allApplicationInfos, - Zone zone, ServiceStatusProvider serviceStatusProvider) { Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances = allApplicationInfos.stream() @@ -36,4 +43,27 @@ public class ModelGenerator { return new ServiceModel(applicationInstances); } + public Set<ApplicationInstanceReference> toApplicationInstanceReferenceSet(List<ApplicationInfo> infos) { + return infos.stream() + .map(info -> new ApplicationInstanceGenerator(info, zone).toApplicationInstanceReference()) + .collect(Collectors.toSet()); + } + + public ApplicationInstance toApplicationInstance(ApplicationInfo applicationInfo, + ServiceStatusProvider serviceStatusProvider) { + var generator = new ApplicationInstanceGenerator(applicationInfo, zone); + return generator.makeApplicationInstance(serviceStatusProvider); + } + + /** + * Make an application instance that contains all services and clusters present on the host, + * but lacking other services and hosts. This is an optimization over + * {@link #toApplicationInstance(ApplicationInfo, ServiceStatusProvider)}. + */ + public ApplicationInstance toApplicationNarrowedToHost(ApplicationInfo applicationInfo, + HostName hostname, + ServiceStatusProvider serviceStatusProvider) { + var generator = new ApplicationInstanceGenerator(applicationInfo, zone); + return generator.makeApplicationInstanceLimitedTo(hostname, serviceStatusProvider); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java index 1b37555a554..29b2832efd0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.service.model; import com.yahoo.jdisc.Timer; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import java.util.function.Supplier; @@ -12,12 +13,11 @@ import java.util.function.Supplier; * * @author hakonhall */ -public class ServiceModelCache implements Supplier<ServiceModel> { +public class ServiceModelCache implements ServiceMonitor { public static final long EXPIRY_MILLIS = 10000; private final Supplier<ServiceModel> expensiveSupplier; private final Timer timer; - private final boolean useCache; private volatile ServiceModel snapshot; private boolean updatePossiblyInProgress = false; @@ -25,18 +25,13 @@ public class ServiceModelCache implements Supplier<ServiceModel> { private final Object updateMonitor = new Object(); private long snapshotMillis; - public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer, boolean useCache) { + public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer) { this.expensiveSupplier = expensiveSupplier; this.timer = timer; - this.useCache = useCache; } @Override - public ServiceModel get() { - if (!useCache) { - return expensiveSupplier.get(); - } - + public ServiceModel getServiceModelSnapshot() { if (snapshot == null) { synchronized (updateMonitor) { if (snapshot == null) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java index c6947810fa0..45ee38ab560 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java @@ -2,33 +2,40 @@ package com.yahoo.vespa.service.model; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.monitor.ServiceModel; +import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import com.yahoo.vespa.service.manager.MonitorManager; -import com.yahoo.vespa.service.duper.DuperModelManager; import java.util.List; -import java.util.function.Supplier; +import java.util.Map; +import java.util.Optional; +import java.util.Set; /** * An uncached supplier of ServiceModel based on the DuperModel and MonitorManager. * * @author hakonhall */ -public class ServiceModelProvider implements Supplier<ServiceModel> { +public class ServiceModelProvider implements ServiceMonitor { private final ServiceMonitorMetrics metrics; private final DuperModelManager duperModelManager; private final ModelGenerator modelGenerator; private final Zone zone; - private final MonitorManager monitorManager; + private final ServiceStatusProvider serviceStatusProvider; - public ServiceModelProvider(MonitorManager monitorManager, + public ServiceModelProvider(ServiceStatusProvider serviceStatusProvider, ServiceMonitorMetrics metrics, DuperModelManager duperModelManager, ModelGenerator modelGenerator, Zone zone) { - this.monitorManager = monitorManager; + this.serviceStatusProvider = serviceStatusProvider; this.metrics = metrics; this.duperModelManager = duperModelManager; this.modelGenerator = modelGenerator; @@ -36,13 +43,57 @@ public class ServiceModelProvider implements Supplier<ServiceModel> { } @Override - public ServiceModel get() { + public ServiceModel getServiceModelSnapshot() { try (LatencyMeasurement measurement = metrics.startServiceModelSnapshotLatencyMeasurement()) { - // WARNING: The monitor manager may be out-of-sync with duper model (no locking) - List<ApplicationInfo> applicationInfos = duperModelManager.getApplicationInfos(); + return modelGenerator.toServiceModel(duperModelManager.getApplicationInfos(), serviceStatusProvider); + } + } + + @Override + public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return modelGenerator.toApplicationInstanceReferenceSet(duperModelManager.getApplicationInfos()); + } - return modelGenerator.toServiceModel(applicationInfos, zone, (ServiceStatusProvider) monitorManager); + @Override + public Optional<ApplicationInstance> getApplication(HostName hostname) { + Optional<ApplicationInfo> applicationInfo = getApplicationInfo(hostname); + if (applicationInfo.isEmpty()) { + return Optional.empty(); } + + return Optional.of(modelGenerator.toApplicationInstance(applicationInfo.get(), serviceStatusProvider)); + } + + @Override + public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return getApplicationInfo(reference) + .map(applicationInfo -> modelGenerator.toApplicationInstance(applicationInfo, serviceStatusProvider)); } + @Override + public Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) { + Optional<ApplicationInfo> applicationInfo = getApplicationInfo(hostname); + if (applicationInfo.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(modelGenerator.toApplicationNarrowedToHost( + applicationInfo.get(), hostname, serviceStatusProvider)); + } + + @Override + public Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return getServiceModelSnapshot().getServiceInstancesByHostName(); + } + + private Optional<ApplicationInfo> getApplicationInfo(ApplicationInstanceReference reference) { + ApplicationId applicationId = ApplicationInstanceGenerator.toApplicationId(reference); + return duperModelManager.getApplicationInfo(applicationId); + } + + private Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { + // The duper model uses HostName from config.provision, which is more natural than applicationmodel. + var configProvisionHostname = com.yahoo.config.provision.HostName.from(hostname.s()); + return duperModelManager.getApplicationInfo(configProvisionHostname); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 67b4e890c29..d3297d711ff 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -5,6 +5,10 @@ import com.google.inject.Inject; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.service.duper.DuperModelManager; @@ -12,9 +16,14 @@ import com.yahoo.vespa.service.manager.UnionMonitorManager; import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + public class ServiceMonitorImpl implements ServiceMonitor { - private final ServiceModelCache serviceModelProvider; + private final ServiceMonitor delegate; @Inject public ServiceMonitorImpl(DuperModelManager duperModelManager, @@ -25,19 +34,47 @@ public class ServiceMonitorImpl implements ServiceMonitor { FlagSource flagSource) { duperModelManager.registerListener(monitorManager); - ServiceModelProvider uncachedServiceModelProvider = new ServiceModelProvider( + ServiceMonitor serviceMonitor = new ServiceModelProvider( monitorManager, new ServiceMonitorMetrics(metric, timer), duperModelManager, - new ModelGenerator(), + new ModelGenerator(zone), zone); - boolean cache = Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value(); - serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer, cache); + + if (Flags.SERVICE_MODEL_CACHE.bindTo(flagSource).value()) { + delegate = new ServiceModelCache(serviceMonitor::getServiceModelSnapshot, timer); + } else { + delegate = serviceMonitor; + } } @Override public ServiceModel getServiceModelSnapshot() { - return serviceModelProvider.get(); + return delegate.getServiceModelSnapshot(); + } + + @Override + public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return delegate.getAllApplicationInstanceReferences(); + } + + @Override + public Optional<ApplicationInstance> getApplication(HostName hostname) { + return delegate.getApplication(hostname); } + @Override + public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return delegate.getApplication(reference); + } + + @Override + public Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) { + return delegate.getApplicationNarrowedTo(hostname); + } + + @Override + public Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return delegate.getServicesByHostname(); + } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java index 49539c61e5d..deafaaace0e 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java @@ -1,6 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceInstance; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + /** * The service monitor interface. A service monitor provides up to date information about the liveness status * (up, down or not known) of each service instance in a Vespa zone @@ -12,7 +22,29 @@ public interface ServiceMonitor { /** * Returns a ServiceModel which contains the current liveness status (up, down or unknown) of all instances * of all services of all clusters of all applications in a zone. + * + * <p>Please use the more specific methods below to avoid the cost of this method.</p> */ ServiceModel getServiceModelSnapshot(); + default Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() { + return getServiceModelSnapshot().getAllApplicationInstances().keySet(); + } + + default Optional<ApplicationInstance> getApplication(HostName hostname) { + return Optional.ofNullable(getServiceModelSnapshot().getApplicationsByHostName().get(hostname)); + } + + default Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) { + return getServiceModelSnapshot().getApplicationInstance(reference); + } + + default Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) { + return getApplication(hostname); + } + + default Map<HostName, List<ServiceInstance>> getServicesByHostname() { + return getServiceModelSnapshot().getServiceInstancesByHostName(); + } + } diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java index 67508f14e5a..3448ce46ff9 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java @@ -71,7 +71,6 @@ public class DuperModelManagerTest { verify(duperModel, times(0)).add(any()); manager.infraApplicationActivated(id, proxyHostHosts); verify(duperModel, times(1)).add(any()); - when(duperModel.contains(id)).thenReturn(true); verify(duperModel, times(0)).remove(any()); manager.infraApplicationRemoved(id); @@ -98,7 +97,6 @@ public class DuperModelManagerTest { List<HostName> hostnames1 = Stream.of("node11", "node12").map(HostName::from).collect(Collectors.toList()); manager.infraApplicationActivated(firstId, hostnames1); verify(duperModel, times(1)).add(any()); - when(duperModel.contains(firstId)).thenReturn(true); // Adding the second config server like application will be ignored List<HostName> hostnames2 = Stream.of("node21", "node22").map(HostName::from).collect(Collectors.toList()); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java index dc90035be71..69b6d3d59f3 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java @@ -2,16 +2,20 @@ package com.yahoo.vespa.service.duper; import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.DuperModelListener; import org.junit.Before; import org.junit.Test; import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -23,37 +27,60 @@ import static org.mockito.Mockito.when; */ public class DuperModelTest { private final DuperModel duperModel = new DuperModel(); - private final ApplicationInfo application1 = mock(ApplicationInfo.class); + private final ApplicationId id1 = ApplicationId.fromSerializedForm("tenant:app1:default"); + private final ApplicationInfo application1 = mock(ApplicationInfo.class); + private final HostName hostname1_1 = HostName.from("hostname1-1"); + private final HostName hostname1_2 = HostName.from("hostname1-2"); + private final ApplicationId id2 = ApplicationId.fromSerializedForm("tenant:app2:default"); private final ApplicationInfo application2 = mock(ApplicationInfo.class); + private final HostName hostname2_1 = HostName.from("hostname2-1"); + private final DuperModelListener listener1 = mock(DuperModelListener.class); @Before public void setUp() { - when(application1.getApplicationId()).thenReturn(id1); - when(application2.getApplicationId()).thenReturn(id2); + setUpApplication(id1, application1, hostname1_1, hostname1_2); + setUpApplication(id2, application2, hostname2_1); + } + + private void setUpApplication(ApplicationId id, ApplicationInfo info, HostName... hostnames) { + when(info.getApplicationId()).thenReturn(id); + + Model model = mock(Model.class); + when(info.getModel()).thenReturn(model); + + List<HostInfo> hostInfos = Arrays.stream(hostnames) + .map(hostname -> new HostInfo(hostname.value(), List.of())) + .collect(Collectors.toList()); + when(model.getHosts()).thenReturn(hostInfos); } @Test - public void test() { + public void testListeners() { + assertEquals(0, duperModel.numberOfApplications()); + duperModel.add(application1); - assertTrue(duperModel.contains(id1)); + assertEquals(Optional.of(application1), duperModel.getApplicationInfo(id1)); assertEquals(Arrays.asList(application1), duperModel.getApplicationInfos()); + assertEquals(1, duperModel.numberOfApplications()); duperModel.registerListener(listener1); verify(listener1, times(1)).applicationActivated(application1); verifyNoMoreInteractions(listener1); duperModel.remove(id2); + assertEquals(1, duperModel.numberOfApplications()); verifyNoMoreInteractions(listener1); duperModel.add(application2); + assertEquals(2, duperModel.numberOfApplications()); verify(listener1, times(1)).applicationActivated(application2); verifyNoMoreInteractions(listener1); duperModel.remove(id1); - assertFalse(duperModel.contains(id1)); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(id1)); verify(listener1, times(1)).applicationRemoved(id1); verifyNoMoreInteractions(listener1); assertEquals(Arrays.asList(application2), duperModel.getApplicationInfos()); @@ -61,4 +88,31 @@ public class DuperModelTest { duperModel.remove(id1); verifyNoMoreInteractions(listener1); } + + @Test + public void hostIndices() { + assertEquals(0, duperModel.numberOfHosts()); + + duperModel.add(application1); + assertEquals(2, duperModel.numberOfHosts()); + assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1)); + + duperModel.add(application2); + assertEquals(3, duperModel.numberOfHosts()); + assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1)); + + duperModel.remove(application1.getApplicationId()); + assertEquals(1, duperModel.numberOfHosts()); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1)); + + // Remove hostname2_1 and add hostname1_1 added to id2 + setUpApplication(id2, application2, hostname1_1); + duperModel.add(application2); + assertEquals(1, duperModel.numberOfHosts()); + assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname1_1)); + assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1)); + } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java index ce13cf60082..7be4ff38a7f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java @@ -38,9 +38,9 @@ public class ModelGeneratorTest { @Test public void toApplicationModel() throws Exception { - ModelGenerator modelGenerator = new ModelGenerator(); - Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION)); + ModelGenerator modelGenerator = new ModelGenerator(zone); + SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class); when(slobrokMonitorManager.getStatus(any(), any(), any(), any())) @@ -49,7 +49,6 @@ public class ModelGeneratorTest { ServiceModel serviceModel = modelGenerator.toServiceModel( getExampleApplicationInfos(), - zone, slobrokMonitorManager); Map<ApplicationInstanceReference, diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java index c2314be1e0f..21c6a49cc18 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java @@ -16,27 +16,27 @@ import static org.mockito.Mockito.when; public class ServiceModelCacheTest { @SuppressWarnings("unchecked") - private final Supplier<ServiceModel> rawSupplier = mock(Supplier.class); + private final Supplier<ServiceModel> expensiveServiceMonitor = mock(Supplier.class); private final Timer timer = mock(Timer.class); - private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer, true); + private final ServiceModelCache cache = new ServiceModelCache(expensiveServiceMonitor, timer); @Test public void sanityCheck() { ServiceModel serviceModel = mock(ServiceModel.class); - when(rawSupplier.get()).thenReturn(serviceModel); + when(expensiveServiceMonitor.get()).thenReturn(serviceModel); long timeMillis = 0; when(timer.currentTimeMillis()).thenReturn(timeMillis); // Will always populate cache the first time - ServiceModel actualServiceModel = cache.get(); + ServiceModel actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel); - verify(rawSupplier, times(1)).get(); + verify(expensiveServiceMonitor, times(1)).get(); // Cache hit timeMillis += ServiceModelCache.EXPIRY_MILLIS / 2; when(timer.currentTimeMillis()).thenReturn(timeMillis); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel); // Cache expired @@ -44,17 +44,17 @@ public class ServiceModelCacheTest { when(timer.currentTimeMillis()).thenReturn(timeMillis); ServiceModel serviceModel2 = mock(ServiceModel.class); - when(rawSupplier.get()).thenReturn(serviceModel2); + when(expensiveServiceMonitor.get()).thenReturn(serviceModel2); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel2); // '2' because it's cumulative with '1' from the first times(1). - verify(rawSupplier, times(2)).get(); + verify(expensiveServiceMonitor, times(2)).get(); // Cache hit #2 timeMillis += 1; when(timer.currentTimeMillis()).thenReturn(timeMillis); - actualServiceModel = cache.get(); + actualServiceModel = cache.getServiceModelSnapshot(); assertTrue(actualServiceModel == serviceModel2); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java index 13f6da1534d..f34d61970ef 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java @@ -37,8 +37,8 @@ public class ServiceModelProviderTest { .collect(Collectors.toList()); when(duperModelManager.getApplicationInfos()).thenReturn(applications); - ServiceModel serviceModel = provider.get(); + ServiceModel serviceModel = provider.getServiceModelSnapshot(); verify(duperModelManager, times(1)).getApplicationInfos(); - verify(modelGenerator).toServiceModel(applications, zone, slobrokMonitorManager); + verify(modelGenerator).toServiceModel(applications, slobrokMonitorManager); } }
\ No newline at end of file |