From a34bb9650b92590ced290da1b2d131aabda10aac Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sun, 8 Mar 2020 14:15:17 +0100 Subject: Avoid building lots of ApplicationInstances Avoid building a full ApplicationInstance for each node... - for all nodes in the node repo when reporting metrics repo every minute, and - for all nodes in any /nodes/v1/node response --- .../vespa/service/model/ServiceMonitorImpl.java | 32 +++++---- .../yahoo/vespa/service/monitor/ServiceModel.java | 77 ++++++++++++++-------- .../vespa/service/monitor/ServiceMonitor.java | 13 ++-- 3 files changed, 71 insertions(+), 51 deletions(-) (limited to 'service-monitor') diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 678b0d3d5ba..e6e6e85c710 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -10,7 +10,6 @@ import com.yahoo.jdisc.Timer; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.service.duper.DuperModelManager; import com.yahoo.vespa.service.manager.MonitorManager; import com.yahoo.vespa.service.manager.UnionMonitorManager; @@ -21,8 +20,6 @@ import com.yahoo.vespa.service.monitor.ServiceModel; import com.yahoo.vespa.service.monitor.ServiceMonitor; import com.yahoo.vespa.service.monitor.ServiceStatusProvider; -import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; @@ -70,13 +67,16 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor { } @Override - public Optional getApplication(HostName hostname) { - Optional applicationInfo = getApplicationInfo(hostname); - if (applicationInfo.isEmpty()) { - return Optional.empty(); - } + public Optional getApplicationInstanceReference(HostName hostname) { + return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname)) + .map(ApplicationInfo::getApplicationId) + .map(modelGenerator::toApplicationInstanceReference); + } - return Optional.of(modelGenerator.toApplicationInstance(applicationInfo.get(), serviceStatusProvider)); + @Override + public Optional getApplication(HostName hostname) { + return getApplicationInfo(hostname) + .map(applicationInfo -> modelGenerator.toApplicationInstance(applicationInfo, serviceStatusProvider)); } @Override @@ -96,11 +96,6 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor { applicationInfo.get(), hostname, serviceStatusProvider)); } - @Override - public Map> getServicesByHostname() { - return getServiceModelSnapshot().getServiceInstancesByHostName(); - } - @Override public void registerListener(ServiceHostListener listener) { var duperModelListener = ServiceHostListenerAdapter.asDuperModelListener(listener, modelGenerator); @@ -118,8 +113,11 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor { } private Optional getApplicationInfo(HostName hostname) { - // The duper model uses HostName from config.provision, which is more natural than applicationmodel. - var configProvisionHostname = com.yahoo.config.provision.HostName.from(hostname.s()); - return duperModelManager.getApplicationInfo(configProvisionHostname); + return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname)); + } + + /** The duper model uses HostName from config.provision. */ + private static com.yahoo.config.provision.HostName toConfigProvisionHostName(HostName hostname) { + return com.yahoo.config.provision.HostName.from(hostname.s()); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java index ed72893400a..c8f491a0fc7 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java @@ -1,66 +1,91 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; /** - * The ServiceModel is almost a mirror of the SuperModel, except that it - * also gives ServiceStatus on each service, and there may be - * artificial applications like the config server "application". + * The service model is the union of the duper model and the service monitor, and presented + * as classes from the {@code application-model} module. + * + *

The duper model contains the latest {@link ApplicationInfo} of both tenant and infrastructure + * applications. The service monitor provides Slobrok or {@code /state/v1/health} information + * on (most) services. The application model presents an application as a set of clusters, and + * each cluster a set of services, and each service is associated with a particular host + * and has a health status.

+ * + * @author hakonhall */ public class ServiceModel { - private final Map applications; - private final Map applicationsByHostName; + private final Map applicationsByReference; - public ServiceModel(Map applications) { - this.applications = Collections.unmodifiableMap(applications); - this.applicationsByHostName = Collections.unmodifiableMap(applicationsByHostNames(applications.values())); + private Map applicationsByHostName = null; + private Map> servicesByHostName = null; + + public ServiceModel(Map applicationsByReference) { + this.applicationsByReference = Collections.unmodifiableMap(Map.copyOf(applicationsByReference)); } public Map getAllApplicationInstances() { - return applications; + return applicationsByReference; } public Optional getApplicationInstance(ApplicationInstanceReference reference) { - return Optional.ofNullable(applications.get(reference)); + return Optional.ofNullable(applicationsByReference.get(reference)); } - public Map getApplicationsByHostName() { - return applicationsByHostName; + public Optional getApplication(HostName hostname) { + if (applicationsByHostName == null) { + fillMaps(); + } + + return Optional.ofNullable(applicationsByHostName.get(hostname)); } public Map> getServiceInstancesByHostName() { - return applications.values().stream() - .flatMap(application -> application.serviceClusters().stream()) - .flatMap(cluster -> cluster.serviceInstances().stream()) - .collect(Collectors.groupingBy(ServiceInstance::hostName, Collectors.toList())); + if (servicesByHostName == null) { + fillMaps(); + } + + return servicesByHostName; } - private static Map applicationsByHostNames(Collection applications) { - Map hosts = new HashMap<>(); - for (ApplicationInstance application : applications) - for (ServiceCluster cluster : application.serviceClusters()) + private void fillMaps() { + Map applicationInstances = new HashMap<>(); + Map> serviceInstances = new HashMap<>(); + + for (ApplicationInstance application : applicationsByReference.values()) { + for (ServiceCluster cluster : application.serviceClusters()) { for (ServiceInstance instance : cluster.serviceInstances()) { - ApplicationInstance previous = hosts.put(instance.hostName(), application); - if (previous != null && ! previous.equals(application)) + + ApplicationInstance previous = applicationInstances.put(instance.hostName(), application); + if (previous != null && !previous.equals(application)) { throw new IllegalStateException("Major assumption broken: Multiple application instances contain host " + - instance.hostName().s() + ": " + Arrays.asList(previous, application)); + instance.hostName().s() + ": " + Arrays.asList(previous, application)); + } + + serviceInstances + .computeIfAbsent(instance.hostName(), key -> new ArrayList<>()) + .add(instance); } - return hosts; + } + } + + applicationsByHostName = Collections.unmodifiableMap(applicationInstances); + servicesByHostName = Collections.unmodifiableMap(serviceInstances); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java index 0d162fee182..f5767229365 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java @@ -4,10 +4,7 @@ package com.yahoo.vespa.service.monitor; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; -import com.yahoo.vespa.applicationmodel.ServiceInstance; -import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; @@ -31,8 +28,12 @@ public interface ServiceMonitor { return getServiceModelSnapshot().getAllApplicationInstances().keySet(); } + default Optional getApplicationInstanceReference(HostName hostname) { + return getServiceModelSnapshot().getApplication(hostname).map(ApplicationInstance::reference); + } + default Optional getApplication(HostName hostname) { - return Optional.ofNullable(getServiceModelSnapshot().getApplicationsByHostName().get(hostname)); + return getServiceModelSnapshot().getApplication(hostname); } default Optional getApplication(ApplicationInstanceReference reference) { @@ -43,10 +44,6 @@ public interface ServiceMonitor { return getApplication(hostname); } - default Map> getServicesByHostname() { - return getServiceModelSnapshot().getServiceInstancesByHostName(); - } - /** * Get notified of changes to the set of applications, or set of hosts assigned to an application. * -- cgit v1.2.3 From ab0fc576e409c72c21d14ce7d3e473b4d4d7005a Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sun, 8 Mar 2020 16:58:41 +0100 Subject: Delegate to potentially overloaded method --- .../src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'service-monitor') diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java index f5767229365..4f52d7d229c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java @@ -29,7 +29,7 @@ public interface ServiceMonitor { } default Optional getApplicationInstanceReference(HostName hostname) { - return getServiceModelSnapshot().getApplication(hostname).map(ApplicationInstance::reference); + return getApplication(hostname).map(ApplicationInstance::reference); } default Optional getApplication(HostName hostname) { -- cgit v1.2.3