diff options
author | Håkon Hallingstad <hakon@oath.com> | 2017-09-28 16:21:45 +0200 |
---|---|---|
committer | Håkon Hallingstad <hakon@oath.com> | 2017-09-28 16:21:45 +0200 |
commit | 2b145fd26c0b6a59d200a10c1bc107a92f6829f3 (patch) | |
tree | 146bc9182fc265508d1913c5a4ef06edc201c4af /service-monitor/src/main/java/com | |
parent | 12694e5879ae0261d1d92e81f7cb900d31a4a051 (diff) |
Map SuperModel to ServiceModel
Diffstat (limited to 'service-monitor/src/main/java/com')
4 files changed, 327 insertions, 0 deletions
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java new file mode 100644 index 00000000000..5cffcec82b8 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java @@ -0,0 +1,52 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A service/application model of the config server with health status. + */ +public class ConfigServerApplication { + public static final ClusterId CLUSTER_ID = new ClusterId("zone-config-servers"); + public static final ServiceType SERVICE_TYPE = new ServiceType("configserver"); + public static final TenantId TENANT_ID = new TenantId("hosted-vespa"); + public static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("zone-config-servers"); + public static final String CONFIG_ID_PREFIX = "configid."; + + ApplicationInstance<ServiceMonitorStatus> toApplicationInstance(List<String> hostnames) { + Set<ServiceInstance<ServiceMonitorStatus>> serviceInstances = hostnames.stream() + .map(hostname -> new ServiceInstance<>( + new ConfigId(CONFIG_ID_PREFIX + hostname), + new HostName(hostname), + ServiceMonitorStatus.NOT_CHECKED)) + .collect(Collectors.toSet()); + + ServiceCluster<ServiceMonitorStatus> serviceCluster = new ServiceCluster<>( + CLUSTER_ID, + SERVICE_TYPE, + serviceInstances); + + Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = + Stream.of(serviceCluster).collect(Collectors.toSet()); + + ApplicationInstance<ServiceMonitorStatus> applicationInstance = new ApplicationInstance<>( + TENANT_ID, + APPLICATION_INSTANCE_ID, + serviceClusters); + + return applicationInstance; + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java new file mode 100644 index 00000000000..ef418f99d47 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java @@ -0,0 +1,119 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.applicationmodel.ClusterId; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.applicationmodel.ServiceCluster; +import com.yahoo.vespa.applicationmodel.ServiceClusterKey; +import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.applicationmodel.ServiceType; +import com.yahoo.vespa.applicationmodel.TenantId; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Util to convert SuperModel to ServiceModel and application model classes + */ +public class ModelGenerator { + public static final String CLUSTER_ID_PROPERTY_NAME = "clustername"; + + ServiceModel toServiceModel( + SuperModel superModel, + Zone zone, + List<String> configServerHosts, + SlobrokMonitor2 slobrokMonitor) { + Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> applicationInstances = new HashMap<>(); + + for (ApplicationInfo applicationInfo : superModel.getAllApplicationInfos()) { + ApplicationInstance<ServiceMonitorStatus> applicationInstance = toApplicationInstance( + applicationInfo, + zone, + slobrokMonitor); + applicationInstances.put(applicationInstance.reference(), applicationInstance); + } + + // The config server is part of the service model (but not super model) + ConfigServerApplication configServerApplication = new ConfigServerApplication(); + ApplicationInstance<ServiceMonitorStatus> configServerApplicationInstance = + configServerApplication.toApplicationInstance(configServerHosts); + applicationInstances.put(configServerApplicationInstance.reference(), configServerApplicationInstance); + + return new ServiceModel(applicationInstances); + } + + ApplicationInstance<ServiceMonitorStatus> toApplicationInstance( + ApplicationInfo applicationInfo, + Zone zone, + SlobrokMonitor2 slobrokMonitor) { + Map<ServiceClusterKey, Set<ServiceInstance<ServiceMonitorStatus>>> groupedServiceInstances = new HashMap<>(); + + for (HostInfo host : applicationInfo.getModel().getHosts()) { + HostName hostName = new HostName(host.getHostname()); + for (ServiceInfo serviceInfo : host.getServices()) { + ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo); + ServiceInstance<ServiceMonitorStatus> serviceInstance = + toServiceInstance(serviceInfo, hostName, slobrokMonitor); + + if (!groupedServiceInstances.containsKey(serviceClusterKey)) { + groupedServiceInstances.put(serviceClusterKey, new HashSet<>()); + } + groupedServiceInstances.get(serviceClusterKey).add(serviceInstance); + } + } + + Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = groupedServiceInstances.entrySet().stream() + .map(entry -> new ServiceCluster<>( + entry.getKey().clusterId(), + entry.getKey().serviceType(), + entry.getValue())) + .collect(Collectors.toSet()); + + ApplicationInstance<ServiceMonitorStatus> applicationInstance = new ApplicationInstance<>( + new TenantId(applicationInfo.getApplicationId().tenant().toString()), + toApplicationInstanceId(applicationInfo, zone), + serviceClusters); + + return applicationInstance; + } + + ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) { + ClusterId clusterId = new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse("")); + ServiceType serviceType = toServiceType(serviceInfo); + return new ServiceClusterKey(clusterId, serviceType); + } + + ServiceInstance<ServiceMonitorStatus> toServiceInstance( + ServiceInfo serviceInfo, + HostName hostName, + SlobrokMonitor2 slobrokMonitor) { + ConfigId configId = new ConfigId(serviceInfo.getConfigId()); + ServiceMonitorStatus serviceStatus = slobrokMonitor.getStatus(toServiceType(serviceInfo), configId); + return new ServiceInstance<>(configId, hostName,serviceStatus); + } + + ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) { + return new ApplicationInstanceId(String.format("%s:%s:%s:%s", + applicationInfo.getApplicationId().application().value(), + zone.environment().value(), + zone.region().value(), + applicationInfo.getApplicationId().instance().value())); + } + + ServiceType toServiceType(ServiceInfo serviceInfo) { + return new ServiceType(serviceInfo.getServiceType()); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java new file mode 100644 index 00000000000..b39af0238c5 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.vespa.applicationmodel.ApplicationInstance; +import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +/** + * The ServiceModel is almost a mirror of the SuperModel, except that it + * also gives ServiceMonitorStatus on each service, and there may be + * artificial applications like the config server "application". + */ +// @Immutable +public class ServiceModel { + private final Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> applications; + + ServiceModel(Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> applications) { + this.applications = Collections.unmodifiableMap(applications); + } + + Map<ApplicationInstanceReference, + ApplicationInstance<ServiceMonitorStatus>> getAllApplicationInstances() { + return applications; + } + + Optional<ApplicationInstance<ServiceMonitorStatus>> getApplicationInstance(ApplicationInstanceReference reference) { + return Optional.ofNullable(applications.get(reference)); + } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java new file mode 100644 index 00000000000..f0cda0eba37 --- /dev/null +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java @@ -0,0 +1,122 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.service.monitor; + +import com.yahoo.config.model.api.ApplicationInfo; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.PortInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.api.SuperModel; +import com.yahoo.jrt.Spec; +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.jrt.slobrok.api.Mirror; +import com.yahoo.jrt.slobrok.api.SlobrokList; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.applicationmodel.ConfigId; +import com.yahoo.vespa.applicationmodel.ServiceType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + +public class SlobrokMonitor2 implements AutoCloseable { + public static final String SLOBROK_RPC_PORT_TAG = "rpc"; + + private static final Logger log = Logger.getLogger(SlobrokMonitor2.class.getName()); + + private final SlobrokList slobrokList; + private final Mirror mirror; + + SlobrokMonitor2() { + this(new SlobrokList()); + } + + // Package-private for testing. + SlobrokMonitor2(SlobrokList slobrokList, Mirror mirror) { + this.slobrokList = slobrokList; + this.mirror = mirror; + } + + private SlobrokMonitor2(SlobrokList slobrokList) { + this(slobrokList, new Mirror(new Supervisor(new Transport()), slobrokList)); + } + + void updateSlobrokList(SuperModel superModel) { + // If we ever need to optimize this method, then we should make this class + // have a Map<ApplicationId, List<String>>, mapping each application to + // its list of specs. Then, whenever a single application is activated or removed, + // only modify that List<String>. + + List<String> slobrokSpecs = new ArrayList<>(); + + for (ApplicationInfo application : superModel.getAllApplicationInfos()) { + for (HostInfo host : application.getModel().getHosts()) { + for (ServiceInfo service : host.getServices()) { + for (PortInfo port : service.getPorts()) { + if (port.getTags().contains(SLOBROK_RPC_PORT_TAG)) { + Spec spec = new Spec(host.getHostname(), port.getPort()); + slobrokSpecs.add(spec.toString()); + } + } + } + } + } + + slobrokList.setup(slobrokSpecs.toArray(new String[0])); + } + + ServiceMonitorStatus getStatus(ServiceType serviceType, ConfigId configId) { + Optional<String> slobrokServiceName = lookup(serviceType, configId); + if (slobrokServiceName.isPresent()) { + if (mirror.lookup(slobrokServiceName.get()).length != 0) { + return ServiceMonitorStatus.UP; + } else { + return ServiceMonitorStatus.DOWN; + } + } else { + return ServiceMonitorStatus.NOT_CHECKED; + } + } + + @Override + public void close() { + mirror.shutdown(); + } + + // Package-private for testing + Optional<String> lookup(ServiceType serviceType, ConfigId configId) { + switch (serviceType.s()) { + case "adminserver": + case "config-sentinel": + case "configproxy": + case "configserver": + case "filedistributorservice": + case "logd": + case "logserver": + case "metricsproxy": + case "slobrok": + case "transactionlogserver": + case "ytracecleaner": + return Optional.empty(); + + case "topleveldispatch": + return Optional.of(configId.s()); + + case "qrserver": + case "container": + case "docprocservice": + case "container-clustercontroller": + return Optional.of("vespa/service/" + configId.s()); + + case "searchnode": //TODO: handle only as storagenode instead of both as searchnode/storagenode + return Optional.of(configId.s() + "/realtimecontroller"); + case "distributor": + case "storagenode": + return Optional.of("storage/cluster." + configId.s()); + default: + log.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() + " with config id " + configId.s()); + return Optional.empty(); + } + } +} |