Map SuperModel to ServiceModel

4 files changed, 327 insertions, 0 deletions

diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java
new file mode 100644
index 00000000000..5cffcec82b8
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ConfigServerApplication.java
@@ -0,0 +1,52 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.applicationmodel.TenantId;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A service/application model of the config server with health status.
+ */
+public class ConfigServerApplication {
+    public static final ClusterId CLUSTER_ID = new ClusterId("zone-config-servers");
+    public static final ServiceType SERVICE_TYPE = new ServiceType("configserver");
+    public static final TenantId TENANT_ID = new TenantId("hosted-vespa");
+    public static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("zone-config-servers");
+    public static final String CONFIG_ID_PREFIX = "configid.";
+
+    ApplicationInstance<ServiceMonitorStatus> toApplicationInstance(List<String> hostnames) {
+        Set<ServiceInstance<ServiceMonitorStatus>> serviceInstances = hostnames.stream()
+                .map(hostname -> new ServiceInstance<>(
+                        new ConfigId(CONFIG_ID_PREFIX + hostname),
+                        new HostName(hostname),
+                        ServiceMonitorStatus.NOT_CHECKED))
+                .collect(Collectors.toSet());
+
+        ServiceCluster<ServiceMonitorStatus> serviceCluster = new ServiceCluster<>(
+                CLUSTER_ID,
+                SERVICE_TYPE,
+                serviceInstances);
+
+        Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters =
+                Stream.of(serviceCluster).collect(Collectors.toSet());
+
+        ApplicationInstance<ServiceMonitorStatus> applicationInstance = new ApplicationInstance<>(
+                TENANT_ID,
+                APPLICATION_INSTANCE_ID,
+                serviceClusters);
+
+        return applicationInstance;
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java
new file mode 100644
index 00000000000..ef418f99d47
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ModelGenerator.java
@@ -0,0 +1,119 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.model.api.SuperModel;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.ClusterId;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.applicationmodel.ServiceCluster;
+import com.yahoo.vespa.applicationmodel.ServiceClusterKey;
+import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+import com.yahoo.vespa.applicationmodel.TenantId;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Util to convert SuperModel to ServiceModel and application model classes
+ */
+public class ModelGenerator {
+    public static final String CLUSTER_ID_PROPERTY_NAME = "clustername";
+
+    ServiceModel toServiceModel(
+            SuperModel superModel,
+            Zone zone,
+            List<String> configServerHosts,
+            SlobrokMonitor2 slobrokMonitor) {
+        Map<ApplicationInstanceReference, ApplicationInstance<ServiceMonitorStatus>> applicationInstances = new HashMap<>();
+
+        for (ApplicationInfo applicationInfo : superModel.getAllApplicationInfos()) {
+            ApplicationInstance<ServiceMonitorStatus> applicationInstance = toApplicationInstance(
+                    applicationInfo,
+                    zone,
+                    slobrokMonitor);
+            applicationInstances.put(applicationInstance.reference(), applicationInstance);
+        }
+
+        // The config server is part of the service model (but not super model)
+        ConfigServerApplication configServerApplication = new ConfigServerApplication();
+        ApplicationInstance<ServiceMonitorStatus> configServerApplicationInstance =
+                configServerApplication.toApplicationInstance(configServerHosts);
+        applicationInstances.put(configServerApplicationInstance.reference(), configServerApplicationInstance);
+
+        return new ServiceModel(applicationInstances);
+    }
+
+    ApplicationInstance<ServiceMonitorStatus> toApplicationInstance(
+            ApplicationInfo applicationInfo,
+            Zone zone,
+            SlobrokMonitor2 slobrokMonitor) {
+        Map<ServiceClusterKey, Set<ServiceInstance<ServiceMonitorStatus>>> groupedServiceInstances = new HashMap<>();
+
+        for (HostInfo host : applicationInfo.getModel().getHosts()) {
+            HostName hostName = new HostName(host.getHostname());
+            for (ServiceInfo serviceInfo : host.getServices()) {
+                ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo);
+                ServiceInstance<ServiceMonitorStatus> serviceInstance =
+                        toServiceInstance(serviceInfo, hostName, slobrokMonitor);
+
+                if (!groupedServiceInstances.containsKey(serviceClusterKey)) {
+                    groupedServiceInstances.put(serviceClusterKey, new HashSet<>());
+                }
+                groupedServiceInstances.get(serviceClusterKey).add(serviceInstance);
+            }
+        }
+
+        Set<ServiceCluster<ServiceMonitorStatus>> serviceClusters = groupedServiceInstances.entrySet().stream()
+                .map(entry -> new ServiceCluster<>(
+                        entry.getKey().clusterId(),
+                        entry.getKey().serviceType(),
+                        entry.getValue()))
+                .collect(Collectors.toSet());
+
+        ApplicationInstance<ServiceMonitorStatus> applicationInstance = new ApplicationInstance<>(
+                new TenantId(applicationInfo.getApplicationId().tenant().toString()),
+                toApplicationInstanceId(applicationInfo, zone),
+                serviceClusters);
+
+        return applicationInstance;
+    }
+
+    ServiceClusterKey toServiceClusterKey(ServiceInfo serviceInfo) {
+        ClusterId clusterId = new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse(""));
+        ServiceType serviceType = toServiceType(serviceInfo);
+        return new ServiceClusterKey(clusterId, serviceType);
+    }
+
+    ServiceInstance<ServiceMonitorStatus> toServiceInstance(
+            ServiceInfo serviceInfo,
+            HostName hostName,
+            SlobrokMonitor2 slobrokMonitor) {
+        ConfigId configId = new ConfigId(serviceInfo.getConfigId());
+        ServiceMonitorStatus serviceStatus = slobrokMonitor.getStatus(toServiceType(serviceInfo), configId);
+        return new ServiceInstance<>(configId, hostName,serviceStatus);
+    }
+
+    ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) {
+        return new ApplicationInstanceId(String.format("%s:%s:%s:%s",
+                applicationInfo.getApplicationId().application().value(),
+                zone.environment().value(),
+                zone.region().value(),
+                applicationInfo.getApplicationId().instance().value()));
+    }
+
+    ServiceType toServiceType(ServiceInfo serviceInfo) {
+        return new ServiceType(serviceInfo.getServiceType());
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
new file mode 100644
index 00000000000..b39af0238c5
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
@@ -0,0 +1,34 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * The ServiceModel is almost a mirror of the SuperModel, except that it
+ * also gives ServiceMonitorStatus on each service, and there may be
+ * artificial applications like the config server "application".
+ */
+// @Immutable
+public class ServiceModel {
+    private final Map<ApplicationInstanceReference,
+            ApplicationInstance<ServiceMonitorStatus>> applications;
+
+    ServiceModel(Map<ApplicationInstanceReference,
+            ApplicationInstance<ServiceMonitorStatus>> applications) {
+        this.applications = Collections.unmodifiableMap(applications);
+    }
+
+    Map<ApplicationInstanceReference,
+            ApplicationInstance<ServiceMonitorStatus>> getAllApplicationInstances() {
+        return applications;
+    }
+
+    Optional<ApplicationInstance<ServiceMonitorStatus>> getApplicationInstance(ApplicationInstanceReference reference) {
+        return Optional.ofNullable(applications.get(reference));
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java
new file mode 100644
index 00000000000..f0cda0eba37
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor2.java
@@ -0,0 +1,122 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.PortInfo;
+import com.yahoo.config.model.api.ServiceInfo;
+import com.yahoo.config.model.api.SuperModel;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Transport;
+import com.yahoo.jrt.slobrok.api.Mirror;
+import com.yahoo.jrt.slobrok.api.SlobrokList;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.applicationmodel.ConfigId;
+import com.yahoo.vespa.applicationmodel.ServiceType;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+public class SlobrokMonitor2 implements AutoCloseable {
+    public static final String SLOBROK_RPC_PORT_TAG = "rpc";
+
+    private static final Logger log = Logger.getLogger(SlobrokMonitor2.class.getName());
+
+    private final SlobrokList slobrokList;
+    private final Mirror mirror;
+
+    SlobrokMonitor2() {
+        this(new SlobrokList());
+    }
+
+    // Package-private for testing.
+    SlobrokMonitor2(SlobrokList slobrokList, Mirror mirror) {
+        this.slobrokList = slobrokList;
+        this.mirror = mirror;
+    }
+
+    private SlobrokMonitor2(SlobrokList slobrokList) {
+        this(slobrokList, new Mirror(new Supervisor(new Transport()), slobrokList));
+    }
+
+    void updateSlobrokList(SuperModel superModel) {
+        // If we ever need to optimize this method, then we should make this class
+        // have a Map<ApplicationId, List<String>>, mapping each application to
+        // its list of specs. Then, whenever a single application is activated or removed,
+        // only modify that List<String>.
+
+        List<String> slobrokSpecs = new ArrayList<>();
+
+        for (ApplicationInfo application : superModel.getAllApplicationInfos()) {
+            for (HostInfo host : application.getModel().getHosts()) {
+                for (ServiceInfo service : host.getServices()) {
+                    for (PortInfo port : service.getPorts()) {
+                        if (port.getTags().contains(SLOBROK_RPC_PORT_TAG)) {
+                            Spec spec = new Spec(host.getHostname(), port.getPort());
+                            slobrokSpecs.add(spec.toString());
+                        }
+                    }
+                }
+            }
+        }
+
+        slobrokList.setup(slobrokSpecs.toArray(new String[0]));
+    }
+
+    ServiceMonitorStatus getStatus(ServiceType serviceType, ConfigId configId) {
+        Optional<String> slobrokServiceName = lookup(serviceType, configId);
+        if (slobrokServiceName.isPresent()) {
+            if (mirror.lookup(slobrokServiceName.get()).length != 0) {
+                return ServiceMonitorStatus.UP;
+            } else {
+                return ServiceMonitorStatus.DOWN;
+            }
+        } else {
+            return ServiceMonitorStatus.NOT_CHECKED;
+        }
+    }
+
+    @Override
+    public void close() {
+        mirror.shutdown();
+    }
+
+    // Package-private for testing
+    Optional<String> lookup(ServiceType serviceType, ConfigId configId) {
+        switch (serviceType.s()) {
+            case "adminserver":
+            case "config-sentinel":
+            case "configproxy":
+            case "configserver":
+            case "filedistributorservice":
+            case "logd":
+            case "logserver":
+            case "metricsproxy":
+            case "slobrok":
+            case "transactionlogserver":
+            case "ytracecleaner":
+                return Optional.empty();
+
+            case "topleveldispatch":
+                return Optional.of(configId.s());
+
+            case "qrserver":
+            case "container":
+            case "docprocservice":
+            case "container-clustercontroller":
+                return Optional.of("vespa/service/" + configId.s());
+
+            case "searchnode": //TODO: handle only as storagenode instead of both as searchnode/storagenode
+                return Optional.of(configId.s() + "/realtimecontroller");
+            case "distributor":
+            case "storagenode":
+                return Optional.of("storage/cluster." + configId.s());
+            default:
+                log.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() + " with config id " + configId.s());
+                return Optional.empty();
+        }
+    }
+}