From a739b0aafcd87e4f115be86fc9ad98cf8e06c829 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 20 Oct 2017 09:21:27 +0200 Subject: Remove Scala ServiceMonitor --- service-monitor/pom.xml | 48 ----- .../vespa/service/monitor/ServiceNameUtil.java | 88 --------- .../vespa/service/monitor/SlobrokMonitor.java | 99 ---------- .../service/monitor/SlobrokServiceNameUtil.java | 61 ------- .../monitor/config/ConfigObservableCreator.java | 61 ------- .../resources/config-models/service-monitor.xml | 4 - .../monitor/SlobrokAndConfigIntersector.scala | 202 --------------------- .../monitor/config/InstancesObservables.scala | 117 ------------ .../monitor/ConfigServerApplicationTest.java | 5 +- .../vespa/service/monitor/ServiceNameUtilTest.java | 39 ---- 10 files changed, 2 insertions(+), 722 deletions(-) delete mode 100644 service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceNameUtil.java delete mode 100644 service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor.java delete mode 100644 service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokServiceNameUtil.java delete mode 100644 service-monitor/src/main/java/com/yahoo/vespa/service/monitor/config/ConfigObservableCreator.java delete mode 100644 service-monitor/src/main/resources/config-models/service-monitor.xml delete mode 100644 service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/SlobrokAndConfigIntersector.scala delete mode 100644 service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/config/InstancesObservables.scala delete mode 100644 service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ServiceNameUtilTest.java (limited to 'service-monitor') diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 80958ca9bee..71273ae589b 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -15,20 +15,6 @@ Service monitor component for hosted vespa. - - org.scala-lang - scala-library - - - io.reactivex - rxjava - 1.0.7 - - - io.reactivex - rxscala_${scala.major-version} - 0.23.1 - com.yahoo.vespa config @@ -71,17 +57,6 @@ ${project.version} provided - - org.json4s - json4s-native_${scala.major-version} - test - - - org.scala-lang - scalap - - - com.yahoo.vespa annotations @@ -111,33 +86,10 @@ - - net.alchim31.maven - scala-maven-plugin - - - - add-source - compile - testCompile - - - - compile-scala-classes-for-use-in-java-classes - process-resources - - compile - - - - com.yahoo.vespa bundle-plugin true - - sun.misc - diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceNameUtil.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceNameUtil.java deleted file mode 100644 index 4f77c5fb0c7..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceNameUtil.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor; - -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * This class provides utilities for working with slobrok-registered service names. - * - * @author bakksjo - */ -public class ServiceNameUtil { - // Utility class; prevents instantiation. - private ServiceNameUtil() { - } - - static Set convertSlobrokServicesToConfigIds(final Set registeredServices) { - return registeredServices.stream() - .map(ALL_RECOGNIZER) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toSet()); - } - - // This is basically a typedef. - private interface ServiceNameRecognizer extends Function> {} - - private static class RegexpServiceNameRecognizer implements ServiceNameRecognizer { - private final Pattern pattern; - private final Function nameConverter; - - public RegexpServiceNameRecognizer( - final String patternString, - final Function nameConverter) { - this.pattern = Pattern.compile(patternString); - this.nameConverter = nameConverter; - } - - @Override - public Optional apply(final String serviceName) { - final Matcher matcher = pattern.matcher(serviceName); - if (!matcher.matches()) { - return Optional.empty(); - } - return Optional.of(nameConverter.apply(matcher)); - } - } - - private static class SingleGroupRegexpServiceNameRecognizer extends RegexpServiceNameRecognizer { - public SingleGroupRegexpServiceNameRecognizer(final String patternString) { - super(patternString, matcher -> matcher.group(1)); - } - } - - - // TODO: The regexps below almost certainly hard-code names that are dynamically set in config. - - // storage/cluster.basicsearch/storage/0 -> basicsearch/storage/0 - static final ServiceNameRecognizer STORAGENODE_RECOGNIZER = new SingleGroupRegexpServiceNameRecognizer( - "^storage/cluster\\.([^/]+/storage/[^/]+)$"); - - // storage/cluster.basicsearch/distributor/0 -> basicsearch/distributor/0 - static final ServiceNameRecognizer DISTRIBUTOR_RECOGNIZER = new SingleGroupRegexpServiceNameRecognizer( - "^storage/cluster\\.([^/]+/distributor/[^/]+)$"); - - // docproc/cluster.basicsearch.indexing/0/chain.indexing -> docproc/cluster.basicsearch.indexing/0 - static final ServiceNameRecognizer DOCPROC_RECOGNIZER = new SingleGroupRegexpServiceNameRecognizer( - "^(docproc/cluster\\.[^/.]+\\.indexing/[^/]+)/.*$"); - - // basicsearch/search/cluster.basicsearch/0/realtimecontroller -> basicsearch/search/cluster.basicsearch/0 - static final ServiceNameRecognizer SEARCH_RECOGNIZER = new SingleGroupRegexpServiceNameRecognizer( - "^(basicsearch/search/cluster.basicsearch/[^/.]+)/.*$"); - - static final ServiceNameRecognizer ALL_RECOGNIZER = serviceName -> Stream.of( - STORAGENODE_RECOGNIZER, - DISTRIBUTOR_RECOGNIZER, - DOCPROC_RECOGNIZER, - SEARCH_RECOGNIZER) - .map(recognizer -> recognizer.apply(serviceName)) - .filter(optional -> optional.isPresent()) - .findFirst() - .orElse(Optional.empty()); -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor.java deleted file mode 100644 index 4154dd598a4..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokMonitor.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.service.monitor; - -import com.yahoo.jrt.Supervisor; -import com.yahoo.jrt.Transport; -import com.yahoo.jrt.slobrok.api.Mirror; -import com.yahoo.jrt.slobrok.api.SlobrokList; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -class SlobrokMonitor { - private final Supervisor supervisor = new Supervisor(new Transport()); - private final SlobrokList slobrokList = new SlobrokList(); - private final Mirror mirror = new Mirror(supervisor, slobrokList); - - void setSlobrokConnectionSpecs(List slobroks) { - slobrokList.setup(slobroks.toArray(new String[0])); - } - - Map getRegisteredServices() { - if (!mirror.ready()) { - return Collections.emptyMap(); - } - - Mirror.Entry[] mirrorEntries = mirror.lookup("**"); - return Arrays.stream(mirrorEntries).collect(Collectors.toMap( - entry -> new SlobrokServiceName(entry.getName()), - entry -> new SlobrokServiceSpec(entry.getSpec()))); - } - - boolean isRegistered(SlobrokServiceName serviceName) { - return mirror.lookup(serviceName.s()).length != 0; - } - - void shutdown() { - mirror.shutdown(); - } - - static class SlobrokServiceName { - private final String name; - - SlobrokServiceName(String name) { - this.name = name; - } - - // TODO: Fix spec - String s() { - return name; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - SlobrokServiceName that = (SlobrokServiceName) o; - - return name.equals(that.name); - } - - @Override - public int hashCode() { - return name.hashCode(); - } - } - - static class SlobrokServiceSpec { - private final String spec; - - SlobrokServiceSpec(String spec) { - this.spec = spec; - } - - // TODO: Fix name - String s() { - return spec; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - SlobrokServiceSpec that = (SlobrokServiceSpec) o; - - return spec.equals(that.spec); - } - - @Override - public int hashCode() { - return spec.hashCode(); - } - } -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokServiceNameUtil.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokServiceNameUtil.java deleted file mode 100644 index 628925bb646..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokServiceNameUtil.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor; - -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.applicationmodel.ConfigId; -import com.yahoo.vespa.applicationmodel.ServiceType; -import com.yahoo.vespa.service.monitor.SlobrokMonitor.SlobrokServiceName; -import scala.Option; - -import java.util.logging.Logger; - -/** - * @author tonytv - */ -public class SlobrokServiceNameUtil { - private static final Logger log = Logger.getLogger(SlobrokServiceNameUtil.class.getName()); - - private static final String configServerServiceTypeString = "configserver"; - public static final ServiceType configServerServiceType = new ServiceType(configServerServiceTypeString); - - private SlobrokServiceNameUtil() {} - - /** - * Returns the name a service instance is registered with in slobrok, - * or empty if the service instance is never registered in slobrok. - */ - public static Option serviceName(ServiceType serviceType, ConfigId configId) { - switch (serviceType.s()) { - case "adminserver": - case "config-sentinel": - case "configproxy": - case configServerServiceTypeString: - case "filedistributorservice": - case "logd": - case "logserver": - case "metricsproxy": - case "slobrok": - case "transactionlogserver": - case "ytracecleaner": - return Option.empty(); - - case "topleveldispatch": - return Option.apply(new SlobrokServiceName(configId.s())); - - case "qrserver": - case "container": - case "docprocservice": - case "container-clustercontroller": - return Option.apply(new SlobrokServiceName("vespa/service/" + configId.s())); - - case "searchnode": //TODO: handle only as storagenode instead of both as searchnode/storagenode - return Option.apply(new SlobrokServiceName(configId.s() + "/realtimecontroller")); - case "distributor": - case "storagenode": - return Option.apply(new SlobrokServiceName("storage/cluster." + configId.s())); - default: - log.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() + " with config id " + configId.s()); - return Option.empty(); - } - } -} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/config/ConfigObservableCreator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/config/ConfigObservableCreator.java deleted file mode 100644 index 810b284cc3b..00000000000 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/config/ConfigObservableCreator.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.config; - -import com.yahoo.config.ConfigInstance; -import com.yahoo.config.subscription.ConfigHandle; -import com.yahoo.config.subscription.ConfigSource; -import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.log.LogLevel; -import rx.Observable; - -import java.util.logging.Logger; - -/** - * @author oyving - */ -public class ConfigObservableCreator { - private static final Logger log = Logger.getLogger(ConfigObservableCreator.class.getName()); - - private ConfigObservableCreator() {} - - public static Observable create( - ConfigSource configSource, - Class configClass, - String configId) { - - return Observable.create( - subscriber -> { - try { - final ConfigSubscriber configSubscriber = new ConfigSubscriber(configSource); - - try { - final ConfigHandle configHandle = configSubscriber.subscribe(configClass, configId); - - log.log(LogLevel.DEBUG, "Subscribing to configuration " + configClass + "@" + configId + " from " + configSource); - - while (!subscriber.isUnsubscribed() && !configSubscriber.isClosed()) { - if (configSubscriber.nextGeneration(1000) && configHandle.isChanged()) { - log.log(LogLevel.DEBUG, "Received new configuration: " + configHandle); - T configuration = configHandle.getConfig(); - log.log(LogLevel.DEBUG, "Received new configuration: " + configuration); - subscriber.onNext(configuration); - } else { - log.log(LogLevel.DEBUG, "Configuration tick with no change: " + configHandle); - } - } - } finally { - configSubscriber.close(); - } - - if (!subscriber.isUnsubscribed()) { - subscriber.onCompleted(); - } - } catch (Exception e) { - if (!subscriber.isUnsubscribed()) { - subscriber.onError(e); - } - } - } - ); - } -} diff --git a/service-monitor/src/main/resources/config-models/service-monitor.xml b/service-monitor/src/main/resources/config-models/service-monitor.xml deleted file mode 100644 index 52316818edc..00000000000 --- a/service-monitor/src/main/resources/config-models/service-monitor.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/SlobrokAndConfigIntersector.scala b/service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/SlobrokAndConfigIntersector.scala deleted file mode 100644 index 5ee19dda0a6..00000000000 --- a/service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/SlobrokAndConfigIntersector.scala +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor - -import java.util.logging.Logger -import java.util.{Collections, Optional} - -import com.google.inject.Inject -import com.yahoo.cloud.config.ConfigserverConfig -import com.yahoo.component.AbstractComponent -import com.yahoo.config.subscription.ConfigSourceSet -import com.yahoo.log.LogLevel -import com.yahoo.vespa.applicationmodel._ -import com.yahoo.vespa.config.GenerationCounter -import com.yahoo.vespa.service.monitor.SlobrokAndConfigIntersector._ -import com.yahoo.vespa.service.monitor.SlobrokMonitor._ -import com.yahoo.vespa.service.monitor.config.InstancesObservables -import com.yahoo.vespa.service.monitor.config.InstancesObservables._ -import rx.lang.scala.{Observable, Subscription} - -import scala.collection.JavaConverters._ -import scala.collection.immutable.Set -import scala.concurrent.duration._ -import scala.language.postfixOps - -/** - * Quick and dirty intersection of slobrok and model config. - * @author tonytv - */ - // TODO: This class is the API of the service monitor. It should have a proper name rather than be named after an implementation detail - // TODO: For the same reason, add javadoc -class SlobrokAndConfigIntersector( - configSourceSet: ConfigSourceSet, - //Config servers that are part of an application instance, i.e. not multi-tenant, are included in lb-services config - multiTenantConfigServerHostNames: Set[HostName], - configCounter: GenerationCounter - ) extends AbstractComponent with ServiceMonitor -{ - private val instancesObservables = new InstancesObservables(configSourceSet) - - @Inject - def this(config: ConfigserverConfig, configCounter: GenerationCounter) = { - this( - new ConfigSourceSet(s"tcp/localhost:${config.rpcport()}"), - if (config.multitenant()) configServerHostNames(config) else Set(), - configCounter) - } - - @volatile - private var latestSlobrokMonitorMap: Map[ApplicationInstanceReference, SlobrokMonitor] = Map() - - private val zoneConfigServerCluster: Map[ApplicationInstanceReference, ApplicationInstance[Void]] = - if (multiTenantConfigServerHostNames.isEmpty) Map() - else Map( - new ApplicationInstanceReference(syntheticHostedVespaTenantId, configServerApplicationInstanceId ) -> - new ApplicationInstance[Void]( - syntheticHostedVespaTenantId, - configServerApplicationInstanceId, - Collections.singleton(new ServiceCluster[Void]( - new ClusterId("zone-config-servers"), - SlobrokServiceNameUtil.configServerServiceType, - configServer_ServerInstances(multiTenantConfigServerHostNames) - )))) - - @Override - def queryStatusOfAllApplicationInstances() - : java.util.Map[ApplicationInstanceReference, ApplicationInstance[ServiceMonitorStatus]] = - { - val applicationInstanceMap: Map[ApplicationInstanceReference, ApplicationInstance[ServiceMonitorStatus]] = for { - (applicationInstanceReference, applicationInstance) <- latestConfiguredServices() - } yield { - val slobrokMonitor = latestSlobrokMonitorMap.get(applicationInstanceReference) - - def monitoredStatus(serviceType: ServiceType, configId: ConfigId) = { - SlobrokServiceNameUtil.serviceName(serviceType, configId) map { name => - if (slobrokMonitor.exists(_.isRegistered(name))) ServiceMonitorStatus.UP - else ServiceMonitorStatus.DOWN - } getOrElse ServiceMonitorStatus.NOT_CHECKED - } - - val serviceClustersWithStatus = applicationInstance.serviceClusters.asScala.map { serviceCluster => - val serviceInstancesWithStatus = serviceCluster.serviceInstances().asScala.map { serviceInstance => - new ServiceInstance[ServiceMonitorStatus]( - serviceInstance.configId(), - serviceInstance.hostName(), - monitoredStatus(serviceCluster.serviceType, serviceInstance.configId)) - } - new ServiceCluster[ServiceMonitorStatus](serviceCluster.clusterId(), serviceCluster.serviceType(), serviceInstancesWithStatus.asJava) - } - val applicationInstanceWithStatus: ApplicationInstance[ServiceMonitorStatus] = new ApplicationInstance( - applicationInstanceReference.tenantId, - applicationInstanceReference.applicationInstanceId, - serviceClustersWithStatus.asJava) - - applicationInstanceReference -> applicationInstanceWithStatus - } - applicationInstanceMap.asJava - } - - instancesObservables.slobroksPerInstance.subscribe { slobrokServiceMap => - val nextSlobrokMonitorMap = slobrokServiceMap.map { case (instanceReference, slobrokServices) => - val slobrokMonitor = latestSlobrokMonitorMap.getOrElse(instanceReference, new SlobrokMonitor()) - slobrokMonitor.setSlobrokConnectionSpecs(asConnectionSpecs(slobrokServices).toList.asJava) - (instanceReference, slobrokMonitor) - } - val removedSlobrokMonitors = (latestSlobrokMonitorMap -- nextSlobrokMonitorMap.keySet).values - latestSlobrokMonitorMap = nextSlobrokMonitorMap - removedSlobrokMonitors.foreach { _.shutdown() } - } - - @volatile private var subscription: Option[Subscription] = None - - private val waitForConfig = Observable.interval(10 seconds) - .map(ignored => configCounter.get()).filter(_ > 0).take(1).subscribe { _ => - subscription = Some(instancesObservables.connect()) - } - - Observable.interval(10 seconds).subscribe { _ => - val applicationInstances: Map[ApplicationInstanceReference, ApplicationInstance[ServiceMonitorStatus]] = - queryStatusOfAllApplicationInstances().asScala.toMap - logServiceStatus(applicationInstances) - } - - object latestConfiguredServices { - private val mostRecentServicesIterator = - instancesObservables.servicesPerInstance. - map(_ ++ zoneConfigServerCluster). - toBlocking. - mostRecent(initialValue = zoneConfigServerCluster). - iterator - - def apply() = mostRecentServicesIterator.synchronized { - mostRecentServicesIterator.next() - } - } - - override def deconstruct(): Unit = { - waitForConfig.unsubscribe() - subscription.foreach(sub => sub.unsubscribe()) - } -} - -object SlobrokAndConfigIntersector { - private val log = Logger.getLogger(getClass.getName) - - val syntheticHostedVespaTenantId = new TenantId("hosted-vespa") - val configServerApplicationInstanceId = new ApplicationInstanceId("zone-config-servers") - - implicit class AsJavaOptional[T <: AnyRef](private val option: Option[T]) extends AnyVal { - def asJava: Optional[T] = option match { - case Some(v) => Optional.of(v) - case None => Optional.empty() - } - } - - def selectFirst[A, B](a: A, b: B) = a - - private def convertSlobrokServicesToConfigIds(registeredServiceNames: Set[SlobrokServiceName]): Set[ConfigId] = { - val registeredServiceNamesJavaSet: java.util.Set[String] = registeredServiceNames.map { _.s }.asJava - val configIdsJavaSet: java.util.Set[String] = ServiceNameUtil.convertSlobrokServicesToConfigIds(registeredServiceNamesJavaSet) - configIdsJavaSet.asScala.toSet.map { x: String => new ConfigId(x) } - } - - private def logServiceStatus(instanceMap: Map[ApplicationInstanceReference, ApplicationInstance[ServiceMonitorStatus]]): Unit = { - instanceMap.values.foreach(logServiceStatus) - } - - private def logServiceStatus(applicationInstance: ApplicationInstance[ServiceMonitorStatus]): Unit = { - val serviceInstances = - for { - serviceCluster <- applicationInstance.serviceClusters.asScala - serviceInstance <- serviceCluster.serviceInstances().asScala - } yield serviceInstance - - val serviceInstancesGroupedByStatus = serviceInstances.groupBy(_.serviceStatus()) - - def mkString(services: Traversable[ServiceInstance[ServiceMonitorStatus]]) = services.mkString("\t", "\n\t", "\n") - - log.log(LogLevel.DEBUG, s"For tenant ${applicationInstance.tenantId}, application instance ${applicationInstance.applicationInstanceId}\n" + - serviceInstancesGroupedByStatus.map { case (monitoredStatus, serviceInstances) => - s" $monitoredStatus\n" + mkString(serviceInstances) - }.mkString("\n")) - } - - private def asConnectionSpecs(slobroks: Traversable[SlobrokService]): Traversable[String] = - slobroks map { case SlobrokService(hostName, port) => s"tcp/$hostName:$port" } - - private def configServerHostNames(config: ConfigserverConfig): Set[HostName] = - //Each Zookeeper server in this config is started by a config server. - //Each config server starts a single zookeeper server. - config.zookeeperserver().asScala map {server => new HostName(server.hostname())} toSet - - private def configServer_ServerInstances(multiTenantConfigServerHostNames: Set[HostName]) - : java.util.Set[ServiceInstance[Void]] = - { - def serviceInstance(hostName: HostName) = new ServiceInstance[Void]( - new ConfigId("configId." + hostName.s), - hostName, - null) - - multiTenantConfigServerHostNames map serviceInstance asJava - } -} diff --git a/service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/config/InstancesObservables.scala b/service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/config/InstancesObservables.scala deleted file mode 100644 index ac9b8f5e5b0..00000000000 --- a/service-monitor/src/main/scala/com/yahoo/vespa/service/monitor/config/InstancesObservables.scala +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor.config - -import java.util.concurrent.TimeUnit -import java.util.logging.{Level, Logger} - -import com.yahoo.cloud.config.LbServicesConfig -import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Hosts -import com.yahoo.cloud.config.LbServicesConfig.Tenants.Applications.Hosts.Services.Ports -import com.yahoo.config.subscription.ConfigSourceSet -import com.yahoo.vespa.applicationmodel.{ApplicationInstance, ApplicationInstanceId, ApplicationInstanceReference, ClusterId, ConfigId, HostName, ServiceCluster, ServiceClusterKey, ServiceInstance, ServiceType, TenantId} -import com.yahoo.vespa.service.monitor.config.InstancesObservables._ -import rx.lang.scala.JavaConversions._ -import rx.lang.scala.{Observable, Subscription} -import rx.schedulers.Schedulers - -import scala.collection.JavaConverters._ -import scala.concurrent.duration._ - -/** - * Provides streams of slobroks and services per application instance. - * @author tonytv - */ -class InstancesObservables(configSourceSet: ConfigSourceSet) { - private val lbServicesConfigObservable = - toScalaObservable(ConfigObservableCreator. - create(configSourceSet, classOf[LbServicesConfig], "*")). - retryWhen( - _.flatMap { throwable => - val delay = Duration(30, TimeUnit.SECONDS) - log.log(Level.WARNING, s"Subscription to LbServicesConfig failed, sources=$configSourceSet. Retrying in $delay", throwable) - Observable.timer(delay) - }). - map(asInstanceReferenceToHostConfigMap). - subscribeOn(Schedulers.io()). - publish - - - val slobroksPerInstance: Observable[Map[ApplicationInstanceReference, Traversable[SlobrokService]]] = - lbServicesConfigObservable.map { _.mapValues(extractSlobrokServices).toMap } - - - val servicesPerInstance: Observable[Map[ApplicationInstanceReference, ApplicationInstance[Void]]] = - lbServicesConfigObservable.map( _.map { case (applicationInstanceReference, x) => - val serviceClusters: java.util.Set[ServiceCluster[Void]] = asServiceClusterSet(x) - val applicationInstance = new ApplicationInstance( - applicationInstanceReference.tenantId, - applicationInstanceReference.applicationInstanceId, - serviceClusters) - applicationInstanceReference -> applicationInstance - }.toMap) - - def connect(): Subscription = lbServicesConfigObservable.connect -} - -object InstancesObservables { - private val log = Logger.getLogger(getClass.getName) - - case class SlobrokService(hostName: String, port: Integer) - - - - private def asInstanceReferenceToHostConfigMap(config: LbServicesConfig) = { - for { - (tenantIdString, tenantConfig) <- config.tenants().asScala - (applicationIdString, applicationConfig) <- tenantConfig.applications().asScala - } yield { - val applicationInstanceReference: ApplicationInstanceReference = new ApplicationInstanceReference( - new TenantId(tenantIdString), - new ApplicationInstanceId(applicationIdString)) - - (applicationInstanceReference, applicationConfig.hosts()) - } - } - - private def extractSlobrokServices(hostsConfigs: java.util.Map[String, Hosts]): Traversable[SlobrokService] = { - def rpcPort(ports: Traversable[Ports]) = ports.collectFirst { - case port if port.tags().contains("rpc") => port.number() - } - - for { - (hostName, hostConfig) <- hostsConfigs.asScala - slobrokService <- Option(hostConfig.services("slobrok")) - } yield SlobrokService( - hostName, - rpcPort(slobrokService.ports().asScala).getOrElse(throw new RuntimeException("Found slobrok without rpc port"))) - } - - private def asServiceClusterSet(hostsConfigs: java.util.Map[String, Hosts]) - : java.util.Set[ServiceCluster[Void]] = { - - val serviceInstancesGroupedByCluster: Map[ServiceClusterKey, Iterable[ServiceInstance[Void]]] = (for { - (hostName, hostConfig) <- hostsConfigs.asScala.view - (serviceName, servicesConfig) <- hostConfig.services().asScala - } yield { - (new ServiceClusterKey(new ClusterId(servicesConfig.clustername()), new ServiceType(servicesConfig.`type`())), - new ServiceInstance(new ConfigId(servicesConfig.configId()), new HostName(hostName), null.asInstanceOf[Void])) - }).groupByKeyWithValue(_._1, _._2) - - val serviceClusterSet: Set[ServiceCluster[Void]] = serviceInstancesGroupedByCluster.map { - case (serviceClusterKey, serviceInstances) => - new ServiceCluster( - serviceClusterKey.clusterId, - serviceClusterKey.serviceType, - serviceInstances.toSet.asJava) - }.toSet - - serviceClusterSet.asJava - } - - implicit class IterableWithImprovedGrouping[A](val iterable: Iterable[A]) { - def groupByKeyWithValue[K, V](keyExtractor: (A) => K, valueExtractor: (A) => V): Map[K, Iterable[V]] = { - val groupedByKey: Map[K, Iterable[A]] = iterable.groupBy(keyExtractor) - groupedByKey.mapValues(_.map(valueExtractor)) - } - } -} diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigServerApplicationTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigServerApplicationTest.java index ec91507c846..fb123da2603 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigServerApplicationTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigServerApplicationTest.java @@ -26,12 +26,11 @@ public class ConfigServerApplicationTest { ApplicationInstance applicationInstance = application.toApplicationInstance(configServerList); - // Backward compatibility check assertEquals( - SlobrokAndConfigIntersector.configServerApplicationInstanceId(), + ConfigServerApplication.APPLICATION_INSTANCE_ID, applicationInstance.applicationInstanceId()); assertEquals( - SlobrokAndConfigIntersector.syntheticHostedVespaTenantId(), + ConfigServerApplication.TENANT_ID, applicationInstance.tenantId()); assertEquals( diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ServiceNameUtilTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ServiceNameUtilTest.java deleted file mode 100644 index d1f4978fa93..00000000000 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ServiceNameUtilTest.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.service.monitor; - -import org.junit.Test; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -public class ServiceNameUtilTest { - @Test - public void testConversionToConfigId() { - final Set slobrokServices = new HashSet<>(); - slobrokServices.add("storage/cluster.basicsearch/storage/0"); - slobrokServices.add("storage/cluster.basicsearch/distributor/0"); - slobrokServices.add("docproc/cluster.basicsearch.indexing/0/chain.indexing"); - slobrokServices.add("storage/cluster.basicsearch/distributor/0/default"); - slobrokServices.add("storage/cluster.basicsearch/storage/0/default"); - slobrokServices.add("basicsearch/search/cluster.basicsearch/0/realtimecontroller"); - - final Set configIds = ServiceNameUtil.convertSlobrokServicesToConfigIds(slobrokServices); - - assertThat(configIds, - is(setOf( - "basicsearch/search/cluster.basicsearch/0", - "basicsearch/distributor/0", - "basicsearch/storage/0", - "docproc/cluster.basicsearch.indexing/0"))); - } - - @SafeVarargs - @SuppressWarnings("varargs") - private static Set setOf(T... t) { - return new HashSet<>(Arrays.asList(t)); - } -} -- cgit v1.2.3 From 1854338baf9b18bc0324c2dd0c2c2c400fe7724d Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 20 Oct 2017 11:43:22 +0200 Subject: Remove Scala code in test --- pom.xml | 5 ---- .../src/test/scala/PrintInstanceObservables.scala | 33 ---------------------- .../src/test/scala/PrintServiceStates.scala | 24 ---------------- 3 files changed, 62 deletions(-) delete mode 100644 service-monitor/src/test/scala/PrintInstanceObservables.scala delete mode 100644 service-monitor/src/test/scala/PrintServiceStates.scala (limited to 'service-monitor') diff --git a/pom.xml b/pom.xml index ef1e90a8846..a61c8f0425e 100644 --- a/pom.xml +++ b/pom.xml @@ -799,11 +799,6 @@ scalatest_${scala.major-version} 2.2.2 - - org.json4s - json4s-native_${scala.major-version} - 3.3.0 - org.slf4j jcl-over-slf4j diff --git a/service-monitor/src/test/scala/PrintInstanceObservables.scala b/service-monitor/src/test/scala/PrintInstanceObservables.scala deleted file mode 100644 index b35fac6efe1..00000000000 --- a/service-monitor/src/test/scala/PrintInstanceObservables.scala +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.config.subscription.ConfigSourceSet -import com.yahoo.vespa.applicationmodel.{ApplicationInstance, ApplicationInstanceReference} -import com.yahoo.vespa.service.monitor.config.InstancesObservables - -import org.json4s.native.Serialization -import org.json4s.{CustomKeySerializer, NoTypeHints} - - -/** - * @author tonytv - */ -object PrintInstanceObservables { - def main(args: Array[String]): Unit = { - val sourceSet = new ConfigSourceSet("tcp/test1-node:19070") - - val observables = new InstancesObservables(sourceSet) - - observables.servicesPerInstance.subscribe(prettyPrint _) - observables.slobroksPerInstance.subscribe(println(_)) - val subscription = observables.connect() - - Thread.sleep(100000) - subscription.unsubscribe() - } - - private def prettyPrint(map: Map[ApplicationInstanceReference, ApplicationInstance[Void]]): Unit = { - implicit val formats = Serialization.formats(NoTypeHints) + - new CustomKeySerializer[Object](formats => ({case string => ???} , { case ref: AnyRef => ref.toString })) - - println(Serialization.writePretty(map)) - } -} diff --git a/service-monitor/src/test/scala/PrintServiceStates.scala b/service-monitor/src/test/scala/PrintServiceStates.scala deleted file mode 100644 index 02a9f5d89ec..00000000000 --- a/service-monitor/src/test/scala/PrintServiceStates.scala +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.config.subscription.ConfigSourceSet -import com.yahoo.vespa.config.GenerationCounter -import com.yahoo.vespa.applicationmodel.HostName -import com.yahoo.vespa.service.monitor.SlobrokAndConfigIntersector - -/** - * Quick hack to just see some results. - * - * @author bakksjo - */ -object PrintServiceStates { - def main(args: Array[String]): Unit = { - val intersector = new SlobrokAndConfigIntersector( - new ConfigSourceSet("tcp/test1-node:19070"), - multiTenantConfigServerHostNames = Set("config-server1", "config-server2").map(s => new HostName(s)), - new GenerationCounter { - override def increment = ??? - override def get = 1L - }) - Thread.sleep(100000) - intersector.deconstruct() - } -} -- cgit v1.2.3