From 72231250ed81e10d66bfe70701e64fa5fe50f712 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2016 23:09:44 +0200 Subject: Publish --- .../standalone/StandaloneContainerActivator.java | 216 +++++++++++++++++++ .../container/impl/ClassLoaderOsgiFramework.scala | 200 ++++++++++++++++++ .../container/impl/StandaloneContainerRunner.scala | 27 +++ .../standalone/CloudConfigYinstVariables.scala | 87 ++++++++ .../com/yahoo/container/standalone/Converter.scala | 26 +++ .../yahoo/container/standalone/Environment.scala | 23 ++ .../yahoo/container/standalone/LocalFileDb.scala | 71 +++++++ .../StandaloneContainerApplication.scala | 234 +++++++++++++++++++++ .../standalone/StandaloneSubscriberFactory.scala | 78 +++++++ .../src/main/sh/jdisc_container_start | 91 ++++++++ .../StandaloneContainerActivatorTest.java | 112 ++++++++++ .../standalone/CloudConfigYinstVariablesTest.scala | 60 ++++++ .../container/standalone/StandaloneContainer.scala | 64 ++++++ .../standalone/StandaloneContainerTest.scala | 110 ++++++++++ .../standalone/StandaloneSubscriberTest.scala | 41 ++++ 15 files changed, 1440 insertions(+) create mode 100644 standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java create mode 100644 standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala create mode 100644 standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala create mode 100644 standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala create mode 100644 standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala create mode 100644 standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala create mode 100644 standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala create mode 100644 standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala create mode 100644 standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala create mode 100755 standalone-container/src/main/sh/jdisc_container_start create mode 100644 standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java create mode 100644 standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala create mode 100644 standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala create mode 100644 standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala create mode 100644 standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala (limited to 'standalone-container/src') diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java new file mode 100644 index 00000000000..daa609ea686 --- /dev/null +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java @@ -0,0 +1,216 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Module; +import com.google.inject.util.Modules; +import com.yahoo.jdisc.application.ContainerActivator; +import com.yahoo.jdisc.application.ContainerBuilder; +import com.yahoo.jdisc.application.DeactivatedContainer; +import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.http.ConnectorFactory; +import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.http.JettyHttpServer; +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleActivator; +import org.osgi.framework.BundleContext; + +import java.io.FileInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.channels.FileChannel; +import java.nio.channels.ServerSocketChannel; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toMap; + +/** + * @author Einar M R Rosenvinge + * @since 5.21.0 + */ +public class StandaloneContainerActivator implements BundleActivator { + @Override + public void start(BundleContext bundleContext) throws Exception { + Container container = getContainer(); + List connectorConfigs = getConnectorConfigs(container); + + Stream keyStorePaths = getKeyStorePaths(connectorConfigs); + Map fileChannels = openFiles(keyStorePaths); + registerKeyStoreFileChannels(bundleContext, fileChannels); + + for (ConnectorConfig config: connectorConfigs) { + ServerSocketChannel socketChannel = bindChannel(config); + registerChannels(bundleContext, config.listenPort(), socketChannel); + } + } + + private void registerKeyStoreFileChannels(BundleContext bundleContext, Map fileChannels) { + Hashtable properties = new Hashtable<>(); + properties.put("role", "com.yahoo.container.standalone.StandaloneContainerActivator.KeyStoreFileChannels"); + //Since Standalone container and jdisc http service don't have a suitable common module for placing a wrapper class for fileChannels, + //we register it with the type map. In the future, we should wrap this. + bundleContext.registerService(Map.class, + Collections.unmodifiableMap(fileChannels), + properties); + } + + private Map openFiles(Stream keyStorePaths) { + return keyStorePaths.collect(toMap( + Function.identity(), + StandaloneContainerActivator::getFileChannel)); + } + + + + private static FileChannel getFileChannel(Path path) { + try { + FileInputStream inputStream = new FileInputStream(path.toFile()); + //don't close the inputStream, as that will close the underlying channel. + return inputStream.getChannel(); + } catch (IOException e) { + throw new RuntimeException("Failed opening path " + path, e); + } + } + + private Stream getKeyStorePaths(List connectorConfigs) { + return connectorConfigs.stream(). + map(ConnectorConfig::ssl). + flatMap(StandaloneContainerActivator::getKeyStorePaths); + } + + private static Stream getKeyStorePaths(ConnectorConfig.Ssl ssl) { + Stream paths = Stream.of( + ssl.keyStorePath(), + ssl.pemKeyStore().certificatePath(), + ssl.pemKeyStore().keyPath()); + + return paths. + filter(path -> !path.isEmpty()). + map(Paths::get); + } + + void registerChannels(BundleContext bundleContext, int listenPort, ServerSocketChannel boundChannel) { + Hashtable properties = new Hashtable<>(); + properties.put("port", listenPort); + bundleContext.registerService(ServerSocketChannel.class, boundChannel, properties); + } + + ServerSocketChannel bindChannel(ConnectorConfig channelInfo) throws IOException { + ServerSocketChannel serverChannel = ServerSocketChannel.open(); + InetSocketAddress bindAddress = new InetSocketAddress(channelInfo.listenPort()); + serverChannel.socket().bind(bindAddress, channelInfo.acceptQueueSize()); + return serverChannel; + } + + @Override + public void stop(BundleContext bundleContext) throws Exception { } + + Container getContainer(Module... modules) { + Module activatorModule = new ActivatorModule(); + List allModules = new ArrayList<>(); + allModules.addAll(Arrays.asList(modules)); + allModules.add(activatorModule); + + StandaloneContainerApplication app = new StandaloneContainerApplication(Guice.createInjector(Modules.combine(allModules))); + return app.container(); + } + + List getConnectorConfigs(Container container) { + Http http = container.getHttp(); + + return (http == null) ? + getConnectorConfigs(container.getDefaultHttpServer()) : + getConnectorConfigs(http.getHttpServer()); + } + + private static List getConnectorConfigs(JettyHttpServer jettyHttpServer) { + if (jettyHttpServer == null) + return Collections.emptyList(); + + return jettyHttpServer.getConnectorFactories().stream(). + map(StandaloneContainerActivator::getConnectorConfig). + collect(Collectors.toList()); + } + + private static ConnectorConfig getConnectorConfig(ConnectorFactory connectorFactory) { + return VespaModel.getConfig(ConnectorConfig.class, connectorFactory); + } + + + private static class ActivatorModule implements Module { + @Override + public void configure(Binder binder) { + binder.bind(OsgiFramework.class).toInstance(new DummyOsgiFramework()); + binder.bind(ContainerActivator.class).toInstance(new DummyActivatorForStandaloneContainerApp()); + } + } + + private static class DummyActivatorForStandaloneContainerApp implements ContainerActivator { + @Override + public ContainerBuilder newContainerBuilder() { + return new ContainerBuilder(new ArrayList()); + } + + @Override + public DeactivatedContainer activateContainer(ContainerBuilder builder) { + return new DeactivatedContainer() { + @Override + public Object appContext() { + return new Object(); + } + + @Override + public void notifyTermination(Runnable task) { + } + }; + } + } + public static class DummyOsgiFramework implements OsgiFramework { + @Override + public List installBundle(String bundleLocation) { + throw new UnsupportedOperationException(); + } + + @Override + public void startBundles(List bundles, boolean privileged) { + throw new UnsupportedOperationException(); + } + + @Override + public void refreshPackages() { + throw new UnsupportedOperationException(); + } + + @Override + public BundleContext bundleContext() { + return null; + } + + @Override + public List bundles() { + return Collections.emptyList(); + } + + @Override + public void start() { + } + + @Override + public void stop() { + } + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala new file mode 100644 index 00000000000..6d45e6fa8a1 --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.impl + +import com.yahoo.jdisc.application.{OsgiHeader, OsgiFramework} +import java.util +import org.osgi.framework._ +import java.io.InputStream +import util.concurrent.atomic.AtomicInteger +import util.jar.JarFile +import util.{Dictionary, Collections, Hashtable} + +import scala.collection.JavaConversions._ +import com.yahoo.container.standalone.StandaloneContainerApplication +import collection.mutable.ArrayBuffer +import java.net.{URL, URLClassLoader} +import org.osgi.framework.wiring._ +import org.osgi.resource.{Wire, Capability, Requirement} + +/** + * @author tonytv + */ +final class ClassLoaderOsgiFramework extends OsgiFramework { + private val bundleLocations = new ArrayBuffer[URL] + private val bundleList = ArrayBuffer[Bundle](SystemBundleImpl) + private var classLoader: ClassLoader = null + + private val nextBundleId = new AtomicInteger(1) + + override def installBundle(bundleLocation: String) = { + if (bundleLocation != "") { + val url = new URL(bundleLocation) + bundleLocations += url + bundleList += new JarBundleImpl(url) + } + + bundles() + } + + def getClassLoader = { + if (bundleLocations.isEmpty) { + getClass.getClassLoader + } else { + if(classLoader == null) + classLoader = new URLClassLoader(bundleLocations.toArray, getClass.getClassLoader) + + classLoader + } + } + + override def startBundles(bundles: util.List[Bundle], privileged: Boolean) {} + + override def refreshPackages() {} + + override def bundleContext():BundleContext = BundleContextImpl + + override def bundles() = bundleList + + override def start() {} + + override def stop() {} + + private abstract class BundleImpl extends Bundle { + override def getState = Bundle.ACTIVE + + override def start(options: Int) {} + override def start() {} + override def stop(options: Int) {} + override def stop() {} + override def update(input: InputStream) {} + override def update() {} + override def uninstall() {} + + override def getHeaders(locale: String) = getHeaders + + override def getSymbolicName = ClassLoaderOsgiFramework.this.getClass.getName + override def getLocation = getSymbolicName + + override def getRegisteredServices = Array[ServiceReference[_]]() + override def getServicesInUse = getRegisteredServices + + override def hasPermission(permission: Any) = true + + override def getResource(name: String) = getClassLoader.getResource(name) + override def loadClass(name: String) = getClassLoader.loadClass(name) + override def getResources(name: String) = getClassLoader.getResources(name) + + override def getEntryPaths(path: String) = throw new UnsupportedOperationException + override def getEntry(path: String) = throw new UnsupportedOperationException + override def findEntries(path: String, filePattern: String, recurse: Boolean) = throw new UnsupportedOperationException + + override def getLastModified = 1L + + override def getBundleContext = throw new UnsupportedOperationException + override def getSignerCertificates(signersType: Int) = Collections.emptyMap() + + override def adapt[A](`type`: Class[A]): A = { + if (`type` == classOf[BundleRevision]) BundleRevisionImpl.asInstanceOf[A] + else if (`type` == classOf[BundleWiring]) BundleWiringImpl.asInstanceOf[A] + else null.asInstanceOf[A] + } + + override def getDataFile(filename: String) = null + override def compareTo(o: Bundle) = getBundleId compareTo o.getBundleId + } + + private object BundleRevisionImpl extends BundleRevision { + override def getSymbolicName: String = this.getClass.getName + override def getDeclaredRequirements(p1: String): util.List[BundleRequirement] = throw new UnsupportedOperationException + override def getVersion: Version = Version.emptyVersion + override def getWiring: BundleWiring = BundleWiringImpl + override def getDeclaredCapabilities(p1: String): util.List[BundleCapability] = throw new UnsupportedOperationException + override def getTypes: Int = 0 + override def getBundle: Bundle = throw new UnsupportedOperationException + override def getCapabilities(p1: String): util.List[Capability] = throw new UnsupportedOperationException + override def getRequirements(p1: String): util.List[Requirement] = throw new UnsupportedOperationException + } + + private object BundleWiringImpl extends BundleWiring { + override def findEntries(p1: String, p2: String, p3: Int): util.List[URL] = ??? + override def getRequiredResourceWires(p1: String): util.List[Wire] = ??? + override def getResourceCapabilities(p1: String): util.List[Capability] = ??? + override def isCurrent: Boolean = ??? + override def getRequiredWires(p1: String): util.List[BundleWire] = ??? + override def getCapabilities(p1: String): util.List[BundleCapability] = ??? + override def getProvidedResourceWires(p1: String): util.List[Wire] = ??? + override def getProvidedWires(p1: String): util.List[BundleWire] = ??? + override def getRevision: BundleRevision = ??? + override def getResourceRequirements(p1: String): util.List[Requirement] = ??? + override def isInUse: Boolean = ??? + override def listResources(p1: String, p2: String, p3: Int): util.Collection[String] = ??? + override def getClassLoader: ClassLoader = ClassLoaderOsgiFramework.this.getClassLoader + override def getRequirements(p1: String): util.List[BundleRequirement] = ??? + override def getResource: BundleRevision = ??? + override def getBundle: Bundle = ??? + } + + private object SystemBundleImpl extends BundleImpl { + override val getBundleId = 0L + override def getVersion = Version.emptyVersion + override def getHeaders: Dictionary[String, String] = new Hashtable[String, String](Map(OsgiHeader.APPLICATION -> classOf[StandaloneContainerApplication].getName)) + } + + + private class JarBundleImpl(location: URL) extends BundleImpl { + override val getBundleId = nextBundleId.getAndIncrement.asInstanceOf[Long] + + private val headers = retrieveHeaders(location) + + override def getHeaders: Dictionary[String, String] = headers + override val getSymbolicName = headers.get("Bundle-SymbolicName") + override val getVersion = Version.parseVersion(headers.get("Bundle-Version")) + + + private def retrieveHeaders(location: URL) = { + val jarFile = new JarFile(location.getFile) + try { + val attributes = jarFile.getManifest.getMainAttributes + new Hashtable[String, String](attributes.entrySet().map( entry => entry.getKey.toString -> entry.getValue.toString).toMap) + } finally { + jarFile.close() + } + } + } + + private object BundleContextImpl extends BundleContext { + private val bundleImpl = SystemBundleImpl + + override def getProperty(key: String) = null + override def getBundle = bundleImpl + override def installBundle(location: String, input: InputStream) = throw new UnsupportedOperationException + override def installBundle(location: String) = throw new UnsupportedOperationException + + override def getBundle(id: Long) = bundleImpl + override def getBundles = Array(bundleImpl) + override def getBundle(location: String) = bundleImpl + + override def addServiceListener(listener: ServiceListener, filter: String) {} + override def addServiceListener(listener: ServiceListener) {} + override def removeServiceListener(listener: ServiceListener) {} + override def addBundleListener(listener: BundleListener) {} + override def removeBundleListener(listener: BundleListener) {} + + override def addFrameworkListener(listener: FrameworkListener) {} + override def removeFrameworkListener(listener: FrameworkListener) {} + + override def registerService(clazzes: Array[String], service: Any, properties: Dictionary[String, _]) = throw new UnsupportedOperationException + override def registerService(clazz: String, service: Any, properties: Dictionary[String, _]) = null + override def registerService[S](clazz: Class[S], service: S, properties: Dictionary[String, _]) = throw new UnsupportedOperationException + override def getServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException + override def getAllServiceReferences(clazz: String, filter: String) = throw new UnsupportedOperationException + override def getServiceReference(clazz: String) = throw new UnsupportedOperationException + override def getServiceReference[S](clazz: Class[S]) = throw new UnsupportedOperationException + override def getServiceReferences[S](clazz: Class[S], filter: String) = Collections.emptyList() + override def getService[S](reference: ServiceReference[S]) = throw new UnsupportedOperationException + override def ungetService(reference: ServiceReference[_]) = throw new UnsupportedOperationException + override def getDataFile(filename: String) = throw new UnsupportedOperationException + override def createFilter(filter: String) = throw new UnsupportedOperationException + + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala new file mode 100644 index 00000000000..57f60febb26 --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/application/container/impl/StandaloneContainerRunner.scala @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.application.container.impl + +import java.nio.file.Files +import com.yahoo.text.Utf8 + +/** + * @author tonytv + */ +final class StandaloneContainerRunner { + + + +} + +object StandaloneContainerRunner { + def createApplicationPackage(servicesXml: String) = { + val applicationDir = Files.createTempDirectory("application") + + val servicesXmlFile = applicationDir.resolve("services.xml"); + var content = servicesXml; + if ( ! servicesXml.startsWith("""" + '\n' + servicesXml + Files.write(servicesXmlFile, Utf8.toBytes(content)) + applicationDir + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala new file mode 100644 index 00000000000..188bedfbc20 --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/CloudConfigYinstVariables.scala @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import java.util.Optional + +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions.ConfigServer + +import scala.language.implicitConversions +import scala.util.Try + +/** + * @author tonytv + */ +class CloudConfigYinstVariables extends CloudConfigOptions { + import CloudConfigYinstVariables._ + + override val rpcPort = optionalYinstVar[Integer]("port_configserver_rpc", "services") + override val allConfigServers = yinstVar("addr_configserver", "services") withDefault Array[ConfigServer]() + override val multiTenant = optionalYinstVar[java.lang.Boolean]("multitenant") + + override val zookeeperBarrierTimeout = optionalYinstVar[java.lang.Long]("zookeeper_barrier_timeout") + override val sessionLifeTimeSecs = optionalYinstVar[java.lang.Long]("session_lifetime") + override val configModelPluginDirs = yinstVar("config_model_plugin_dirs") withDefault Array[String]() + override val zookeeperClientPort = optionalYinstVar[Integer]("zookeeper_clientPort") + override val zookeeperQuorumPort = optionalYinstVar[Integer]("zookeeper_quoromPort") + override val zookeeperElectionPort = optionalYinstVar[Integer]("zookeeper_electionPort") + override val payloadCompressionType = optionalYinstVar[java.lang.String]("payload_compression_type") + override val environment = optionalYinstVar[java.lang.String]("environment") + override val region = optionalYinstVar[java.lang.String]("region") + override val defaultFlavor = optionalYinstVar[java.lang.String]("default_flavor") + override val defaultAdminFlavor = optionalYinstVar[java.lang.String]("default_admin_flavor") + override val defaultContainerFlavor = optionalYinstVar[java.lang.String]("default_container_flavor") + override val defaultContentFlavor = optionalYinstVar[java.lang.String]("default_content_flavor") + override val useVespaVersionInRequest = optionalYinstVar[java.lang.Boolean]("use_vespa_version_in_request") + override val hostedVespa = optionalYinstVar[java.lang.Boolean]("hosted_vespa") + override val numParallelTenantLoaders = optionalYinstVar[java.lang.Integer]("num_parallel_tenant_loaders") +} + +object CloudConfigYinstVariables { + private class YinstVariable(yinstPkg:String, name: String) { + val value = Environment.optionalYinstVariable(yinstPkg + "." + name) + + def withDefault[T](defaultValue: T)(implicit c: Converter[T]) : T = { + value map { implicitly[Converter[T]].convert } getOrElse defaultValue + } + } + + private def yinstVar(setting:String, yinstPkg: String = "cloudconfig_server") = new YinstVariable(yinstPkg, setting) + + private def optionalYinstVar[T](setting:String, yinstPkg: String = "cloudconfig_server")(implicit c: Converter[T]): Optional[T] = { + Environment.optionalYinstVariable(yinstPkg + "." + setting) map ( c.convert ) + } + + implicit val configServerConverter: Converter[Array[ConfigServer]] = new Converter[Array[ConfigServer]] { + override def convert(s: String) = { + s split "[, ]" filter { !_.isEmpty } map { toConfigServer } + } + } + + implicit val stringArrayConverter: Converter[Array[String]] = new Converter[Array[String]] { + override def convert(s: String) = { + s split "[, ]" filter { !_.isEmpty } + } + } + + private def toConfigServer(hostPort: String): ConfigServer = Try { + val (host, portStr) = splitFirst(hostPort, ':') + val port = portStr map { _.toInt } + new ConfigServer(host, port) + }.getOrElse(throw new IllegalArgumentException(s"Invalid config server '$hostPort'")) + + private def splitFirst(string: String, separator: Character): (String, Option[String]) = { + val (beginning, endWithSeparator) = string span { _ != separator } + (beginning, tailOption(endWithSeparator)) + } + + def tailOption(s: String) = { + if (s.isEmpty) None + else Some(s.tail) + } + + implicit def toJavaOptional[U <% V, V](option: Option[U]): Optional[V] = option match { + case Some(u) => Optional.of(u: V) + case None => Optional.empty() + } +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala new file mode 100644 index 00000000000..443806d014b --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/Converter.scala @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +/** + * @author tonytv + */ +trait Converter[T] { + def convert(s: String): T +} + +object Converter { + def toConverter[T](f: String => T) = new Converter[T] { + override def convert(s: String) = f(s) + } + + implicit val intConverter = toConverter(_.toInt) + implicit val longConverter = toConverter(_.toLong) + implicit val boolConverter = toConverter(_.toBoolean) + implicit val stringConverter = toConverter(identity) + + implicit val javaIntegerConverter:Converter[Integer] = toConverter(_.toInt) + implicit val javaLongConverter:Converter[java.lang.Long] = toConverter(_.toLong) + implicit val javaBooleanConverter:Converter[java.lang.Boolean] = toConverter(_.toBoolean) + + +} \ No newline at end of file diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala new file mode 100644 index 00000000000..98671e2addf --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/Environment.scala @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +/** + * @author tonytv + * TODO: copied from standalone-container. Move to separate lib module instead. + */ +object Environment { + def optionalYinstVariable(name: String) = { + env(name.replace(".", "__")). + orElse(systemProperty(name)) //for unit testing + } + + def yinstVariable(name: String) = { + optionalYinstVariable(name). + getOrElse { + throw new IllegalStateException("Environment variable not set: " + name) + } + } + + def env(name: String) = Option(System.getenv(name)) + def systemProperty(name: String) = Option(System.getProperty(name)) +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala new file mode 100644 index 00000000000..7af08a4685d --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import java.io.File +import java.lang.reflect.Constructor +import java.util +import java.util.concurrent.TimeUnit +import com.yahoo.config.FileReference +import com.yahoo.config.application.api.FileRegistry +import com.yahoo.config.application.api.FileRegistry.Entry +import com.yahoo.filedistribution.fileacquirer.FileAcquirer +import com.yahoo.net.HostName +import scala.collection.JavaConversions._ + + +import LocalFileDb._ +import scala.collection.mutable +import java.nio.file.Path + + +/** + * FileAcquirer and FileRegistry working on a local directory. + * @author tonytv + */ +class LocalFileDb(appPath: Path) extends FileAcquirer with FileRegistry { + private val fileReferenceToFile = mutable.Map[FileReference, File]() + + /** *** FileAcquirer overrides *****/ + def waitFor(reference: FileReference, l: Long, timeUnit: TimeUnit): File = { + synchronized { + fileReferenceToFile.get(reference).getOrElse { + throw new RuntimeException("Invalid file reference " + reference) + } + } + } + + override def shutdown() {} + + /** *** FileRegistry overrides *****/ + def addFile(relativePath: String): FileReference = { + val file = appPath.resolve(relativePath).toFile + if (!file.exists) { + throw new RuntimeException("The file does not exist: " + file.getPath) + } + + val fileReference: FileReference = fileReferenceConstructor.newInstance("LocalFileDb:" + relativePath) + fileReferenceToFile.put(fileReference, file) + fileReference + } + + def fileSourceHost: String = + HostName.getLocalhost + + def allRelativePaths: java.util.Set[String] = { + new java.util.HashSet(fileReferenceToFile.values.map(_.getPath)) + } + + override def export(): util.List[Entry] = { + new java.util.ArrayList(fileReferenceToFile.keys.map{ (ref: FileReference) => new Entry(fileReferenceToFile.get(ref).get.getPath, ref)}) + } +} + +object LocalFileDb { + private def createFileReferenceConstructor: Constructor[FileReference] = { + val method: Constructor[FileReference] = classOf[FileReference].getDeclaredConstructor(classOf[String]) + method.setAccessible(true) + method + } + + private val fileReferenceConstructor: Constructor[FileReference] = createFileReferenceConstructor +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala new file mode 100644 index 00000000000..324de2771f4 --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -0,0 +1,234 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import com.google.inject.{Key, AbstractModule, Injector, Inject} +import com.yahoo.config.application.api.{DeployLogger, RuleConfigDeriver, FileRegistry, ApplicationPackage} +import com.yahoo.config.provision.Zone +import com.yahoo.jdisc.application.Application +import com.yahoo.container.jdisc.ConfiguredApplication +import java.io.{IOException, File} +import com.yahoo.config.model.test.MockRoot +import com.yahoo.config.model.application.provider._ +import com.yahoo.vespa.defaults.Defaults +import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ManhattanContainerModelBuilder, ContainerModelBuilder} +import org.w3c.dom.Element +import com.yahoo.config.model.builder.xml.XmlHelper +import com.yahoo.vespa.model.container.Container +import com.yahoo.collections.CollectionUtil.first +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder +import com.yahoo.io.IOUtils +import com.yahoo.container.di.config.SubscriberFactory +import StandaloneContainerApplication._ +import com.google.inject.name.Names +import scala.util.Try +import java.nio.file.{FileSystems, Path, Paths, Files} +import com.yahoo.config.model.{ConfigModelRepo, ApplicationConfigProducerRoot} +import scala.collection.JavaConversions._ +import com.yahoo.text.XML +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking + +import java.lang.{ Boolean => JBoolean } +import Environment._ +import com.yahoo.config.model.deploy.DeployState + +/** + * @author tonytv + * @author gjoranv + */ +class StandaloneContainerApplication @Inject()(injector: Injector) extends Application { + + ConfiguredApplication.ensureVespaLoggingInitialized() + + val applicationPath: Path = injectedApplicationPath.getOrElse(yinstApplicationPath) + + val distributedFiles = new LocalFileDb(applicationPath) + + val configModelRepo = Try { injector.getInstance(Key.get(classOf[ConfigModelRepo], configModelRepoName))}.getOrElse(new ConfigModelRepo) + + val networkingOption = Try { + injector.getInstance(Key.get(classOf[JBoolean], Names.named(disableNetworkingAnnotation))) + }.map { + case JBoolean.TRUE => Networking.disable + case JBoolean.FALSE => Networking.enable + }.getOrElse(Networking.enable) + + val (modelRoot, container) = withTempDir( + preprocessedApplicationDir => createContainerModel(applicationPath, distributedFiles, preprocessedApplicationDir, networkingOption, configModelRepo)) + + val configuredApplication = createConfiguredApplication(container) + + def createConfiguredApplication(container: Container): Application = { + val augmentedInjector = injector.createChildInjector(new AbstractModule { + def configure() { + bind(classOf[SubscriberFactory]).toInstance(new StandaloneSubscriberFactory(modelRoot)) + } + }) + + System.setProperty("config.id", container.getConfigId) //TODO: DRY + augmentedInjector.getInstance(classOf[ConfiguredApplication]) + } + + def injectedApplicationPath = Try { + injector.getInstance(Key.get(classOf[Path], applicationPathName)) + }.toOption + + def yinstApplicationPath = path(yinstVariable(applicationLocationYinstVariable)) + + override def start() { + try { + com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles) + configuredApplication.start() + } + catch { + case e: Exception => { com.yahoo.container.Container.resetInstance(); throw e; } + } + } + + override def stop() { + configuredApplication.stop() + } + + override def destroy() { + com.yahoo.container.Container.resetInstance() + configuredApplication.destroy() + } +} + +object StandaloneContainerApplication { + val packageName = "standalone_jdisc_container" + val applicationLocationYinstVariable = s"$packageName.app_location" + val deploymentProfileYinstVariable = s"$packageName.deployment_profile" + val manhattanHttpPortYinstVariable = s"$packageName.manhattan_http_port" + + val applicationPathName = Names.named(applicationLocationYinstVariable) + + val disableNetworkingAnnotation = "JDisc.disableNetworking" + val configModelRepoName = Names.named("ConfigModelRepo") + val configDefinitionRepo = new StaticConfigDefinitionRepo() + + val defaultTmpBaseDir = Defaults.getDefaults().underVespaHome("tmp") + val tmpDirName = "standalone_container" + + private def withTempDir[T](f: File => T): T = { + val tmpDir = createTempDir() + try { + f(tmpDir) + } finally { + IOUtils.recursiveDeleteDir(tmpDir) + } + } + + private def createTempDir(): File = { + def getBaseDir: Path = { + val tmpBaseDir = + if (new File(defaultTmpBaseDir).exists()) + defaultTmpBaseDir + else + System.getProperty("java.io.tmpdir") + + Paths.get(tmpBaseDir) + } + + val basePath: Path = getBaseDir + val tmpDir = Files.createTempDirectory(basePath, tmpDirName) + tmpDir.toFile + } + + private def validateApplication(applicationPackage: ApplicationPackage, logger: DeployLogger) = { + try { + applicationPackage.validateXML(logger) + } catch { + case e: IOException => throw new IllegalArgumentException(e) + } + } + + def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = { + optionalYinstVariable(deploymentProfileYinstVariable) match { + case None => new ContainerModelBuilder(true, networkingOption) + case Some("manhattan") => new ManhattanContainerModelBuilder(manhattanHttpPort) + case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigYinstVariables) + case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'") + } + } + + def manhattanHttpPort: Int = { + val port = yinstVariable(manhattanHttpPortYinstVariable) + Try { + Integer.parseInt(port) + } filter( _ > 0) getOrElse { + throw new RuntimeException(s"$manhattanHttpPortYinstVariable is not a valid port: '$port'") + } + } + + def createContainerModel(applicationPath: Path, + fileRegistry: FileRegistry, + preprocessedApplicationDir: File, + networkingOption: Networking, + configModelRepo: ConfigModelRepo = new ConfigModelRepo): (MockRoot, Container) = { + val logger = new BaseDeployLogger + val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).preprocessedDir(preprocessedApplicationDir).build() + // TODO: Needed until we get rid of semantic rules + val applicationPackage = rawApplicationPackage.preprocess(Zone.defaultZone(), new RuleConfigDeriver { + override def derive(ruleBaseDir: String, outputDir: String): Unit = {} + }, logger) + validateApplication(applicationPackage, logger) + val deployState = new DeployState.Builder(). + applicationPackage(applicationPackage). + fileRegistry(fileRegistry). + deployLogger(logger). + configDefinitionRepo(configDefinitionRepo). + build() + + val root = new MockRoot("", deployState) + val vespaRoot = new ApplicationConfigProducerRoot(root, + "vespa", + deployState.getDocumentModel, + deployState.getProperties.vespaVersion(), + deployState.getProperties.applicationId()) + + val spec = containerRootElement(applicationPackage) + val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec) + containerModel.getCluster().prepare() + containerModel.initialize(configModelRepo) + val container = first(containerModel.getCluster().getContainers) + + // Always disable rpc server for standalone container. This server will soon be removed anyway. + container.setRpcServerEnabled(false) + container.setHttpServerEnabled(networkingOption == Networking.enable) + + initializeContainer(container, spec) + root.freezeModelTopology() + (root, container) + } + + def initializeContainer(container: Container, spec: Element) { + val host = container.getRoot.getHostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC) + + container.setBasePort(VespaDomBuilder.getXmlWantedPort(spec)) + container.setHostResource(host) + container.initService() + } + + def getJDiscInServices(element: Element): Element = { + def nameAndId(elements: List[Element]): List[String] = { + elements map { e => s"${e.getNodeName} id='${e.getAttribute("id")}'" } + } + + val jDiscElements = ContainerModelBuilder.configModelIds flatMap { name => XML.getChildren(element, name.getName) } + jDiscElements.toList match { + case List(e) => e + case Nil => throw new RuntimeException("No jdisc element found under services.") + case multipleElements: List[Element] => throw new RuntimeException("Found multiple JDisc elements: " + nameAndId(multipleElements).mkString(", ")) + } + } + + def containerRootElement(applicationPackage: ApplicationPackage) : Element = { + val element = XmlHelper.getDocument(applicationPackage.getServices).getDocumentElement + val nodeName = element.getNodeName + + if (ContainerModelBuilder.configModelIds.map(_.getName).contains(nodeName)) element + else getJDiscInServices(element) + } + + def path(s: String) = FileSystems.getDefault.getPath(s) +} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala new file mode 100644 index 00000000000..432e5b82946 --- /dev/null +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import com.yahoo.config.model.test.MockRoot +import com.yahoo.config.{ConfigBuilder, ConfigInstance} +import com.yahoo.container.di.ConfigKeyT +import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ +import com.yahoo.vespa.config.ConfigKey +import com.yahoo.container.di.config.{SubscriberFactory, Subscriber} +import StandaloneSubscriberFactory._ + +/** + * @author tonytv + * @author gjoranv + */ +class StandaloneSubscriberFactory(root: MockRoot) extends SubscriberFactory { + class StandaloneSubscriber(configKeys: Set[ConfigKeyT]) extends Subscriber { + override def configChanged = + generation == 0 + + override def close() {} + + override def config = { + + def getConfig(key: ConfigKeyT) = { + val builderWithModelConfig = root.getConfig(newBuilderInstance(key), key.getConfigId) + + require(builderWithModelConfig != null, "Invalid config id " + key.getConfigId ) + (key.asInstanceOf[ConfigKey[ConfigInstance]], newConfigInstance(builderWithModelConfig)) + } + + (configKeys map getConfig).toMap.asJava + } + + override def waitNextGeneration() = { + generation += 1 + + if (generation != 0) { + while (!Thread.interrupted()) + Thread.sleep(10000) + } + + generation + } + + //if waitNextGeneration has not yet been called, -1 should be returned + var generation = -1L + } + + override def getSubscriber(configKeys: java.util.Set[_ <: ConfigKey[_]]) = + new StandaloneSubscriber(configKeys.toSet.asInstanceOf[Set[ConfigKeyT]]) + + def reloadActiveSubscribers(generation: Long) { + throw new RuntimeException("unsupported") + } +} + +object StandaloneSubscriberFactory { + + private def newBuilderInstance(key: ConfigKeyT) = + builderClass(key).newInstance() + + private def builderClass(key: ConfigKeyT) = { + val nestedClasses = key.getConfigClass.getClasses + nestedClasses. + filter {_.getName.equals(key.getConfigClass.getName + "$Builder")}. + head. + asInstanceOf[Class[ConfigInstance.Builder]] + } + + private def newConfigInstance(builder: ConfigBuilder) = + configClass(builder).getConstructor(builder.getClass).newInstance(builder) + + private def configClass(builder: ConfigBuilder) = + builder.getClass.getEnclosingClass.asInstanceOf[Class[ConfigInstance]] + +} diff --git a/standalone-container/src/main/sh/jdisc_container_start b/standalone-container/src/main/sh/jdisc_container_start new file mode 100755 index 00000000000..aa50496243d --- /dev/null +++ b/standalone-container/src/main/sh/jdisc_container_start @@ -0,0 +1,91 @@ +#!/bin/bash +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + # ensure it ends with "/" : + VESPA_HOME=${VESPA_HOME%/}/ + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +source $VESPA_HOME/libexec/vespa/container_start.sh + +VESPA_CONFIG_ID="container/standalone" +export APP_LOCATION="standalone-container-jar-with-dependencies.jar" +export WAIT_FOR_JVM_INIT=0 + +if [ "$1" ]; then + export standalone_jdisc_container__app_location=$1 +fi +if [ -z "${standalone_jdisc_container__app_location}" ]; then + echo "Usage: `basename $0` " 1>&2 + exit -1 +fi +shift + +export LOGGER_ENABLED="false" +export LOGGER_LEVEL="ALL" +export LOGGER_TAG="${VESPA_CONFIG_ID}" + +if [ -z "$VESPA_LOG_CONTROL_FILE" ]; then + VESPA_LOG_CONTROL_FILE="$VESPA_HOME/var/db/jdisc/logcontrol/jdisc-container.logcontrol" +fi +export VESPA_LOG_CONTROL_FILE + +export CACHE_PATH="$VESPA_HOME/var/vespa/bundlecache/standalone" + +export PRIVILEGED=1 + +start_container "$@" diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java new file mode 100644 index 00000000000..484d4c4d50e --- /dev/null +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.container.Container; +import org.junit.Test; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static java.util.Arrays.asList; +import static java.util.stream.Collectors.toList; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.junit.Assert.assertThat; + +/** + * @author Einar M R Rosenvinge + * @since 5.22.0 + */ +public class StandaloneContainerActivatorTest { + + private String getJdiscXml(String contents) throws ParserConfigurationException, IOException, SAXException { + return "\n" + + "\n" + + " \n" + + contents + + " \n" + + ""; + } + + private void writeApplicationPackage(String servicesXml, Path tmpDir) throws IOException { + FileWriter fw = new FileWriter(tmpDir.resolve("services.xml").toFile(), false); + fw.write(servicesXml); + fw.close(); + } + + @Test + public void requireThatPortsCanBeFoundBasic() throws IOException, ParserConfigurationException, SAXException { + final Path applicationDir = Files.createTempDirectory("application"); + try { + writeApplicationPackage(getJdiscXml(""), applicationDir); + StandaloneContainerActivator activator = new StandaloneContainerActivator(); + Container container = activator.getContainer(newAppDirBinding(applicationDir)); + List ports = getPorts(activator, container); + assertThat(ports, is(asList(Defaults.getDefaults().vespaWebServicePort()))); + } finally { + IOUtils.recursiveDeleteDir(applicationDir.toFile()); + } + } + + private List getPorts(StandaloneContainerActivator activator, Container container) { + return activator.getConnectorConfigs(container).stream(). + map(ConnectorConfig::listenPort). + collect(toList()); + } + + @Test + public void requireThatPortsCanBeFoundNoHttp() throws IOException, ParserConfigurationException, SAXException { + final Path applicationDir = Files.createTempDirectory("application"); + try { + writeApplicationPackage(getJdiscXml(""), applicationDir); + StandaloneContainerActivator activator = new StandaloneContainerActivator(); + Container container = activator.getContainer(newAppDirBinding(applicationDir)); + List ports = getPorts(activator, container); + assertThat(ports, empty()); + } finally { + IOUtils.recursiveDeleteDir(applicationDir.toFile()); + } + } + + @Test + public void requireThatPortsCanBeFoundHttpThreeServers() throws IOException, ParserConfigurationException, SAXException { + final Path applicationDir = Files.createTempDirectory("application"); + try { + final String contents = + "\n" + + " \n" + + " \n" + + " \n" + + "\n"; + writeApplicationPackage(getJdiscXml(contents), applicationDir); + StandaloneContainerActivator activator = new StandaloneContainerActivator(); + Container container = activator.getContainer(newAppDirBinding(applicationDir)); + List ports = getPorts(activator, container); + assertThat(ports, is(asList(123, 456, 789))); + } finally { + IOUtils.recursiveDeleteDir(applicationDir.toFile()); + } + } + + private Module newAppDirBinding(final Path applicationDir) { + return new Module() { + @Override + public void configure(Binder binder) { + binder.bind(Path.class) + .annotatedWith(StandaloneContainerApplication.applicationPathName()) + .toInstance(applicationDir); + } + }; + } + +} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala new file mode 100644 index 00000000000..0a56f9ac19a --- /dev/null +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/CloudConfigYinstVariablesTest.scala @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import org.junit.Test +import org.junit.Assert.assertThat +import org.hamcrest.CoreMatchers.is +import org.hamcrest.Matchers.{arrayContaining} + +/** + * @author lulf + * @author tonytv + * @since 5. + */ +class CloudConfigYinstVariablesTest { + def convert = CloudConfigYinstVariables.configServerConverter.convert _ + + @Test + def test_configserver_parsing { + val parsed = convert("test2-lulf.trondheim.corp.yahoo.com") + assertThat(parsed.length, is(1)) + } + + @Test + def port_can_be_configured { + val parsed = convert("test1-tonyv:123") + val port: Int = parsed(0).port.get() + assertThat(port, is(123)) + } + + @Test + def multiple_spaces_are_supported { + val parsed = convert("test1 test2") + assertThat(parsed.size, is(2)) + + val hostNames = parsed.map(_.hostName) + assertThat(hostNames, arrayContaining("test1", "test2")) + } + + @Test(expected = classOf[IllegalArgumentException]) + def missing_port_gives_exception { + convert("test1-tonyv:") + } + + @Test(expected = classOf[IllegalArgumentException]) + def non_numeric_port_gives_exception { + convert("test1-tonyv:non-numeric") + } + + @Test + def string_arrays_are_split_on_spaces { + val parsed = convert("/home/vespa/foo /home/vespa/bar ") + assertThat(parsed.size, is(2)) + } + + @Test + def string_arrays_are_split_on_comma { + val parsed = convert("/home/vespa/foo,/home/vespa/bar,") + assertThat(parsed.size, is(2)) + } +} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala new file mode 100644 index 00000000000..7152c0c0af1 --- /dev/null +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import com.yahoo.config.model.producer.AbstractConfigProducerRoot +import com.yahoo.config.model.test.MockRoot +import com.yahoo.container.Container +import com.yahoo.jdisc.test.TestDriver +import scala.xml.Node +import com.yahoo.vespa.model.VespaModel +import com.yahoo.io.IOUtils +import java.nio.file.{Files, Path} +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking + +/** + * Creates a local application from vespa-services fragments. + * + * @author tonytv + */ +object StandaloneContainer { + def firstContainerId(root: AbstractConfigProducerRoot): String = { + root.getConfigProducer("container").get().getConfigId + } + + def withStandaloneContainer[T](containerNode: Node) { + withTempDirectory { applicationDirectory => + System.setProperty(StandaloneContainerApplication.applicationLocationYinstVariable, applicationDirectory.toString) + createServicesXml(applicationDirectory, containerNode) + + val driver = TestDriver.newInjectedApplicationInstance(classOf[StandaloneContainerApplication]) + driver.close() + Container.resetInstance() + } + } + + def withContainerModel[T](containerNode: Node)(f: MockRoot => T) { + withTempDirectory { applicationPath => + createServicesXml(applicationPath, containerNode) + + val distributedFiles = new LocalFileDb(applicationPath) + val (root, container) = StandaloneContainerApplication.createContainerModel( + applicationPath, + distributedFiles, + applicationPath.resolve("preprocesedApp").toFile, + networkingOption = Networking.enable) + f(root) + } + } + + private def withTempDirectory[T](f : Path => T) : T = { + val directory = Files.createTempDirectory("application") + try { + f(directory) + } finally { + IOUtils.recursiveDeleteDir(directory.toFile) + } + } + + private def createServicesXml(applicationPath : Path, + containerNode: Node) { + + scala.xml.XML.save(applicationPath.resolve("services.xml").toFile.getAbsolutePath, + containerNode, xmlDecl = true) + } +} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala new file mode 100644 index 00000000000..41026e1c263 --- /dev/null +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + + +import org.junit.Assert._ +import org.junit.Test +import org.hamcrest.CoreMatchers.is +import org.hamcrest.text.StringContainsInOrder.stringContainsInOrder +import StandaloneContainerTest._ +import com.yahoo.vespa.model.AbstractService +import java.util.Arrays.asList +import scala.util.Try + + +/** + * @author tonytv + * @author gjoranv + */ + +class StandaloneContainerTest { + @Test + def container_is_allowed_root_element() { + StandaloneContainer.withContainerModel(plainXml) { root => } + } + + @Test + def services_is_allowed_root_element() { + val servicesXml = + + + + + StandaloneContainer.withContainerModel(servicesXml) { root => } + } + + @Test + def multiple_container_elements_cannot_be_deployed() { + val twoContainersXml = + + + + + + assertTrue( + Try { + StandaloneContainer.withContainerModel(twoContainersXml) { root => } + }.isFailure) + } + + @Test + def application_preprocessor_is_run() { + val servicesXml = + + + container-1 + + + + StandaloneContainer.withContainerModel(servicesXml) { + root => + assertNotNull(root.getProducer("container-1/standalone")) + } + } + + @Test + def no_default_ports_are_enabled_when_using_http() { + val xml = + + + + + + + StandaloneContainer.withContainerModel(xml) { root => + val container = root.getProducer("jdisc/standalone").asInstanceOf[AbstractService] + println("portCnt: " + container.getPortCount) + println("numPorts: " + container.getNumPortsAllocated) + assertThat(container.getNumPortsAllocated, is(1)) + } + } + + @Test + def manhattan_http_port_yinst_variable_must_be_set() { + System.clearProperty(StandaloneContainerApplication.manhattanHttpPortYinstVariable) + try { + StandaloneContainerApplication.manhattanHttpPort + fail("Port should be a required setting") + } catch { + case e: Exception => + assertThat(e.getMessage, stringContainsInOrder(asList("Environment variable not set", "manhattan_http_port"))) + } + } + + @Test + def manhattan_http_port_must_be_positive_integer() { + assertThat(setAndGetPort(1234), is(1234)) + assertTrue(Try { setAndGetPort(-1) }.isFailure) + assertTrue(Try { setAndGetPort( 0) }.isFailure) + } + + def setAndGetPort(port: Int): Int = { + System.setProperty(StandaloneContainerApplication.manhattanHttpPortYinstVariable, Integer.toString(port)) + StandaloneContainerApplication.manhattanHttpPort + } +} + +object StandaloneContainerTest { + + val plainXml = +} diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala new file mode 100644 index 00000000000..40e4480be37 --- /dev/null +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneSubscriberTest.scala @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.standalone + +import org.junit.{Ignore, Test} +import org.junit.Assert.assertThat +import org.hamcrest.CoreMatchers.is +import org.hamcrest.number.OrderingComparison.greaterThan + +import StandaloneContainer.withContainerModel +import com.yahoo.vespa.config.ConfigKey +import com.yahoo.config.ConfigInstance +import com.yahoo.container.{ComponentsConfig, BundlesConfig, di} +import scala.collection.JavaConverters._ + +/** + * @author tonytv + */ +class StandaloneSubscriberTest { + val bundlesKey = key("bundles") + val componentsKey = key("components") + + def key(name: String) = new ConfigKey(name, "container", "container").asInstanceOf[ConfigKey[ConfigInstance]] + + def box(i: Int) = new java.lang.Integer(i) + + @Test + @Ignore + def standalone_subscriber() { + withContainerModel( ) { root => + val subscriber = new StandaloneSubscriberFactory(root).getSubscriber(Set(bundlesKey, componentsKey).asJava) + val config = subscriber.config.asScala + assertThat(config.size, is(2)) + + val bundlesConfig = config(bundlesKey).asInstanceOf[BundlesConfig] + val componentsConfig = config(componentsKey).asInstanceOf[ComponentsConfig] + + assertThat(bundlesConfig.bundle().size(), is(0)) + assertThat(box(componentsConfig.components().size()), greaterThan(box(10))) + } + } +} -- cgit v1.2.3