From 958dc05771460cf6264b0ed03a812d152d429deb Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 16 May 2017 14:24:00 +0200 Subject: Remove Manhattan integration --- .../xml/ManhattanContainerModelBuilder.java | 172 --------------------- .../xml/ManhattanContainerModelBuilderTest.scala | 144 ----------------- .../StandaloneContainerApplication.scala | 57 +++---- .../standalone/StandaloneContainerTest.scala | 31 +--- 4 files changed, 26 insertions(+), 378 deletions(-) delete mode 100644 config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java delete mode 100644 config-model/src/test/scala/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilderTest.scala diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java deleted file mode 100644 index f6ed6c2eb7d..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml; - -import com.yahoo.component.ComponentId; -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.container.jdisc.config.MetricDefaultsConfig; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.model.container.ContainerCluster; -import com.yahoo.vespa.model.container.component.AccessLogComponent; -import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; -import com.yahoo.vespa.model.container.http.ConnectorFactory; -import com.yahoo.vespa.model.container.http.FilterChains; -import com.yahoo.vespa.model.container.http.Http; -import com.yahoo.vespa.model.container.http.JettyHttpServer; -import org.w3c.dom.Element; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.yahoo.collections.CollectionUtil.first; -import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme; -import static com.yahoo.vespa.model.container.xml.BundleMapper.absoluteBundlePath; - -/** - * @author tonytv - */ -public final class ManhattanContainerModelBuilder extends ContainerModelBuilder { - - static final String MANHATTAN_FILE_NAME_PATTERN = Defaults.getDefaults().vespaHome() + "logs/jdisc_core/access.%Y-%m-%d-%H"; - static final String MANHATTAN_ROTATION_INTERVAL = "0 60 ..."; - static final RotateScheme.Enum MANHATTAN_ROTATION_SCHEME = RotateScheme.DATE; - static final String MANHATTAN_SYMLINK_NAME = "access"; - - public interface BundleFiles { - // TODO: move constants to the DH code base. - Set dhBundles = new HashSet<>(Arrays.asList( - Paths.get("apache_avro/avro.jar"), - Paths.get("apache_avro/commons-compress.jar"), - Paths.get("apache_avro/paranamer.jar"), - Paths.get("apache_avro/jackson-core-asl.jar"), - Paths.get("apache_avro/jackson-mapper-asl.jar"), - Paths.get("dh_rainbow_client_api_java.jar"), - Paths.get("dh_rainbow_util_batch_java.jar"), - Paths.get("dh_rainbow_util_java.jar"))); - } - - private final int httpPort; - private JettyHttpServer jettyHttpServer; - - public ManhattanContainerModelBuilder(int httpPort) { - super(true, Networking.enable); - this.httpPort = httpPort; - } - - @Override - protected void addBundlesForPlatformComponents(ContainerCluster cluster) { - super.addBundlesForPlatformComponents(cluster); - BundleFiles.dhBundles.forEach( - bundleFile -> cluster.addPlatformBundle(absoluteBundlePath(bundleFile))); - } - - @Override - protected void setDefaultMetricConsumerFactory(ContainerCluster cluster) { - cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD); - } - - @Override - protected void addAccessLogs(ContainerCluster cluster, Element spec) { - warnIfAccessLogsDefined(spec); - - checkNotNull(jettyHttpServer, "addHttp must be called first"); - cluster.addComponent(createManhattanAccessLog()); - } - - private Component createManhattanAccessLog() { - return new AccessLogComponent(AccessLogComponent.AccessLogType.yApacheAccessLog, - MANHATTAN_FILE_NAME_PATTERN, - MANHATTAN_ROTATION_INTERVAL, - MANHATTAN_ROTATION_SCHEME, - MANHATTAN_SYMLINK_NAME); - } - - private void warnIfAccessLogsDefined(Element spec) { - List accessLogElements = getAccessLogElements(spec); - if (!accessLogElements.isEmpty()) { - logManhattanInfo("Ignoring " + accessLogElements.size() + - " access log elements in services.xml, using default yapache access logging instead."); - } - } - - @Override - protected void addDefaultHandlers(ContainerCluster cluster) { - addDefaultHandlersExceptStatus(cluster); - } - - @Override - protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) { - addStatusHandlerForJDiscStatusPackage(cluster, "status.html"); //jdisc_status - addStatusHandlerForJDiscStatusPackage(cluster, "akamai"); //jdisc_akamai - } - - private static void addStatusHandlerForJDiscStatusPackage(ContainerCluster cluster, String name) { - cluster.addComponent( - new FileStatusHandlerComponent(name + "-status-handler", Defaults.getDefaults().vespaHome() + "libexec/jdisc/" + name, - "http://*/" + name, "https://*/" + name)); - } - - @Override - protected void addHttp(Element spec, ContainerCluster cluster) { - super.addHttp(spec, cluster); - ensureHasHttp(cluster); - ensureOneHttpServer(cluster.getHttp()); - } - - private void ensureHasHttp(ContainerCluster cluster) { - if (cluster.getHttp() == null) - cluster.setHttp(createHttp()); - } - - private Http createHttp() { - Http http = new Http(Collections.emptyList()); - http.setFilterChains(new FilterChains(http)); - return http; - } - - private void ensureOneHttpServer(Http http) { - if (http.getHttpServer() == null || http.getHttpServer().getConnectorFactories().isEmpty()) { - JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("main-http-server")); - http.setHttpServer(jettyHttpServer); - ConnectorFactory connectorFactory = new ConnectorFactory("main-http-connector", - httpPort, null); - http.getHttpServer().addConnector(connectorFactory); - } else { - removeAllButOneConnector(http.getHttpServer()); - ConnectorFactory connectorFactory = first(http.getHttpServer().getConnectorFactories()); - connectorFactory.setListenPort(httpPort); - } - jettyHttpServer = http.getHttpServer(); - } - - private void removeAllButOneConnector(JettyHttpServer jettyHttpServer) { - int removed = 0; - - if (jettyHttpServer.getConnectorFactories().size() > 1) { - for (int i = jettyHttpServer.getConnectorFactories().size() - 1; i > 0; i--) { - ConnectorFactory c = jettyHttpServer.getConnectorFactories().get(i); - jettyHttpServer.removeConnector(c); - ++removed; - } - } - - if (removed > 0) { - logManhattanInfo("Using only the first http server " + jettyHttpServer.getConnectorFactories().get(0).getName()); - } - } - - private static List tail(List list) { - return list.subList(1, list.size()); - } - - private void logManhattanInfo(String message) { - log.log(Level.INFO, "[Manhattan] " + message); - } -} diff --git a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilderTest.scala b/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilderTest.scala deleted file mode 100644 index 4a437994656..00000000000 --- a/config-model/src/test/scala/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilderTest.scala +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.xml - - -import org.junit.Test -import scala.xml.{PrettyPrinter, Elem} - -import ManhattanContainerModelBuilderTest._ -import com.yahoo.config.model.test.MockRoot -import org.apache.commons.io.IOUtils -import com.yahoo.vespa.model.container.ContainerCluster -import com.yahoo.vespa.model.container.component.{Component, AccessLogComponent} -import scala.collection.JavaConversions._ -import scala.reflect.ClassTag -import com.yahoo.config.model.producer.AbstractConfigProducer -import com.yahoo.osgi.provider.model.ComponentModel -import org.junit.Assert.{assertThat, assertNotNull} -import org.hamcrest.CoreMatchers.is -import com.yahoo.container.handler.VipStatusHandler -import com.yahoo.config.model.builder.xml.XmlHelper.getDocumentBuilder -import com.yahoo.vespa.model.container.search.searchchain.FederationSearcherTest -import com.yahoo.container.jdisc.config.HttpServerConfig -import com.yahoo.config.model.deploy.DeployState - -import scala.language.reflectiveCalls - -/** - * @author tonytv - */ -class ManhattanContainerModelBuilderTest { - - val emptyJDiscElement = - - @Test - def multiple_access_logs_configured() { - val container = buildManhattanContainer( - - - - ) - } - - @Test - def status_html_and_akamai_handlers_configured() { - val container = buildManhattanContainer(emptyJDiscElement) - - val vipStatusComponents = getComponentsWithModelClass[VipStatusHandler](container) - val ids = vipStatusComponents map { _.model.getComponentId.getName } - - assertThat(ids.toSet, is(Set("status.html-status-handler", "akamai-status-handler"))) - } - - @Test - def http_server_added_automatically() { - val container = buildManhattanContainer(emptyJDiscElement) - - assertThat(((container.getHttp.getHttpServer != null) && (container.getHttp.getHttpServer.getConnectorFactories.size() == 1)), is(true)) - assertThat(container.getHttp.getHttpServer.getConnectorFactories.head.getListenPort, is(httpPort)) - } - - @Test - def only_the_first_http_server_is_kept() { - val container = buildManhattanContainer( - - - - - - ) - - assertThat(((container.getHttp.getHttpServer != null) && (container.getHttp.getHttpServer.getConnectorFactories.size() == 1)), is(true)) - assertThat(container.getHttp.getHttpServer.getComponentId.getName, is("jdisc-jetty")) - assertThat(container.getHttp.getHttpServer.getConnectorFactories.head.getName, is("server1")) - assertThat(container.getHttp.getHttpServer.getConnectorFactories.head.getListenPort, is(httpPort)) - } - - @Test - def filters_and_bindings_are_preserved() { - val container = buildManhattanContainer( - - - - - - - http://*:123/my-binding - - - - - ) - - val binding = container.getHttp.getBindings.head - assertThat(binding.filterId.getName, is("my-chain")) - assertThat(binding.binding, is("http://*:123/my-binding")) - - val filterChains = container.getHttp.getFilterChains - assertNotNull("Missing filter", filterChains.componentsRegistry().getComponent("my-filter")) - assertNotNull("Missing chain", filterChains.allChains().getComponent("my-chain")) - } -} - -object ManhattanContainerModelBuilderTest { - type ACP = AbstractConfigProducer[_] - type COMPONENT = Component[_ <: ACP, _ <: ComponentModel] - - val httpPort = 9876 - - def getComponents[T <: COMPONENT](cluster: ContainerCluster)(implicit tag: ClassTag[T]): Iterable[T] = { - fixType(cluster.getComponentsMap.values()) collect { case c: T => c } - } - - def getComponentsWithModelClass[T <: AnyRef](cluster: ContainerCluster)(implicit tag: ClassTag[T]) = { - val className = tag.runtimeClass.getName - fixType(cluster.getAllComponents) filter { _.model.getClassId.getName == className } - } - - def modelClassIdMatches(name: String): PartialFunction[COMPONENT, COMPONENT] = { - case c: COMPONENT if c.model.getClassId.getName == name => c - } - - def fixType(components: java.util.Collection[_ <: Component[_, _]]): java.util.Collection[COMPONENT] = - components.asInstanceOf[java.util.Collection[COMPONENT]] - - def buildManhattanContainer(elem: Elem) = { - val root = new MockRoot() - val containerModel = new ManhattanContainerModelBuilder(httpPort).build(DeployState.createTestState(), null, root, domElement(elem)) - root.freezeModelTopology() - containerModel.getCluster() - } - - def xmlStringBuilder(elem: Elem) = { - val printer = new PrettyPrinter(240, 2) - val builder = new StringBuilder - builder.append("\n") - printer.format(elem, builder) - builder - } - - def domElement(elem: Elem) = { - val stream = IOUtils.toInputStream(xmlStringBuilder(elem)) - getDocumentBuilder.parse(stream).getDocumentElement - } -} diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala index fc57353c194..c8eb8666bed 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -1,36 +1,36 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.standalone -import com.google.inject.{Key, AbstractModule, Injector, Inject} -import com.yahoo.config.application.api.{RuleConfigDeriver, FileRegistry, ApplicationPackage} +import java.io.{File, IOException} +import java.lang.{Boolean => JBoolean} +import java.nio.file.{FileSystems, Files, Path, Paths} + +import com.google.inject.name.Names +import com.google.inject.{AbstractModule, Inject, Injector, Key} +import com.yahoo.collections.CollectionUtil.first +import com.yahoo.config.application.api.{ApplicationPackage, FileRegistry, RuleConfigDeriver} +import com.yahoo.config.model.application.provider._ +import com.yahoo.config.model.builder.xml.XmlHelper +import com.yahoo.config.model.deploy.DeployState +import com.yahoo.config.model.{ApplicationConfigProducerRoot, ConfigModelRepo} import com.yahoo.config.provision.Zone -import com.yahoo.jdisc.application.Application +import com.yahoo.container.di.config.SubscriberFactory import com.yahoo.container.jdisc.ConfiguredApplication -import java.io.{IOException, File} -import com.yahoo.config.model.test.MockRoot -import com.yahoo.config.model.application.provider._ +import com.yahoo.container.standalone.Environment._ +import com.yahoo.container.standalone.StandaloneContainerApplication._ +import com.yahoo.io.IOUtils +import com.yahoo.jdisc.application.Application +import com.yahoo.text.XML import com.yahoo.vespa.defaults.Defaults import com.yahoo.vespa.model.VespaModel -import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ManhattanContainerModelBuilder, ContainerModelBuilder} -import org.w3c.dom.Element -import com.yahoo.config.model.builder.xml.XmlHelper -import com.yahoo.vespa.model.container.Container -import com.yahoo.collections.CollectionUtil.first import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder -import com.yahoo.io.IOUtils -import com.yahoo.container.di.config.SubscriberFactory -import StandaloneContainerApplication._ -import com.google.inject.name.Names -import scala.util.Try -import java.nio.file.{FileSystems, Path, Paths, Files} -import com.yahoo.config.model.{ConfigModelRepo, ApplicationConfigProducerRoot} -import scala.collection.JavaConversions._ -import com.yahoo.text.XML +import com.yahoo.vespa.model.container.Container import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking +import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ContainerModelBuilder} +import org.w3c.dom.Element -import java.lang.{ Boolean => JBoolean } -import Environment._ -import com.yahoo.config.model.deploy.DeployState +import scala.collection.JavaConversions._ +import scala.util.Try /** * @author tonytv @@ -99,7 +99,6 @@ object StandaloneContainerApplication { val packageName = "standalone_jdisc_container" val applicationLocationYinstVariable = s"$packageName.app_location" val deploymentProfileYinstVariable = s"$packageName.deployment_profile" - val manhattanHttpPortYinstVariable = s"$packageName.manhattan_http_port" val applicationPathName = Names.named(applicationLocationYinstVariable) @@ -146,21 +145,11 @@ object StandaloneContainerApplication { def newContainerModelBuilder(networkingOption: Networking): ContainerModelBuilder = { optionalYinstVariable(deploymentProfileYinstVariable) match { case None => new ContainerModelBuilder(true, networkingOption) - case Some("manhattan") => new ManhattanContainerModelBuilder(manhattanHttpPort) case Some("configserver") => new ConfigServerContainerModelBuilder(new CloudConfigYinstVariables) case profileName => throw new RuntimeException(s"Invalid deployment profile '$profileName'") } } - def manhattanHttpPort: Int = { - val port = yinstVariable(manhattanHttpPortYinstVariable) - Try { - Integer.parseInt(port) - } filter( _ > 0) getOrElse { - throw new RuntimeException(s"$manhattanHttpPortYinstVariable is not a valid port: '$port'") - } - } - def createContainerModel(applicationPath: Path, fileRegistry: FileRegistry, preprocessedApplicationDir: File, diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala index 2705322ab32..00d84d36165 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala @@ -2,13 +2,11 @@ package com.yahoo.container.standalone +import com.yahoo.container.standalone.StandaloneContainerTest._ +import com.yahoo.vespa.model.AbstractService import org.junit.Assert._ import org.junit.Test -import org.hamcrest.CoreMatchers.is -import org.hamcrest.text.StringContainsInOrder.stringContainsInOrder -import StandaloneContainerTest._ -import com.yahoo.vespa.model.AbstractService -import java.util.Arrays.asList + import scala.util.Try @@ -79,29 +77,6 @@ class StandaloneContainerTest { } } - @Test - def manhattan_http_port_yinst_variable_must_be_set() { - System.clearProperty(StandaloneContainerApplication.manhattanHttpPortYinstVariable) - try { - StandaloneContainerApplication.manhattanHttpPort - fail("Port should be a required setting") - } catch { - case e: Exception => - assertThat(e.getMessage, stringContainsInOrder(asList("Environment variable not set", "manhattan_http_port"))) - } - } - - @Test - def manhattan_http_port_must_be_positive_integer() { - assertThat(setAndGetPort(1234), is(1234)) - assertTrue(Try { setAndGetPort(-1) }.isFailure) - assertTrue(Try { setAndGetPort( 0) }.isFailure) - } - - def setAndGetPort(port: Int): Int = { - System.setProperty(StandaloneContainerApplication.manhattanHttpPortYinstVariable, Integer.toString(port)) - StandaloneContainerApplication.manhattanHttpPort - } } object StandaloneContainerTest { -- cgit v1.2.3