diff options
13 files changed, 370 insertions, 8 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index c1c08e6e702..8d8ea3203b5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -279,12 +279,17 @@ <artifactId>filedistribution</artifactId> <version>${project.version}</version> </dependency> - <dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>searchsummary</artifactId> <version>${project.version}</version> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>logforwarder</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.major-version}</artifactId> <scope>test</scope> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java new file mode 100644 index 00000000000..90fb66fc400 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.cloud.config.LogforwarderConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; + +import java.util.ArrayList; +import java.util.List; + +public class LogForwarder extends AbstractService implements LogforwarderConfig.Producer { + + private final String type; + private final List<String> sources; + private final String endpoints; + private final String indexName; + + /** + * Creates a new LogForwarder instance. + */ + // TODO: Use proper types? + public LogForwarder(AbstractConfigProducer parent, int index, String type, List<String> sources, String endpoints, String indexName) { + super(parent, "logforwarder." + index); + this.type = type; + this.sources = sources; + this.endpoints = endpoints; + this.indexName = indexName; + setProp("clustertype", "hosts"); + setProp("clustername", "admin"); + } + + /** + * LogForwarder does not need any ports. + * + * @return The number of ports reserved by the LogForwarder + */ + public int getPortCount() { return 0; } + + /** + * @return The command used to start LogForwarder + */ + public String getStartupCommand() { return "exec $ROOT/libexec/vespa/vespa-logforwarder-start " + getConfigId(); } + + @Override + public void getConfig(LogforwarderConfig.Builder builder) { + List<LogforwarderConfig.Sources.Builder> sourceList = new ArrayList<>(); + for (String s : this.sources) { + LogforwarderConfig.Sources.Builder source = new LogforwarderConfig.Sources.Builder(); + source.log(s); + sourceList.add(source); + } + + builder.type(type); + builder.endpoint(endpoints); + builder.index(indexName); + builder.sources(sourceList); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 908481aad63..79340e9e7ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -19,6 +19,7 @@ import com.yahoo.config.application.api.FileRegistry; import org.w3c.dom.Element; import java.util.*; +import java.util.stream.Collectors; import static com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets.predefinedMetricSets; @@ -111,4 +112,27 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu return minutes; } + void addLogForwarders(ModelElement logForwardingElement, Admin admin) { + if (logForwardingElement == null) return; + + int i = 0; + for (ModelElement e : logForwardingElement.getChildren("forward")) { + String type = e.getStringAttribute("type"); + List<String> sources = e.getChild("source").getChildren("log") + .stream() + .map(ModelElement::asString) + .collect(Collectors.toList()); + String index = e.getChild("destination").getChild("index").asString(); + String endpoint = e.getChild("destination").getChild("endpoint").asString(); + + for (HostResource host : admin.getHostSystem().getHosts()) { + System.out.println("Adding " + e + " on host " + host.getHostName()); + LogForwarder logForwarder = new LogForwarder(admin, i, type, sources, endpoint, index); + logForwarder.setHostResource(host); + logForwarder.initService(); + i++; + } + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 180cf4eadec..dd1d4e36255 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -49,6 +49,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { admin.addSlobroks(getSlobroks(admin, XML.getChild(adminE, "slobroks"))); if ( ! admin.multitenant()) admin.setClusterControllers(addConfiguredClusterControllers(admin, adminE)); + + ModelElement adminElement = new ModelElement(adminE); + addLogForwarders(adminElement.getChild("logforwarding"), admin); } private List<Configserver> parseConfigservers(Admin admin, Element adminE) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 5e22ba3961f..5642c788c0d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -19,6 +19,8 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; /** * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used. @@ -51,6 +53,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { assignSlobroks(requestedSlobroks.orElse(NodesSpecification.nonDedicated(3, version)), admin); assignLogserver(requestedLogservers.orElse(NodesSpecification.nonDedicated(1, version)), admin); + + addLogForwarders(adminElement.getChild("logforwarding"), admin); } private void assignSlobroks(NodesSpecification nodesSpecification, Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java index c5bc275a9e1..1fc70e343f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java @@ -11,7 +11,7 @@ import java.util.List; import java.util.StringTokenizer; /** - * A w3c Element wrapper whith a better API. + * A w3c Element wrapper with a better API. * * Author unknown. */ @@ -46,6 +46,18 @@ public class ModelElement { return null; } + /** + * If not found, return empty list + */ + public List<ModelElement> getChildren(String name) { + List<Element> e = XML.getChildren(xml, name); + + List<ModelElement> list = new ArrayList<>(); + e.forEach(element -> list.add(new ModelElement(element))); + + return list; + } + public ModelElement getChildByPath(String path) { StringTokenizer tokenizer = new StringTokenizer(path, "."); ModelElement curElem = this; diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 26705784a34..93807e985f0 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -13,7 +13,8 @@ AdminV2 = AdminSlobroks? & (LegacyAdminMonitoring | AdminMonitoring)? & (LegacyMetricConsumers | Metrics)? & - ClusterControllers? + ClusterControllers? & + LogForwarding? } AdminV3 = @@ -32,7 +33,8 @@ AdminV4 = AdminV4LogServers? & GenericConfig* & (LegacyAdminMonitoring | AdminMonitoring)? & - (LegacyMetricConsumers | Metrics)? + (LegacyMetricConsumers | Metrics)? & + LogForwarding? } AdminV4Slobroks = @@ -112,3 +114,17 @@ ClusterControllers = element cluster-controllers { service.attlist }+ } + +LogForwarding = element logforwarding { + element forward { + attribute type {xsd:string } & + element source { + element log { "access"| "vespa" }+ + } & + element destination { + element endpoint { xsd:string } & + element index { xsd:string } # TODO: Mandatory or with a default value? + } + }+ + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 2dfef135425..5ee0528ca7e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -8,6 +8,7 @@ import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.ApplicationId; @@ -22,9 +23,12 @@ import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.StatisticsComponent; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Ignore; import org.junit.Test; import java.util.Set; +import java.util.stream.IntStream; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; @@ -246,7 +250,47 @@ public class AdminTestCase { assertEquals(sc.values(0).operations(0).name(), StatisticsConfig.Values.Operations.Name.REGULAR); assertEquals(sc.values(0).operations(0).arguments(0).key(), "limits"); assertEquals(sc.values(0).operations(0).arguments(0).value(), "25,50,100,500"); - } + @Test + public void testLogForwarding() throws Exception { + String hosts = "<hosts>" + + " <host name=\"myhost0\">" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + String services = "<services>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " <logforwarding>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " <log>vespa</log>" + + " </source>" + + " <destination>" + + " <endpoint>host1:port,host2:port</endpoint>" + + " <index>all</index>" + + " </destination>" + + " </forward>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " </source>" + + " <destination>" + + " <endpoint>host3:port</endpoint>" + + " <index>access</index>" + + " </destination>" + + " </forward>" + + " </logforwarding>" + + " </admin>" + + "</services>"; + + VespaModel vespaModel = new VespaModelCreatorWithMockPkg(hosts, services).create(); + + Set<String> configIds = vespaModel.getConfigIds(); + // 2 logforwarders on each host + IntStream.of(0, 1).forEach(i -> assertTrue(configIds.toString(), configIds.contains("admin/logforwarder." + i))); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index 5b5094a9c43..8b451381dbe 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.cloud.config.LogforwarderConfig; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.application.api.ApplicationPackage; @@ -16,13 +17,13 @@ import org.junit.Test; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; /** * @author lulf @@ -141,6 +142,80 @@ public class DedicatedAdminV4Test { "slobrok", "logd", "filedistributorservice", "qrserver"); } + @Test + public void testLogForwarding() throws IOException, SAXException { + String services = "<services>" + + " <admin version='4.0'>" + + " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + + " <logservers><nodes count='1' dedicated='true'/></logservers>" + + " <logforwarding>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " <log>vespa</log>" + + " </source>" + + " <destination>" + + " <endpoint>host1:port,host2:port</endpoint>" + + " <index>all</index>" + + " </destination>" + + " </forward>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " </source>" + + " <destination>" + + " <endpoint>host3:port</endpoint>" + + " <index>access</index>" + + " </destination>" + + " </forward>" + + " </logforwarding>" + + " </admin>" + + "</services>"; + + VespaModel model = createModel(hosts, services); + assertEquals(3, model.getHosts().size()); + + assertHostContainsServices(model, "hosts/myhost0", + "filedistributorservice", "logd", "logforwarder", "logforwarder2", "slobrok"); + assertHostContainsServices(model, "hosts/myhost1", + "filedistributorservice", "logd", "logforwarder", "logforwarder2", "slobrok"); + assertHostContainsServices(model, "hosts/myhost2", + "filedistributorservice", "logd", "logforwarder", "logforwarder2", "logserver"); + + Set<String> configIds = model.getConfigIds(); + // 2 logforwarders on each host + IntStream.of(0, 1, 2, 3, 4, 5).forEach(i -> assertTrue(configIds.toString(), configIds.contains("admin/logforwarder." + i))); + + // First forwarder + { + LogforwarderConfig.Builder builder = new LogforwarderConfig.Builder(); + model.getConfig(builder, "admin/logforwarder.0"); + LogforwarderConfig config = new LogforwarderConfig(builder); + + assertEquals("splunk", config.type()); + assertEquals("host1:port,host2:port", config.endpoint()); + assertEquals("all", config.index()); + List<LogforwarderConfig.Sources> sources = config.sources(); + assertEquals(2, sources.size()); + assertEquals("access", sources.get(0).log()); + assertEquals("vespa", sources.get(1).log()); + } + + // Two forwarders on each host, so forwarder with id admin/logforwarder.3 should be different + { + LogforwarderConfig.Builder builder = new LogforwarderConfig.Builder(); + model.getConfig(builder, "admin/logforwarder.3"); + LogforwarderConfig config = new LogforwarderConfig(builder); + + assertEquals("splunk", config.type()); + assertEquals("host3:port", config.endpoint()); + assertEquals("access", config.index()); + List<LogforwarderConfig.Sources> sources = config.sources(); + assertEquals(1, sources.size()); + assertEquals("access", sources.get(0).log()); + } + } + private Set<String> serviceNames(VespaModel model, String hostname) { SentinelConfig config = model.getConfig(SentinelConfig.class, hostname); return config.service().stream().map(SentinelConfig.Service::name).collect(Collectors.toSet()); diff --git a/logforwarder/pom.xml b/logforwarder/pom.xml new file mode 100644 index 00000000000..505ca9f108b --- /dev/null +++ b/logforwarder/pom.xml @@ -0,0 +1,40 @@ +<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + </parent> + <artifactId>logforwarder</artifactId> + <version>6-SNAPSHOT</version> + <packaging>jar</packaging> + <name>${project.artifactId}</name> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-class-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <id>config-gen</id> + <goals> + <goal>config-gen</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/logforwarder/src/main/resources/configdefinitions/logforwarder.def b/logforwarder/src/main/resources/configdefinitions/logforwarder.def new file mode 100644 index 00000000000..66750a81af8 --- /dev/null +++ b/logforwarder/src/main/resources/configdefinitions/logforwarder.def @@ -0,0 +1,7 @@ +namespace=cloud.config + +# Forwarder that forwards onle or more types of log to an endpoint +type string +sources[].log string +endpoint string +index string diff --git a/logforwarder/src/main/sh/vespa-logforwarder-start.sh b/logforwarder/src/main/sh/vespa-logforwarder-start.sh new file mode 100755 index 00000000000..20864d7b4e6 --- /dev/null +++ b/logforwarder/src/main/sh/vespa-logforwarder-start.sh @@ -0,0 +1,72 @@ +#!/bin/sh +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +findpath () { + myname=${0} + mypath=${myname%/*} + myname=${myname##*/} + if [ "$mypath" ] && [ -d "$mypath" ]; then + return + fi + mypath=$(pwd) + if [ -f "${mypath}/${myname}" ]; then + return + fi + echo "FATAL: Could not figure out the path where $myname lives from $0" + exit 1 +} + +COMMON_ENV=libexec/vespa/common-env.sh + +source_common_env () { + if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then + export VESPA_HOME + common_env=$VESPA_HOME/$COMMON_ENV + if [ -f "$common_env" ]; then + . $common_env + return + fi + fi + return 1 +} + +findroot () { + source_common_env && return + if [ "$VESPA_HOME" ]; then + echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" + exit 1 + fi + if [ "$ROOT" ] && [ -d "$ROOT" ]; then + VESPA_HOME="$ROOT" + source_common_env && return + fi + findpath + while [ "$mypath" ]; do + VESPA_HOME=${mypath} + source_common_env && return + mypath=${mypath%/*} + done + echo "FATAL: missing VESPA_HOME environment variable" + echo "Could not locate $COMMON_ENV anywhere" + exit 1 +} + +findroot + +# END environment bootstrap section + +ROOT=${VESPA_HOME%/} +export ROOT +cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } + +# TODO: Get config and start logforwarder properly + +configid=$1 + +config=`$VESPA_HOME/bin/vespa-get-config -j -n cloud.config.logforwarder -i $configid` +echo $config + +sleep 10000000 @@ -977,6 +977,7 @@ <module>libmlr</module> <module>linguistics</module> <module>logd</module> + <module>logforwarder</module> <module>logserver</module> <module>messagebus-disc</module> <module>messagebus</module> |