diff options
author | Harald Musum <musum@oath.com> | 2017-09-24 21:37:13 +0200 |
---|---|---|
committer | Arne Juul <arnej@yahoo-inc.com> | 2017-10-05 06:24:52 +0000 |
commit | dee4a9914b6928c1712058a162d7d4a133cd57c0 (patch) | |
tree | fc1bb66aafb4f0b327e11c9c71297697d3fc1f8e /config-model | |
parent | fd2fbd3e6137e6b01d8e2466ff066f29d93fe291 (diff) |
Add support for logforwarder
Diffstat (limited to 'config-model')
9 files changed, 250 insertions, 8 deletions
diff --git a/config-model/pom.xml b/config-model/pom.xml index c1c08e6e702..8d8ea3203b5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -279,12 +279,17 @@ <artifactId>filedistribution</artifactId> <version>${project.version}</version> </dependency> - <dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>searchsummary</artifactId> <version>${project.version}</version> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>logforwarder</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.major-version}</artifactId> <scope>test</scope> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java new file mode 100644 index 00000000000..90fb66fc400 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java @@ -0,0 +1,59 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.cloud.config.LogforwarderConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; + +import java.util.ArrayList; +import java.util.List; + +public class LogForwarder extends AbstractService implements LogforwarderConfig.Producer { + + private final String type; + private final List<String> sources; + private final String endpoints; + private final String indexName; + + /** + * Creates a new LogForwarder instance. + */ + // TODO: Use proper types? + public LogForwarder(AbstractConfigProducer parent, int index, String type, List<String> sources, String endpoints, String indexName) { + super(parent, "logforwarder." + index); + this.type = type; + this.sources = sources; + this.endpoints = endpoints; + this.indexName = indexName; + setProp("clustertype", "hosts"); + setProp("clustername", "admin"); + } + + /** + * LogForwarder does not need any ports. + * + * @return The number of ports reserved by the LogForwarder + */ + public int getPortCount() { return 0; } + + /** + * @return The command used to start LogForwarder + */ + public String getStartupCommand() { return "exec $ROOT/libexec/vespa/vespa-logforwarder-start " + getConfigId(); } + + @Override + public void getConfig(LogforwarderConfig.Builder builder) { + List<LogforwarderConfig.Sources.Builder> sourceList = new ArrayList<>(); + for (String s : this.sources) { + LogforwarderConfig.Sources.Builder source = new LogforwarderConfig.Sources.Builder(); + source.log(s); + sourceList.add(source); + } + + builder.type(type); + builder.endpoint(endpoints); + builder.index(indexName); + builder.sources(sourceList); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index 908481aad63..79340e9e7ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -19,6 +19,7 @@ import com.yahoo.config.application.api.FileRegistry; import org.w3c.dom.Element; import java.util.*; +import java.util.stream.Collectors; import static com.yahoo.vespa.model.admin.monitoring.builder.PredefinedMetricSets.predefinedMetricSets; @@ -111,4 +112,27 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu return minutes; } + void addLogForwarders(ModelElement logForwardingElement, Admin admin) { + if (logForwardingElement == null) return; + + int i = 0; + for (ModelElement e : logForwardingElement.getChildren("forward")) { + String type = e.getStringAttribute("type"); + List<String> sources = e.getChild("source").getChildren("log") + .stream() + .map(ModelElement::asString) + .collect(Collectors.toList()); + String index = e.getChild("destination").getChild("index").asString(); + String endpoint = e.getChild("destination").getChild("endpoint").asString(); + + for (HostResource host : admin.getHostSystem().getHosts()) { + System.out.println("Adding " + e + " on host " + host.getHostName()); + LogForwarder logForwarder = new LogForwarder(admin, i, type, sources, endpoint, index); + logForwarder.setHostResource(host); + logForwarder.initService(); + i++; + } + } + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 180cf4eadec..dd1d4e36255 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -49,6 +49,9 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { admin.addSlobroks(getSlobroks(admin, XML.getChild(adminE, "slobroks"))); if ( ! admin.multitenant()) admin.setClusterControllers(addConfiguredClusterControllers(admin, adminE)); + + ModelElement adminElement = new ModelElement(adminE); + addLogForwarders(adminElement.getChild("logforwarding"), admin); } private List<Configserver> parseConfigservers(Admin admin, Element adminE) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 5e22ba3961f..5642c788c0d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -19,6 +19,8 @@ import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; /** * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used. @@ -51,6 +53,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { assignSlobroks(requestedSlobroks.orElse(NodesSpecification.nonDedicated(3, version)), admin); assignLogserver(requestedLogservers.orElse(NodesSpecification.nonDedicated(1, version)), admin); + + addLogForwarders(adminElement.getChild("logforwarding"), admin); } private void assignSlobroks(NodesSpecification nodesSpecification, Admin admin) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java index c5bc275a9e1..1fc70e343f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java @@ -11,7 +11,7 @@ import java.util.List; import java.util.StringTokenizer; /** - * A w3c Element wrapper whith a better API. + * A w3c Element wrapper with a better API. * * Author unknown. */ @@ -46,6 +46,18 @@ public class ModelElement { return null; } + /** + * If not found, return empty list + */ + public List<ModelElement> getChildren(String name) { + List<Element> e = XML.getChildren(xml, name); + + List<ModelElement> list = new ArrayList<>(); + e.forEach(element -> list.add(new ModelElement(element))); + + return list; + } + public ModelElement getChildByPath(String path) { StringTokenizer tokenizer = new StringTokenizer(path, "."); ModelElement curElem = this; diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 26705784a34..93807e985f0 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -13,7 +13,8 @@ AdminV2 = AdminSlobroks? & (LegacyAdminMonitoring | AdminMonitoring)? & (LegacyMetricConsumers | Metrics)? & - ClusterControllers? + ClusterControllers? & + LogForwarding? } AdminV3 = @@ -32,7 +33,8 @@ AdminV4 = AdminV4LogServers? & GenericConfig* & (LegacyAdminMonitoring | AdminMonitoring)? & - (LegacyMetricConsumers | Metrics)? + (LegacyMetricConsumers | Metrics)? & + LogForwarding? } AdminV4Slobroks = @@ -112,3 +114,17 @@ ClusterControllers = element cluster-controllers { service.attlist }+ } + +LogForwarding = element logforwarding { + element forward { + attribute type {xsd:string } & + element source { + element log { "access"| "vespa" }+ + } & + element destination { + element endpoint { xsd:string } & + element index { xsd:string } # TODO: Mandatory or with a default value? + } + }+ + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 2dfef135425..5ee0528ca7e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -8,6 +8,7 @@ import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.ApplicationId; @@ -22,9 +23,12 @@ import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.StatisticsComponent; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.Ignore; import org.junit.Test; import java.util.Set; +import java.util.stream.IntStream; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; @@ -246,7 +250,47 @@ public class AdminTestCase { assertEquals(sc.values(0).operations(0).name(), StatisticsConfig.Values.Operations.Name.REGULAR); assertEquals(sc.values(0).operations(0).arguments(0).key(), "limits"); assertEquals(sc.values(0).operations(0).arguments(0).value(), "25,50,100,500"); - } + @Test + public void testLogForwarding() throws Exception { + String hosts = "<hosts>" + + " <host name=\"myhost0\">" + + " <alias>node0</alias>" + + " </host>" + + "</hosts>"; + + String services = "<services>" + + " <admin version='2.0'>" + + " <adminserver hostalias='node0' />" + + " <logforwarding>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " <log>vespa</log>" + + " </source>" + + " <destination>" + + " <endpoint>host1:port,host2:port</endpoint>" + + " <index>all</index>" + + " </destination>" + + " </forward>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " </source>" + + " <destination>" + + " <endpoint>host3:port</endpoint>" + + " <index>access</index>" + + " </destination>" + + " </forward>" + + " </logforwarding>" + + " </admin>" + + "</services>"; + + VespaModel vespaModel = new VespaModelCreatorWithMockPkg(hosts, services).create(); + + Set<String> configIds = vespaModel.getConfigIds(); + // 2 logforwarders on each host + IntStream.of(0, 1).forEach(i -> assertTrue(configIds.toString(), configIds.contains("admin/logforwarder." + i))); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index 5b5094a9c43..8b451381dbe 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; +import com.yahoo.cloud.config.LogforwarderConfig; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.application.api.ApplicationPackage; @@ -16,13 +17,13 @@ import org.junit.Test; import org.xml.sax.SAXException; import java.io.IOException; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; /** * @author lulf @@ -141,6 +142,80 @@ public class DedicatedAdminV4Test { "slobrok", "logd", "filedistributorservice", "qrserver"); } + @Test + public void testLogForwarding() throws IOException, SAXException { + String services = "<services>" + + " <admin version='4.0'>" + + " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + + " <logservers><nodes count='1' dedicated='true'/></logservers>" + + " <logforwarding>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " <log>vespa</log>" + + " </source>" + + " <destination>" + + " <endpoint>host1:port,host2:port</endpoint>" + + " <index>all</index>" + + " </destination>" + + " </forward>" + + " <forward type='splunk'>" + + " <source>" + + " <log>access</log>" + + " </source>" + + " <destination>" + + " <endpoint>host3:port</endpoint>" + + " <index>access</index>" + + " </destination>" + + " </forward>" + + " </logforwarding>" + + " </admin>" + + "</services>"; + + VespaModel model = createModel(hosts, services); + assertEquals(3, model.getHosts().size()); + + assertHostContainsServices(model, "hosts/myhost0", + "filedistributorservice", "logd", "logforwarder", "logforwarder2", "slobrok"); + assertHostContainsServices(model, "hosts/myhost1", + "filedistributorservice", "logd", "logforwarder", "logforwarder2", "slobrok"); + assertHostContainsServices(model, "hosts/myhost2", + "filedistributorservice", "logd", "logforwarder", "logforwarder2", "logserver"); + + Set<String> configIds = model.getConfigIds(); + // 2 logforwarders on each host + IntStream.of(0, 1, 2, 3, 4, 5).forEach(i -> assertTrue(configIds.toString(), configIds.contains("admin/logforwarder." + i))); + + // First forwarder + { + LogforwarderConfig.Builder builder = new LogforwarderConfig.Builder(); + model.getConfig(builder, "admin/logforwarder.0"); + LogforwarderConfig config = new LogforwarderConfig(builder); + + assertEquals("splunk", config.type()); + assertEquals("host1:port,host2:port", config.endpoint()); + assertEquals("all", config.index()); + List<LogforwarderConfig.Sources> sources = config.sources(); + assertEquals(2, sources.size()); + assertEquals("access", sources.get(0).log()); + assertEquals("vespa", sources.get(1).log()); + } + + // Two forwarders on each host, so forwarder with id admin/logforwarder.3 should be different + { + LogforwarderConfig.Builder builder = new LogforwarderConfig.Builder(); + model.getConfig(builder, "admin/logforwarder.3"); + LogforwarderConfig config = new LogforwarderConfig(builder); + + assertEquals("splunk", config.type()); + assertEquals("host3:port", config.endpoint()); + assertEquals("access", config.index()); + List<LogforwarderConfig.Sources> sources = config.sources(); + assertEquals(1, sources.size()); + assertEquals("access", sources.get(0).log()); + } + } + private Set<String> serviceNames(VespaModel model, String hostname) { SentinelConfig config = model.getConfig(SentinelConfig.class, hostname); return config.service().stream().map(SentinelConfig.Service::name).collect(Collectors.toSet()); |