From 2c91925f741655a821de9aefadf26bcf66494b71 Mon Sep 17 00:00:00 2001 From: gjoranv Date: Tue, 18 Feb 2020 11:27:41 +0100 Subject: Add xml schema for 'cloudwatch' element. --- config-model/src/main/resources/schema/admin.rnc | 17 ++++++++++++++++- config-model/src/test/schema-test-files/services.xml | 17 ++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 7a3e2916f94..055f57dd7c0 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -82,10 +82,25 @@ Metrics = element metrics { element metric { attribute id { xsd:Name } & attribute display-name { xsd:Name }? - }* + }* & + Cloudwatch? }+ } +Cloudwatch = element cloudwatch { + attribute region { xsd:Name } & + attribute namespace { xsd:Name } & + ( + ( + element access-key-name { xsd:Name } & + element secret-key-name { xsd:Name } + ) + | + element profile { xsd:Name } + )? + +} + ClusterControllers = element cluster-controllers { attribute standalone-zookeeper { xsd:string }? & element cluster-controller { diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 1bf42650123..b06c93d6406 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -15,14 +15,25 @@ - + + + my-access-key + my-secret-key + - - + + + + + + + + profile-in-credentials-file + -- cgit v1.2.3 From 0bd8944f6c675ccb67750a940a079d674f8569bc Mon Sep 17 00:00:00 2001 From: gjoranv Date: Tue, 18 Feb 2020 11:58:34 +0100 Subject: Move consumer inside 'cloudWatch' - Each cloudwatch is declared inside its consumer. --- metrics-proxy/src/main/resources/configdefinitions/telegraf.def | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def index f3b5db35d52..6abbd7921b5 100644 --- a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def +++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def @@ -5,9 +5,8 @@ package=ai.vespa.metricsproxy.telegraf intervalSeconds int default=60 -# The consumer to get metrics for -vespa.consumer string default="default" - +# The Vespa metrics consumer to get metrics for +cloudWatch[].consumer string cloudWatch[].region string default="us-east-1" cloudWatch[].namespace string -- cgit v1.2.3 From 66cefb721e2e63871430357e7a0ac1e8f4cf0b5d Mon Sep 17 00:00:00 2001 From: gjoranv Date: Tue, 18 Feb 2020 17:43:35 +0100 Subject: Propagate cloudwatch setup to telegraf config. --- .../metricsproxy/MetricsProxyContainerCluster.java | 21 ++++++ .../vespa/model/admin/monitoring/CloudWatch.java | 49 ++++++++++++ .../model/admin/monitoring/MetricsConsumer.java | 14 ++++ .../monitoring/builder/xml/CloudWatchBuilder.java | 36 +++++++++ .../monitoring/builder/xml/MetricsBuilder.java | 8 +- .../MetricsProxyContainerClusterTest.java | 5 +- .../model/admin/metricsproxy/TelegrafTest.java | 87 ++++++++++++++++++++++ 7 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java create mode 100644 config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 071666b5bc7..f81757ac568 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -20,6 +20,7 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import ai.vespa.metricsproxy.rpc.RpcServer; import ai.vespa.metricsproxy.service.ConfigSentinelClient; import ai.vespa.metricsproxy.service.SystemPollerProvider; +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; @@ -67,6 +68,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster cloudWatchBuilder + .accessKeyName(hostedAuth.accessKeyName) + .secretKeyName(hostedAuth.secretKeyName)); + cloudWatch.profile().ifPresent(cloudWatchBuilder::profile); + builder.cloudWatch(cloudWatchBuilder); + } + } + } + @Override public void getConfig(ThreadpoolConfig.Builder builder) { builder.maxthreads(10); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java new file mode 100644 index 00000000000..fd290409ea5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.model.admin.monitoring; + +import java.util.Optional; + +/** + * Helper object for CloudWatch configuration. + * + * @author gjoranv + */ +public class CloudWatch { + private final String region; + private final String namespace; + private final MetricsConsumer consumer; + + private HostedAuth hostedAuth; + private String profile; + + public CloudWatch(String region, String namespace, MetricsConsumer consumer) { + this.region = region; + this.namespace = namespace; + this.consumer = consumer; + } + + public String region() { return region; } + public String namespace() { return namespace; } + public String consumer() { return consumer.getId(); } + + public Optional hostedAuth() {return Optional.ofNullable(hostedAuth); } + public Optional profile() { return Optional.ofNullable(profile); } + + public void setHostedAuth(HostedAuth hostedAuth) { + this.hostedAuth = hostedAuth; + } + + public void setProfile(String profile) { + this.profile = profile; + } + + public static class HostedAuth { + public final String accessKeyName; + public final String secretKeyName; + + public HostedAuth(String accessKeyName, String secretKeyName) { + this.accessKeyName = accessKeyName; + this.secretKeyName = secretKeyName; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index 529ed6ecf67..9c752f3aa0d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -2,9 +2,13 @@ package com.yahoo.vespa.model.admin.monitoring; import javax.annotation.concurrent.Immutable; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; +import static java.util.Collections.unmodifiableList; + /** * Represents an arbitrary metric consumer * @@ -16,6 +20,8 @@ public class MetricsConsumer { private final String id; private final MetricSet metricSet; + private final List cloudWatches = new ArrayList<>(); + /** * @param id The consumer * @param metricSet The metrics for this consumer @@ -38,4 +44,12 @@ public class MetricsConsumer { return metricSet.getMetrics(); } + public void addCloudWatch(CloudWatch cloudWatch) { + cloudWatches.add(cloudWatch); + } + + public List cloudWatches() { + return unmodifiableList(cloudWatches); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java new file mode 100644 index 00000000000..4b9d5542aa9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.model.admin.monitoring.builder.xml; + +import com.yahoo.vespa.model.admin.monitoring.CloudWatch; +import com.yahoo.vespa.model.admin.monitoring.CloudWatch.HostedAuth; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; +import org.w3c.dom.Element; + +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChildValue; + +/** + * @author gjoranv + */ +public class CloudWatchBuilder { + + private static final String REGION_ATTRIBUTE = "region"; + private static final String NAMESPACE_ATTRIBUTE = "namespace"; + private static final String ACCESS_KEY_ELEMENT = "access-key-name"; + private static final String SECRET_KEY_ELEMENT = "secret-key-name"; + private static final String PROFILE_ELEMENT = "profile"; + + public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) { + CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE), + cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE), + consumer); + + getOptionalChildValue(cloudwatchElement, PROFILE_ELEMENT).ifPresent(cloudWatch::setProfile); + + getOptionalChildValue(cloudwatchElement, ACCESS_KEY_ELEMENT) + .ifPresent(accessKey -> cloudWatch.setHostedAuth( + new HostedAuth(accessKey, + getOptionalChildValue(cloudwatchElement, SECRET_KEY_ELEMENT) + .orElseThrow(() -> new IllegalArgumentException("Access key given without a secret key."))))); + return cloudWatch; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index f029dad01a9..b686288868f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -42,7 +42,11 @@ public class MetricsBuilder { throwIfIllegalConsumerId(metrics, consumerId); MetricSet metricSet = buildMetricSet(consumerId, consumerElement); - metrics.addConsumer(new MetricsConsumer(consumerId, metricSet)); + var consumer = new MetricsConsumer(consumerId, metricSet); + for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) { + consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer)); + } + metrics.addConsumer(consumer); } return metrics; } @@ -58,7 +62,7 @@ public class MetricsBuilder { private MetricSet buildMetricSet(String consumerId, Element consumerElement) { List metrics = XML.getChildren(consumerElement, "metric").stream() - .map(metricElement -> metricFromElement(metricElement)) + .map(MetricsBuilder::metricFromElement) .collect(Collectors.toCollection(LinkedList::new)); List metricSets = XML.getChildren(consumerElement, "metric-set").stream() diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 9265e4437f1..8adca89d140 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -6,9 +6,9 @@ package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.core.ConsumersConfig; -import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler; import ai.vespa.metricsproxy.http.application.MetricsNodesConfig; +import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler; import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler; import ai.vespa.metricsproxy.http.yamas.YamasHandler; import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig; @@ -59,6 +59,7 @@ import static java.util.stream.Collectors.toList; import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -105,7 +106,7 @@ public class MetricsProxyContainerClusterTest { assertEquals(512, qrStartConfig.jvm().heapsize()); assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); - assertEquals(false, qrStartConfig.jvm().verbosegc()); + assertFalse(qrStartConfig.jvm().verbosegc()); assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java new file mode 100644 index 00000000000..144c45a7dd2 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java @@ -0,0 +1,87 @@ +package com.yahoo.vespa.model.admin.metricsproxy; + +import ai.vespa.metricsproxy.telegraf.TelegrafConfig; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Test; + +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted; +import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel; +import static org.junit.Assert.assertEquals; + +/** + * @author gjoranv + */ +public class TelegrafTest { + + @Test + public void telegraf_config_is_generated_for_cloudwatch_in_services() { + String services = String.join("\n", + "", + " ", + " ", + " ", + " ", + " ", + " ", + " my-access-key", + " my-secret-key", + " ", + " ", + " ", + " ", + "" + ); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("my-namespace", cloudWatch0.namespace()); + assertEquals("my-access-key", cloudWatch0.accessKeyName()); + assertEquals("my-secret-key", cloudWatch0.secretKeyName()); + assertEquals("", cloudWatch0.profile()); + } + + @Test + public void multiple_cloudwatches_are_allowed_for_the_same_consumer() { + String services = String.join("\n", + "", + " ", + " ", + " ", + " ", + " ", + " ", + " access-key-1", + " secret-key-1", + " ", + " ", + " profile-2", + " ", + " ", + " ", + " ", + "" + ); + VespaModel hostedModel = getModel(services, hosted); + TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID); + + var cloudWatch0 = config.cloudWatch(0); + assertEquals("cloudwatch-consumer", cloudWatch0.consumer()); + assertEquals("us-east-1", cloudWatch0.region()); + assertEquals("namespace-1", cloudWatch0.namespace()); + assertEquals("access-key-1", cloudWatch0.accessKeyName()); + assertEquals("secret-key-1", cloudWatch0.secretKeyName()); + assertEquals("", cloudWatch0.profile()); + + var cloudWatch1 = config.cloudWatch(1); + assertEquals("cloudwatch-consumer", cloudWatch1.consumer()); + assertEquals("us-east-1", cloudWatch1.region()); + assertEquals("namespace-2", cloudWatch1.namespace()); + assertEquals("", cloudWatch1.accessKeyName()); + assertEquals("", cloudWatch1.secretKeyName()); + assertEquals("profile-2", cloudWatch1.profile()); + } + +} -- cgit v1.2.3