Merge pull request #12261 from vespa-engine/gjoranv/cloudwatch-consumer-syntax

Gjoranv/cloudwatch consumer syntax 

9 files changed, 246 insertions, 8 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index 071666b5bc7..f81757ac568 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -20,6 +20,7 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
 import ai.vespa.metricsproxy.rpc.RpcServer;
 import ai.vespa.metricsproxy.service.ConfigSentinelClient;
 import ai.vespa.metricsproxy.service.SystemPollerProvider;
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
@@ -67,6 +68,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
         ApplicationDimensionsConfig.Producer,
         ConsumersConfig.Producer,
         MonitoringConfig.Producer,
+        TelegrafConfig.Producer,
         ThreadpoolConfig.Producer,
         MetricsNodesConfig.Producer
 {
@@ -161,6 +163,25 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
     }
 
     @Override
+    public void getConfig(TelegrafConfig.Builder builder) {
+        var userConsumers = getUserMetricsConsumers();
+        for (var consumer : userConsumers.values()) {
+            for (var cloudWatch : consumer.cloudWatches()) {
+                var cloudWatchBuilder  = new TelegrafConfig.CloudWatch.Builder();
+                cloudWatchBuilder
+                        .region(cloudWatch.region())
+                        .namespace(cloudWatch.namespace())
+                        .consumer(cloudWatch.consumer());
+                cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder
+                        .accessKeyName(hostedAuth.accessKeyName)
+                        .secretKeyName(hostedAuth.secretKeyName));
+                cloudWatch.profile().ifPresent(cloudWatchBuilder::profile);
+                builder.cloudWatch(cloudWatchBuilder);
+            }
+        }
+    }
+
+    @Override
     public void getConfig(ThreadpoolConfig.Builder builder) {
         builder.maxthreads(10);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
new file mode 100644
index 00000000000..fd290409ea5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
@@ -0,0 +1,49 @@
+package com.yahoo.vespa.model.admin.monitoring;
+
+import java.util.Optional;
+
+/**
+ * Helper object for CloudWatch configuration.
+ *
+ * @author gjoranv
+ */
+public class CloudWatch {
+    private final String region;
+    private final String namespace;
+    private final MetricsConsumer consumer;
+
+    private HostedAuth hostedAuth;
+    private String profile;
+
+    public CloudWatch(String region, String namespace, MetricsConsumer consumer) {
+        this.region = region;
+        this.namespace = namespace;
+        this.consumer = consumer;
+    }
+
+    public String region() { return region; }
+    public String namespace() { return namespace; }
+    public String consumer() { return consumer.getId(); }
+
+    public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); }
+    public Optional<String> profile() { return Optional.ofNullable(profile); }
+
+    public void setHostedAuth(HostedAuth hostedAuth) {
+        this.hostedAuth = hostedAuth;
+    }
+
+    public void setProfile(String profile) {
+        this.profile = profile;
+    }
+
+    public static class HostedAuth {
+        public final String accessKeyName;
+        public final String secretKeyName;
+
+        public HostedAuth(String accessKeyName, String secretKeyName) {
+            this.accessKeyName = accessKeyName;
+            this.secretKeyName = secretKeyName;
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
index 529ed6ecf67..9c752f3aa0d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
@@ -2,9 +2,13 @@
 package com.yahoo.vespa.model.admin.monitoring;
 
 import javax.annotation.concurrent.Immutable;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Collections.unmodifiableList;
+
 /**
  * Represents an arbitrary metric consumer
  *
@@ -16,6 +20,8 @@ public class MetricsConsumer {
     private final String id;
     private final MetricSet metricSet;
 
+    private final List<CloudWatch> cloudWatches = new ArrayList<>();
+
     /**
      * @param id The consumer
      * @param metricSet  The metrics for this consumer
@@ -38,4 +44,12 @@ public class MetricsConsumer {
         return metricSet.getMetrics();
     }
 
+    public void addCloudWatch(CloudWatch cloudWatch) {
+        cloudWatches.add(cloudWatch);
+    }
+
+    public List<CloudWatch> cloudWatches() {
+        return unmodifiableList(cloudWatches);
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
new file mode 100644
index 00000000000..4b9d5542aa9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
@@ -0,0 +1,36 @@
+package com.yahoo.vespa.model.admin.monitoring.builder.xml;
+
+import com.yahoo.vespa.model.admin.monitoring.CloudWatch;
+import com.yahoo.vespa.model.admin.monitoring.CloudWatch.HostedAuth;
+import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
+import org.w3c.dom.Element;
+
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChildValue;
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchBuilder {
+
+    private static final String REGION_ATTRIBUTE = "region";
+    private static final String NAMESPACE_ATTRIBUTE = "namespace";
+    private static final String ACCESS_KEY_ELEMENT = "access-key-name";
+    private static final String SECRET_KEY_ELEMENT = "secret-key-name";
+    private static final String PROFILE_ELEMENT = "profile";
+
+    public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) {
+        CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE),
+                                               cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE),
+                                               consumer);
+
+        getOptionalChildValue(cloudwatchElement, PROFILE_ELEMENT).ifPresent(cloudWatch::setProfile);
+
+        getOptionalChildValue(cloudwatchElement, ACCESS_KEY_ELEMENT)
+                .ifPresent(accessKey -> cloudWatch.setHostedAuth(
+                        new HostedAuth(accessKey,
+                                       getOptionalChildValue(cloudwatchElement, SECRET_KEY_ELEMENT)
+                                               .orElseThrow(() -> new IllegalArgumentException("Access key given without a secret key.")))));
+        return cloudWatch;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
index f029dad01a9..b686288868f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
@@ -42,7 +42,11 @@ public class MetricsBuilder {
             throwIfIllegalConsumerId(metrics, consumerId);
 
             MetricSet metricSet = buildMetricSet(consumerId, consumerElement);
-            metrics.addConsumer(new MetricsConsumer(consumerId, metricSet));
+            var consumer = new MetricsConsumer(consumerId, metricSet);
+            for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) {
+                consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer));
+            }
+            metrics.addConsumer(consumer);
         }
         return metrics;
     }
@@ -58,7 +62,7 @@ public class MetricsBuilder {
 
     private MetricSet buildMetricSet(String consumerId, Element consumerElement) {
         List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream()
-                .map(metricElement -> metricFromElement(metricElement))
+                .map(MetricsBuilder::metricFromElement)
                 .collect(Collectors.toCollection(LinkedList::new));
 
         List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream()
diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc
index 7a3e2916f94..055f57dd7c0 100644
--- a/config-model/src/main/resources/schema/admin.rnc
+++ b/config-model/src/main/resources/schema/admin.rnc
@@ -82,10 +82,25 @@ Metrics = element metrics {
         element metric {
             attribute id { xsd:Name } &
             attribute display-name { xsd:Name }?
-        }*
+        }* &
+        Cloudwatch?
     }+
 }
 
+Cloudwatch = element cloudwatch {
+    attribute region { xsd:Name } &
+    attribute namespace { xsd:Name } &
+    (
+      (
+        element access-key-name { xsd:Name } &
+        element secret-key-name { xsd:Name }
+      )
+      |
+      element profile { xsd:Name }
+    )?
+
+}
+
 ClusterControllers = element cluster-controllers {
     attribute standalone-zookeeper { xsd:string  }? &
     element cluster-controller {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
index 9265e4437f1..8adca89d140 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
@@ -6,9 +6,9 @@
 package com.yahoo.vespa.model.admin.metricsproxy;
 
 import ai.vespa.metricsproxy.core.ConsumersConfig;
-import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
 import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler;
 import ai.vespa.metricsproxy.http.application.MetricsNodesConfig;
+import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
 import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler;
 import ai.vespa.metricsproxy.http.yamas.YamasHandler;
 import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
@@ -59,6 +59,7 @@ import static java.util.stream.Collectors.toList;
 import static org.hamcrest.CoreMatchers.endsWith;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
@@ -105,7 +106,7 @@ public class MetricsProxyContainerClusterTest {
         assertEquals(512, qrStartConfig.jvm().heapsize());
         assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
         assertEquals(2, qrStartConfig.jvm().availableProcessors());
-        assertEquals(false, qrStartConfig.jvm().verbosegc());
+        assertFalse(qrStartConfig.jvm().verbosegc());
         assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts());
         assertEquals(512, qrStartConfig.jvm().stacksize());
         assertEquals(0, qrStartConfig.jvm().directMemorySizeCache());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
new file mode 100644
index 00000000000..144c45a7dd2
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
@@ -0,0 +1,87 @@
+package com.yahoo.vespa.model.admin.metricsproxy;
+
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Test;
+
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author gjoranv
+ */
+public class TelegrafTest {
+
+    @Test
+    public void telegraf_config_is_generated_for_cloudwatch_in_services() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='cloudwatch-consumer'>",
+                                      "                <metric id='my-metric'/>",
+                                      "                <cloudwatch region='us-east-1' namespace='my-namespace' >",
+                                      "                    <access-key-name>my-access-key</access-key-name>",
+                                      "                    <secret-key-name>my-secret-key</secret-key-name>",
+                                      "                </cloudwatch>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        VespaModel hostedModel = getModel(services, hosted);
+        TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+        var cloudWatch0 = config.cloudWatch(0);
+        assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+        assertEquals("us-east-1", cloudWatch0.region());
+        assertEquals("my-namespace", cloudWatch0.namespace());
+        assertEquals("my-access-key", cloudWatch0.accessKeyName());
+        assertEquals("my-secret-key", cloudWatch0.secretKeyName());
+        assertEquals("", cloudWatch0.profile());
+    }
+
+    @Test
+    public void multiple_cloudwatches_are_allowed_for_the_same_consumer() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='cloudwatch-consumer'>",
+                                      "                <metric id='my-metric'/>",
+                                      "                <cloudwatch region='us-east-1' namespace='namespace-1' >",
+                                      "                    <access-key-name>access-key-1</access-key-name>",
+                                      "                    <secret-key-name>secret-key-1</secret-key-name>",
+                                      "                </cloudwatch>",
+                                      "                <cloudwatch region='us-east-1' namespace='namespace-2' >",
+                                      "                    <profile>profile-2</profile>",
+                                      "                </cloudwatch>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        VespaModel hostedModel = getModel(services, hosted);
+        TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+
+        var cloudWatch0 = config.cloudWatch(0);
+        assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+        assertEquals("us-east-1", cloudWatch0.region());
+        assertEquals("namespace-1", cloudWatch0.namespace());
+        assertEquals("access-key-1", cloudWatch0.accessKeyName());
+        assertEquals("secret-key-1", cloudWatch0.secretKeyName());
+        assertEquals("", cloudWatch0.profile());
+
+        var cloudWatch1 = config.cloudWatch(1);
+        assertEquals("cloudwatch-consumer", cloudWatch1.consumer());
+        assertEquals("us-east-1", cloudWatch1.region());
+        assertEquals("namespace-2", cloudWatch1.namespace());
+        assertEquals("", cloudWatch1.accessKeyName());
+        assertEquals("", cloudWatch1.secretKeyName());
+        assertEquals("profile-2", cloudWatch1.profile());
+    }
+
+}
diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml
index 1bf42650123..b06c93d6406 100644
--- a/config-model/src/test/schema-test-files/services.xml
+++ b/config-model/src/test/schema-test-files/services.xml
@@ -15,14 +15,25 @@
       <slobrok hostalias="rtc-1" />
     </slobroks>
     <metrics>
-      <consumer id="my-consumer">
+      <consumer id="cloudwatch-hosted">
         <metric-set id="my-set" />
         <metric id="my-metric"/>
         <metric id="my-metric2" display-name="my-metric3"/>
         <metric display-name="my-metric4" id="my-metric4.avg"/>
+        <cloudwatch region="us-east1" namespace="my-namespace">
+          <access-key-name>my-access-key</access-key-name>
+          <secret-key-name>my-secret-key</secret-key-name>
+        </cloudwatch>
       </consumer>
-      <consumer id="my-consumer2">
-        <metric-set id="my-set2" />
+      <consumer id="cloudwatch-self-hosted-with-default-auth">
+        <metric-set id="public" />
+        <cloudwatch region="us-east1" namespace="my-namespace" />
+      </consumer>
+      <consumer id="cloudwatch-self-hosted-with-profile">
+        <metric id="my-custom-metric" />
+        <cloudwatch region="us-east1" namespace="another-namespace">
+          <profile>profile-in-credentials-file</profile>
+        </cloudwatch>
       </consumer>
     </metrics>
     <logforwarding>