Merge pull request #12346 from vespa-engine/gjoranv/add-telegraf-component

Add Telegraf components to metrics-proxy clusters

3 files changed, 72 insertions, 11 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index f81757ac568..b30cf9248a1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -20,7 +20,9 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
 import ai.vespa.metricsproxy.rpc.RpcServer;
 import ai.vespa.metricsproxy.service.ConfigSentinelClient;
 import ai.vespa.metricsproxy.service.SystemPollerProvider;
+import ai.vespa.metricsproxy.telegraf.Telegraf;
 import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import ai.vespa.metricsproxy.telegraf.TelegrafRegistry;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
@@ -118,6 +120,8 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
 
         addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.V1_PATH);
         addMetricsProxyComponent(ApplicationMetricsRetriever.class);
+
+        addTelegrafComponents();
     }
 
     private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) {
@@ -133,6 +137,15 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
         return metricsHandler;
     }
 
+    private void addTelegrafComponents() {
+        getAdmin().ifPresent(admin -> {
+            if (admin.getUserMetrics().usesExternalMetricSystems()) {
+                addMetricsProxyComponent(Telegraf.class);
+                addMetricsProxyComponent(TelegrafRegistry.class);
+            }
+        });
+    }
+
     @Override
     protected void doPrepare(DeployState deployState) { }
 
@@ -224,7 +237,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
                 Optional.of(monitoring.getInterval()) : Optional.empty();
     }
 
-    private void  addMetricsProxyComponent(Class<?> componentClass) {
+    private void addMetricsProxyComponent(Class<?> componentClass) {
         addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
index 9b0d9dbfadc..1f81f16a80b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
@@ -28,4 +28,13 @@ public class Metrics {
         return consumers.keySet().stream()
                 .anyMatch(existing -> existing.equalsIgnoreCase(id));
     }
+
+    /**
+     * Returns true if any of the consumers have specified external metric systems.
+     */
+    public boolean usesExternalMetricSystems() {
+        return consumers.values().stream()
+                .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty());
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
index 144c45a7dd2..dbcbad7bf49 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
@@ -1,13 +1,19 @@
 package com.yahoo.vespa.model.admin.metricsproxy;
 
+import ai.vespa.metricsproxy.telegraf.Telegraf;
 import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import ai.vespa.metricsproxy.telegraf.TelegrafRegistry;
+import com.yahoo.component.ComponentId;
 import com.yahoo.vespa.model.VespaModel;
 import org.junit.Test;
 
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.not;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
 
 /**
  * @author gjoranv
@@ -15,25 +21,39 @@ import static org.junit.Assert.assertEquals;
 public class TelegrafTest {
 
     @Test
-    public void telegraf_config_is_generated_for_cloudwatch_in_services() {
+    public void telegraf_components_are_set_up_when_cloudwatch_is_configured() {
+        String services = servicesWithCloudwatch();
+        VespaModel hostedModel = getModel(services, hosted);
+
+        var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap();
+        assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(Telegraf.class.getName())));
+        assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(TelegrafRegistry.class.getName())));
+    }
+
+    @Test
+    public void telegraf_components_are_not_set_up_when_no_external_systems_are_added_in_services() {
         String services = String.join("\n",
                                       "<services>",
                                       "    <admin version='2.0'>",
                                       "        <adminserver hostalias='node1'/>",
                                       "        <metrics>",
-                                      "            <consumer id='cloudwatch-consumer'>",
-                                      "                <metric id='my-metric'/>",
-                                      "                <cloudwatch region='us-east-1' namespace='my-namespace' >",
-                                      "                    <access-key-name>my-access-key</access-key-name>",
-                                      "                    <secret-key-name>my-secret-key</secret-key-name>",
-                                      "                </cloudwatch>",
-                                      "            </consumer>",
+                                      "            <consumer id='foo' />",
                                       "        </metrics>",
                                       "    </admin>",
-                                      "</services>"
-        );
+                                      "</services>");
+        VespaModel hostedModel = getModel(services, hosted);
+
+        var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap();
+        assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(Telegraf.class.getName()))));
+        assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(TelegrafRegistry.class.getName()))));
+    }
+
+    @Test
+    public void telegraf_config_is_generated_for_cloudwatch_in_services() {
+        String services = servicesWithCloudwatch();
         VespaModel hostedModel = getModel(services, hosted);
         TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+
         var cloudWatch0 = config.cloudWatch(0);
         assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
         assertEquals("us-east-1", cloudWatch0.region());
@@ -43,6 +63,25 @@ public class TelegrafTest {
         assertEquals("", cloudWatch0.profile());
     }
 
+    private String servicesWithCloudwatch() {
+        return String.join("\n",
+                           "<services>",
+                           "    <admin version='2.0'>",
+                           "        <adminserver hostalias='node1'/>",
+                           "        <metrics>",
+                           "            <consumer id='cloudwatch-consumer'>",
+                           "                <metric id='my-metric'/>",
+                           "                <cloudwatch region='us-east-1' namespace='my-namespace' >",
+                           "                    <access-key-name>my-access-key</access-key-name>",
+                           "                    <secret-key-name>my-secret-key</secret-key-name>",
+                           "                </cloudwatch>",
+                           "            </consumer>",
+                           "        </metrics>",
+                           "    </admin>",
+                           "</services>"
+        );
+    }
+
     @Test
     public void multiple_cloudwatches_are_allowed_for_the_same_consumer() {
         String services = String.join("\n",