diff options
author | Ola Aunronning <olaa@yahooinc.com> | 2023-03-10 10:15:32 +0100 |
---|---|---|
committer | Ola Aunronning <olaa@yahooinc.com> | 2023-03-10 10:15:32 +0100 |
commit | bc8ce77595e7a2224bdfd973759bffa103f4a853 (patch) | |
tree | 53af09d801cc67537aa3667456e1e5a2e0d800ab /metrics-proxy | |
parent | 88bef544c54e3784e80dbcb8270dd2857d8a6559 (diff) |
Remove telegraf metrics-proxy integration
Diffstat (limited to 'metrics-proxy')
9 files changed, 0 insertions, 490 deletions
diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt index 4cc59300058..21889d83da5 100644 --- a/metrics-proxy/CMakeLists.txt +++ b/metrics-proxy/CMakeLists.txt @@ -1,9 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_jar(metrics-proxy-jar-with-dependencies.jar) -install(DIRECTORY DESTINATION conf/telegraf) -install(DIRECTORY DESTINATION logs/telegraf) -vespa_install_script(src/main/sh/start-telegraf.sh libexec/vespa) -vespa_install_script(src/main/sh/stop-telegraf.sh libexec/vespa) - install_config_definitions() diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java deleted file mode 100644 index aa35832a3be..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.telegraf; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.component.AbstractComponent; - -import java.io.File; -import java.util.logging.Level; -import com.yahoo.system.execution.ProcessExecutor; -import com.yahoo.system.execution.ProcessResult; -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.VelocityEngine; - -import java.io.FileWriter; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.Writer; -import java.util.logging.Logger; - -import static com.yahoo.vespa.defaults.Defaults.getDefaults; -import static com.yahoo.yolean.Exceptions.uncheck; - -/** - * @author olaa - */ -public class Telegraf extends AbstractComponent { - - // These paths must coincide with the paths in the start/stop-telegraf shell scripts. - private static final String TELEGRAF_CONFIG_PATH = getDefaults().underVespaHome("conf/telegraf/telegraf.conf"); - private static final String TELEGRAF_LOG_FILE_PATH = getDefaults().underVespaHome("logs/telegraf/telegraf.log"); - - private static final String START_TELEGRAF_SCRIPT = getDefaults().underVespaHome("libexec/vespa/start-telegraf.sh"); - private static final String STOP_TELEGRAF_SCRIPT = getDefaults().underVespaHome("libexec/vespa/stop-telegraf.sh"); - - private static final String TELEGRAF_CONFIG_TEMPLATE_PATH = "templates/telegraf.conf.vm"; - - private final TelegrafRegistry telegrafRegistry; - - private static final Logger logger = Logger.getLogger(Telegraf.class.getName()); - - @Inject - public Telegraf(TelegrafRegistry telegrafRegistry, TelegrafConfig telegrafConfig) { - this.telegrafRegistry = telegrafRegistry; - telegrafRegistry.addInstance(this); - writeConfig(telegrafConfig, getConfigWriter(), TELEGRAF_LOG_FILE_PATH); - restartTelegraf(); - } - - protected static void writeConfig(TelegrafConfig telegrafConfig, Writer writer, String logFilePath) { - VelocityContext context = new VelocityContext(); - context.put("logFilePath", logFilePath); - context.put("intervalSeconds", telegrafConfig.intervalSeconds()); - context.put("cloudwatchPlugins", telegrafConfig.cloudWatch()); - context.put("protocol", telegrafConfig.isHostedVespa() ? "https" : "http"); - // TODO: Add node cert if hosted - - VelocityEngine velocityEngine = new VelocityEngine(); - velocityEngine.init(); - velocityEngine.evaluate(context, writer, "TelegrafConfigWriter", getTemplateReader()); - uncheck(writer::close); - } - - private void restartTelegraf() { - executeCommand(STOP_TELEGRAF_SCRIPT); - executeCommand(START_TELEGRAF_SCRIPT); - } - - private void stopTelegraf() { - executeCommand(STOP_TELEGRAF_SCRIPT); - } - - private void executeCommand(String command) { - logger.info(String.format("Running command: %s", command)); - ProcessExecutor processExecutor = new ProcessExecutor - .Builder(10) - .successExitCodes(0) - .build(); - ProcessResult processResult = uncheck(() -> processExecutor.execute(command)) - .orElseThrow(() -> new RuntimeException("Timed out running command: " + command)); - - logger.log(Level.FINE, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s", - processResult.exitCode, - processResult.stdOut, - processResult.stdErr)); - - if (!processResult.stdErr.isBlank()) - logger.warning(String.format("stdErr not empty: %s", processResult.stdErr)); - } - - @SuppressWarnings("ConstantConditions") - private static Reader getTemplateReader() { - return new InputStreamReader(Telegraf.class.getClassLoader() - .getResourceAsStream(TELEGRAF_CONFIG_TEMPLATE_PATH) - ); - - } - - private static Writer getConfigWriter() { - File configFile = new File(TELEGRAF_CONFIG_PATH); - configFile.getParentFile().mkdirs(); - return uncheck(() -> new FileWriter(configFile)); - } - - @Override - public void deconstruct() { - telegrafRegistry.removeInstance(this); - if (telegrafRegistry.isEmpty()) { - stopTelegraf(); - } - } -} diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java deleted file mode 100644 index 2129ae38f89..00000000000 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.telegraf; - -import java.util.logging.Level; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.Logger; - -/** - * @author olaa - */ -public class TelegrafRegistry { - - private static final List<Telegraf> telegrafInstances = Collections.synchronizedList(new ArrayList<>()); - - private static final Logger logger = Logger.getLogger(TelegrafRegistry.class.getName()); - - public void addInstance(Telegraf telegraf) { - logger.log(Level.FINE, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode()); - telegrafInstances.add(telegraf); - } - - public void removeInstance(Telegraf telegraf) { - logger.log(Level.FINE, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode()); - telegrafInstances.remove(telegraf); - } - - public boolean isEmpty() { - return telegrafInstances.isEmpty(); - } -} diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def deleted file mode 100644 index d92997879aa..00000000000 --- a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package=ai.vespa.metricsproxy.telegraf - -isHostedVespa bool default=false - -# Metrics pull/push interval -intervalSeconds int default=60 - - -# The Vespa metrics consumer to get metrics for -cloudWatch[].consumer string - -cloudWatch[].region string default="us-east-1" -cloudWatch[].namespace string - -# Only valid and required for hosted Vespa -cloudWatch[].accessKeyName string default="" -cloudWatch[].secretKeyName string default="" - -# Only valid and optional for self-hosted Vespa -cloudWatch[].file string default="" -cloudWatch[].profile string default="default" diff --git a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm deleted file mode 100644 index 5a5f2d5f712..00000000000 --- a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm +++ /dev/null @@ -1,44 +0,0 @@ -# Configuration for telegraf agent -[agent] - interval = "${intervalSeconds}s" - round_interval = true - metric_batch_size = 1000 - metric_buffer_limit = 10000 - collection_jitter = "0s" - flush_interval = "${intervalSeconds}s" - flush_jitter = "0s" - precision = "" - logtarget = "file" - logfile = "$logFilePath" - logfile_rotation_interval = "1d" - logfile_rotation_max_size = "20MB" - logfile_rotation_max_archives = 5 - -#foreach( $cloudwatch in $cloudwatchPlugins ) -# Configuration for AWS CloudWatch output. -[[outputs.cloudwatch]] - region = "$cloudwatch.region()" - namespace = "$cloudwatch.namespace()" -#if( $cloudwatch.accessKeyName() != "" ) - access_key = "$cloudwatch.accessKeyName()" - secret_key = "$cloudwatch.secretKeyName()" -#elseif( $cloudwatch.profile() != "" ) - profile = "$cloudwatch.profile()" -#end - tagexclude = ["vespa_consumer"] - [outputs.cloudwatch.tagpass] - vespa_consumer = ["$cloudwatch.consumer()"] - -# Configuration for Vespa input plugin -[[inputs.vespa]] - url = "${protocol}://localhost:19092/metrics/v2/values?consumer=$cloudwatch.consumer()" - [inputs.vespa.tags] - vespa_consumer = "$cloudwatch.consumer()" -#* TODO: Add node cert if hosted -#if( $isHosted ) - tls_cert = "${VESPA_CERTIFICATE_PATH}" - tls_key = "${VESPA_KEY_PATH}" - insecure_skip_verify = true -#end -*### -#end
\ No newline at end of file diff --git a/metrics-proxy/src/main/sh/start-telegraf.sh b/metrics-proxy/src/main/sh/start-telegraf.sh deleted file mode 100644 index 60677e9f065..00000000000 --- a/metrics-proxy/src/main/sh/start-telegraf.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/sh -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -findpath () { - myname=${0} - mypath=${myname%/*} - myname=${myname##*/} - empty_if_start_slash=${mypath%%/*} - if [ "${empty_if_start_slash}" ]; then - mypath=$(pwd)/${mypath} - fi - if [ "$mypath" ] && [ -d "$mypath" ]; then - return - fi - mypath=$(pwd) - if [ -f "${mypath}/${myname}" ]; then - return - fi - echo "FATAL: Could not figure out the path where $myname lives from $0" - exit 1 -} - -COMMON_ENV=libexec/vespa/common-env.sh - -source_common_env () { - if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then - export VESPA_HOME - common_env=$VESPA_HOME/$COMMON_ENV - if [ -f "$common_env" ]; then - . $common_env - return - fi - fi - return 1 -} - -findroot () { - source_common_env && return - if [ "$VESPA_HOME" ]; then - echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" - exit 1 - fi - if [ "$ROOT" ] && [ -d "$ROOT" ]; then - VESPA_HOME="$ROOT" - source_common_env && return - fi - findpath - while [ "$mypath" ]; do - VESPA_HOME=${mypath} - source_common_env && return - mypath=${mypath%/*} - done - echo "FATAL: missing VESPA_HOME environment variable" - echo "Could not locate $COMMON_ENV anywhere" - exit 1 -} - -findhost () { - if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 - fi - validate="${VESPA_HOME}/bin/vespa-validate-hostname" - if [ -f "$validate" ]; then - "$validate" "${VESPA_HOSTNAME}" || exit 1 - fi - export VESPA_HOSTNAME -} - -findroot -findhost - -ROOT=${VESPA_HOME%/} -export ROOT - -# END environment bootstrap section - -fixddir () { - if ! [ -d $1 ]; then - echo "Creating data directory $1" - mkdir -p $1 || exit 1 - fi - if [ "${VESPA_USER}" ] && [ "$(id -u)" -eq 0 ]; then - chown ${VESPA_USER} $1 - fi - chmod 755 $1 -} - -# Note: these directories must coincide with the paths defined in the Telegraf Java component -conf_dir=${VESPA_HOME}/conf/telegraf -log_dir=${VESPA_HOME}/logs/telegraf -fixddir ${conf_dir} -fixddir ${log_dir} - -configfile=${conf_dir}/telegraf.conf -pidfile="${VESPA_HOME}/var/run/telegraf.pid" - -TELEGRAF_CMD=/opt/vespa-deps/bin/telegraf - -vespa-run-as-vespa-user vespa-runserver -s telegraf -r 30 -p $pidfile -- \ -${TELEGRAF_CMD} --config ${configfile} - diff --git a/metrics-proxy/src/main/sh/stop-telegraf.sh b/metrics-proxy/src/main/sh/stop-telegraf.sh deleted file mode 100644 index 595e4068ee5..00000000000 --- a/metrics-proxy/src/main/sh/stop-telegraf.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/sh -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -findpath () { - myname=${0} - mypath=${myname%/*} - myname=${myname##*/} - empty_if_start_slash=${mypath%%/*} - if [ "${empty_if_start_slash}" ]; then - mypath=$(pwd)/${mypath} - fi - if [ "$mypath" ] && [ -d "$mypath" ]; then - return - fi - mypath=$(pwd) - if [ -f "${mypath}/${myname}" ]; then - return - fi - echo "FATAL: Could not figure out the path where $myname lives from $0" - exit 1 -} - -COMMON_ENV=libexec/vespa/common-env.sh - -source_common_env () { - if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then - export VESPA_HOME - common_env=$VESPA_HOME/$COMMON_ENV - if [ -f "$common_env" ]; then - . $common_env - return - fi - fi - return 1 -} - -findroot () { - source_common_env && return - if [ "$VESPA_HOME" ]; then - echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'" - exit 1 - fi - if [ "$ROOT" ] && [ -d "$ROOT" ]; then - VESPA_HOME="$ROOT" - source_common_env && return - fi - findpath - while [ "$mypath" ]; do - VESPA_HOME=${mypath} - source_common_env && return - mypath=${mypath%/*} - done - echo "FATAL: missing VESPA_HOME environment variable" - echo "Could not locate $COMMON_ENV anywhere" - exit 1 -} - -findhost () { - if [ "${VESPA_HOSTNAME}" = "" ]; then - VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1 - fi - validate="${VESPA_HOME}/bin/vespa-validate-hostname" - if [ -f "$validate" ]; then - "$validate" "${VESPA_HOSTNAME}" || exit 1 - fi - export VESPA_HOSTNAME -} - -findroot -findhost - -ROOT=${VESPA_HOME%/} -export ROOT - -# END environment bootstrap section - -pidfile="${VESPA_HOME}/var/run/telegraf.pid" -vespa-run-as-vespa-user vespa-runserver -s telegraf -p $pidfile -S diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java deleted file mode 100644 index 5f13561a332..00000000000 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.metricsproxy.telegraf; - -import ai.vespa.metricsproxy.TestUtil; -import org.junit.Test; - -import java.io.StringWriter; - -import static org.junit.Assert.*; - -/** - * @author olaa - */ -public class TelegrafTest { - - @Test - public void test_writing_correct_telegraf_plugin_config() { - TelegrafConfig telegrafConfig = new TelegrafConfig.Builder() - .cloudWatch( - new TelegrafConfig.CloudWatch.Builder() - .accessKeyName("accessKey1") - .namespace("namespace1") - .secretKeyName("secretKey1") - .region("us-east-1") - .consumer("consumer1") - ) - .cloudWatch( - new TelegrafConfig.CloudWatch.Builder() - .namespace("namespace2") - .profile("awsprofile") - .region("us-east-2") - .consumer("consumer2") - ) - .intervalSeconds(300) - .isHostedVespa(true) - .build(); - StringWriter stringWriter = new StringWriter(); - String logFilePath = "/path/to/logs/telegraf/telegraf.log"; - Telegraf.writeConfig(telegrafConfig, stringWriter, logFilePath); - String expectedConfig = TestUtil.getFileContents( "telegraf-config-with-two-cloudwatch-plugins.txt"); - assertEquals(expectedConfig, stringWriter.toString()); - } - -} diff --git a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt deleted file mode 100644 index 3194b290b78..00000000000 --- a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt +++ /dev/null @@ -1,46 +0,0 @@ -# Configuration for telegraf agent -[agent] - interval = "300s" - round_interval = true - metric_batch_size = 1000 - metric_buffer_limit = 10000 - collection_jitter = "0s" - flush_interval = "300s" - flush_jitter = "0s" - precision = "" - logtarget = "file" - logfile = "/path/to/logs/telegraf/telegraf.log" - logfile_rotation_interval = "1d" - logfile_rotation_max_size = "20MB" - logfile_rotation_max_archives = 5 - -# Configuration for AWS CloudWatch output. -[[outputs.cloudwatch]] - region = "us-east-1" - namespace = "namespace1" - access_key = "accessKey1" - secret_key = "secretKey1" - tagexclude = ["vespa_consumer"] - [outputs.cloudwatch.tagpass] - vespa_consumer = ["consumer1"] - -# Configuration for Vespa input plugin -[[inputs.vespa]] - url = "https://localhost:19092/metrics/v2/values?consumer=consumer1" - [inputs.vespa.tags] - vespa_consumer = "consumer1" -# Configuration for AWS CloudWatch output. -[[outputs.cloudwatch]] - region = "us-east-2" - namespace = "namespace2" - profile = "awsprofile" - tagexclude = ["vespa_consumer"] - [outputs.cloudwatch.tagpass] - vespa_consumer = ["consumer2"] - -# Configuration for Vespa input plugin -[[inputs.vespa]] - url = "https://localhost:19092/metrics/v2/values?consumer=consumer2" - [inputs.vespa.tags] - vespa_consumer = "consumer2" - |