diff options
author | Morten Tokle <mortent@yahoo-inc.com> | 2017-03-21 10:52:16 +0100 |
---|---|---|
committer | Morten Tokle <mortent@yahoo-inc.com> | 2017-03-21 10:52:16 +0100 |
commit | b087651049343c1a0820f699a008cf4c037a5ace (patch) | |
tree | 804e7c3a2d464d4f1026836de041f1a351b598e3 /node-admin/src | |
parent | 128d08bc0ad644cf3683c613ea6134568bf5beee (diff) |
Move filebeat template
Diffstat (limited to 'node-admin/src')
3 files changed, 169 insertions, 177 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java index d038614b1ff..786bcce16a1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java @@ -3,10 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.logging; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.util.Environment; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -35,14 +31,14 @@ public class FilebeatConfigProvider { this.environment = environment; } - public Optional<String> getConfig(ContainerNodeSpec containerNodeSpec) throws IOException { + public Optional<String> getConfig(ContainerNodeSpec containerNodeSpec) { if (environment.getLogstashNodes().size() == 0 || !containerNodeSpec.owner.isPresent()) { return Optional.empty(); } ContainerNodeSpec.Owner owner = containerNodeSpec.owner.get(); int spoolSize = environment.getLogstashNodes().size() * logstashWorkers * logstashBulkMaxSize; - return Optional.of(readTemplate() + return Optional.of(getTemplate() .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment()) .replaceAll(REGION_FIELD, environment.getRegion()) .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize)) @@ -54,9 +50,165 @@ public class FilebeatConfigProvider { .replaceAll(INSTANCE_FIELD, owner.instance)); } - private String readTemplate() throws IOException { - String file = getClass().getClassLoader().getResource(TEMPLATE).getFile(); - List<String> lines = Files.readAllLines(Paths.get(file)); - return lines.stream().collect(Collectors.joining("\n")); + private static String getTemplate() { + return "################### Filebeat Configuration Example #########################\n" + + "\n" + + "############################# Filebeat ######################################\n" + + "filebeat:\n" + + " # List of prospectors to fetch data.\n" + + " prospectors:\n" + + "\n" + + " # vespa\n" + + " - paths:\n" + + " - /home/y/logs/vespa/vespa.log\n" + + " exclude_files: [\".gz$\"]\n" + + " document_type: vespa\n" + + " fields:\n" + + " HV-tenant: %%TENANT%%\n" + + " HV-application: %%APPLICATION%%\n" + + " HV-instance: %%INSTANCE%%\n" + + " HV-region: %%REGION%%\n" + + " HV-environment: %%ENVIRONMENT%%\n" + + " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + + " fields_under_root: true\n" + + " close_older: 20m\n" + + " force_close_files: true\n" + + "\n" + + " # vespa qrs\n" + + " - paths:\n" + + " - /home/y/logs/vespa/qrs/QueryAccessLog.*.*\n" + + " exclude_files: [\".gz$\"]\n" + + " exclude_lines: [\"reserved-for-internal-use/feedapi\"]\n" + + " document_type: vespa-qrs\n" + + " fields:\n" + + " HV-tenant: %%TENANT%%\n" + + " HV-application: %%APPLICATION%%\n" + + " HV-instance: %%INSTANCE%%\n" + + " HV-region: %%REGION%%\n" + + " HV-environment: %%ENVIRONMENT%%\n" + + " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + + " fields_under_root: true\n" + + " close_older: 20m\n" + + " force_close_files: true\n" + + "\n" + + " # General filebeat configuration options\n" + + " #\n" + + " # Event count spool threshold - forces network flush if exceeded\n" + + " spool_size: %%FILEBEAT_SPOOL_SIZE%%\n" + + "\n" + + " # Defines how often the spooler is flushed. After idle_timeout the spooler is\n" + + " # Flush even though spool_size is not reached.\n" + + " #idle_timeout: 5s\n" + + " publish_async: false\n" + + "\n" + + " # Name of the registry file. Per default it is put in the current working\n" + + " # directory. In case the working directory is changed after when running\n" + + " # filebeat again, indexing starts from the beginning again.\n" + + " registry_file: /var/lib/filebeat/registry\n" + + "\n" + + " # Full Path to directory with additional prospector configuration files. Each file must end with .yml\n" + + " # These config files must have the full filebeat config part inside, but only\n" + + " # the prospector part is processed. All global options like spool_size are ignored.\n" + + " # The config_dir MUST point to a different directory then where the main filebeat config file is in.\n" + + " #config_dir:\n" + + "\n" + + "###############################################################################\n" + + "############################# Libbeat Config ##################################\n" + + "# Base config file used by all other beats for using libbeat features\n" + + "\n" + + "############################# Output ##########################################\n" + + "\n" + + "# Configure what outputs to use when sending the data collected by the beat.\n" + + "# Multiple outputs may be used.\n" + + "output:\n" + + "\n" + + " ### Logstash as output\n" + + " logstash:\n" + + " # The Logstash hosts\n" + + " hosts: [%%LOGSTASH_HOSTS%%]\n" + + "\n" + + " timeout: 15\n" + + "\n" + + " # Number of workers per Logstash host.\n" + + " worker: %%LOGSTASH_WORKERS%%\n" + + "\n" + + " # Set gzip compression level.\n" + + " compression_level: 3\n" + + "\n" + + " # Optional load balance the events between the Logstash hosts\n" + + " loadbalance: true\n" + + "\n" + + " # Optional index name. The default index name depends on the each beat.\n" + + " # For Packetbeat, the default is set to packetbeat, for Topbeat\n" + + " # top topbeat and for Filebeat to filebeat.\n" + + " #index: filebeat\n" + + "\n" + + " bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%%\n" + + "\n" + + " # Optional TLS. By default is off.\n" + + " #tls:\n" + + " # List of root certificates for HTTPS server verifications\n" + + " #certificate_authorities: [\"/etc/pki/root/ca.pem\"]\n" + + "\n" + + " # Certificate for TLS client authentication\n" + + " #certificate: \"/etc/pki/client/cert.pem\"\n" + + "\n" + + " # Client Certificate Key\n" + + " #certificate_key: \"/etc/pki/client/cert.key\"\n" + + "\n" + + " # Controls whether the client verifies server certificates and host name.\n" + + " # If insecure is set to true, all server host names and certificates will be\n" + + " # accepted. In this mode TLS based connections are susceptible to\n" + + " # man-in-the-middle attacks. Use only for testing.\n" + + " #insecure: true\n" + + "\n" + + " # Configure cipher suites to be used for TLS connections\n" + + " #cipher_suites: []\n" + + "\n" + + " # Configure curve types for ECDHE based cipher suites\n" + + " #curve_types: []\n" + + "\n" + + "############################# Shipper #########################################\n" + + "\n" + + "shipper:\n" + + "\n" + + "############################# Logging #########################################\n" + + "\n" + + "# There are three options for the log ouput: syslog, file, stderr.\n" + + "# Under Windos systems, the log files are per default sent to the file output,\n" + + "# under all other system per default to syslog.\n" + + "logging:\n" + + "\n" + + " # Send all logging output to syslog. On Windows default is false, otherwise\n" + + " # default is true.\n" + + " to_syslog: false\n" + + "\n" + + " # Write all logging output to files. Beats automatically rotate files if rotateeverybytes\n" + + " # limit is reached.\n" + + " to_files: true\n" + + "\n" + + " # To enable logging to files, to_files option has to be set to true\n" + + " files:\n" + + " # The directory where the log files will written to.\n" + + " path: /home/y/logs/filebeat\n" + + "\n" + + " # The name of the files where the logs are written to.\n" + + " name: filebeat\n" + + "\n" + + " # Configure log file size limit. If limit is reached, log file will be\n" + + " # automatically rotated\n" + + " rotateeverybytes: 10485760 # = 10MB\n" + + "\n" + + " # Number of rotated log files to keep. Oldest files will be deleted first.\n" + + " keepfiles: 7\n" + + "\n" + + " # Enable debug output for selected components. To enable all selectors use [\"*\"]\n" + + " # Other available selectors are beat, publish, service\n" + + " # Multiple selectors can be chained.\n" + + " #selectors: [ ]\n" + + "\n" + + " # Sets log level. The default log level is error.\n" + + " # Available log levels are: critical, error, warning, info, debug\n" + + " level: warning\n"; } } diff --git a/node-admin/src/main/resources/filebeat.yml.template b/node-admin/src/main/resources/filebeat.yml.template deleted file mode 100644 index 7ab4aa95728..00000000000 --- a/node-admin/src/main/resources/filebeat.yml.template +++ /dev/null @@ -1,159 +0,0 @@ -################### Filebeat Configuration Example ######################### - -############################# Filebeat ###################################### -filebeat: - # List of prospectors to fetch data. - prospectors: - - # vespa - - paths: - - /home/y/logs/vespa/vespa.log - exclude_files: [".gz$"] - document_type: vespa - fields: - HV-tenant: %%TENANT%% - HV-application: %%APPLICATION%% - HV-instance: %%INSTANCE%% - HV-region: %%REGION%% - HV-environment: %%ENVIRONMENT%% - index_source: "hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%" - fields_under_root: true - close_older: 20m - force_close_files: true - - # vespa qrs - - paths: - - /home/y/logs/vespa/qrs/QueryAccessLog.*.* - exclude_files: [".gz$"] - exclude_lines: ["reserved-for-internal-use/feedapi"] - document_type: vespa-qrs - fields: - HV-tenant: %%TENANT%% - HV-application: %%APPLICATION%% - HV-instance: %%INSTANCE%% - HV-region: %%REGION%% - HV-environment: %%ENVIRONMENT%% - index_source: "hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%" - fields_under_root: true - close_older: 20m - force_close_files: true - - # General filebeat configuration options - # - # Event count spool threshold - forces network flush if exceeded - spool_size: %%FILEBEAT_SPOOL_SIZE%% - - # Defines how often the spooler is flushed. After idle_timeout the spooler is - # Flush even though spool_size is not reached. - #idle_timeout: 5s - publish_async: false - - # Name of the registry file. Per default it is put in the current working - # directory. In case the working directory is changed after when running - # filebeat again, indexing starts from the beginning again. - registry_file: /var/lib/filebeat/registry - - # Full Path to directory with additional prospector configuration files. Each file must end with .yml - # These config files must have the full filebeat config part inside, but only - # the prospector part is processed. All global options like spool_size are ignored. - # The config_dir MUST point to a different directory then where the main filebeat config file is in. - #config_dir: - -############################################################################### -############################# Libbeat Config ################################## -# Base config file used by all other beats for using libbeat features - -############################# Output ########################################## - -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. -output: - - ### Logstash as output - logstash: - # The Logstash hosts - hosts: [%%LOGSTASH_HOSTS%%] - - timeout: 15 - - # Number of workers per Logstash host. - worker: %%LOGSTASH_WORKERS%% - - # Set gzip compression level. - compression_level: 3 - - # Optional load balance the events between the Logstash hosts - loadbalance: true - - # Optional index name. The default index name depends on the each beat. - # For Packetbeat, the default is set to packetbeat, for Topbeat - # top topbeat and for Filebeat to filebeat. - #index: filebeat - - bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%% - - # Optional TLS. By default is off. - #tls: - # List of root certificates for HTTPS server verifications - #certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for TLS client authentication - #certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #certificate_key: "/etc/pki/client/cert.key" - - # Controls whether the client verifies server certificates and host name. - # If insecure is set to true, all server host names and certificates will be - # accepted. In this mode TLS based connections are susceptible to - # man-in-the-middle attacks. Use only for testing. - #insecure: true - - # Configure cipher suites to be used for TLS connections - #cipher_suites: [] - - # Configure curve types for ECDHE based cipher suites - #curve_types: [] - -############################# Shipper ######################################### - -shipper: - -############################# Logging ######################################### - -# There are three options for the log ouput: syslog, file, stderr. -# Under Windos systems, the log files are per default sent to the file output, -# under all other system per default to syslog. -logging: - - # Send all logging output to syslog. On Windows default is false, otherwise - # default is true. - to_syslog: false - - # Write all logging output to files. Beats automatically rotate files if rotateeverybytes - # limit is reached. - to_files: true - - # To enable logging to files, to_files option has to be set to true - files: - # The directory where the log files will written to. - path: /home/y/logs/filebeat - - # The name of the files where the logs are written to. - name: filebeat - - # Configure log file size limit. If limit is reached, log file will be - # automatically rotated - rotateeverybytes: 10485760 # = 10MB - - # Number of rotated log files to keep. Oldest files will be deleted first. - keepfiles: 7 - - # Enable debug output for selected components. To enable all selectors use ["*"] - # Other available selectors are beat, publish, service - # Multiple selectors can be chained. - #selectors: [ ] - - # Sets log level. The default log level is error. - # Available log levels are: critical, error, warning, info, debug - level: warning diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java index ba433051a9f..cc26eb288ae 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.hosted.node.admin.util.Environment; import com.yahoo.vespa.hosted.provision.Node; import org.junit.Test; -import java.io.IOException; import java.util.List; import java.util.Optional; @@ -28,7 +27,7 @@ public class FilebeatConfigProviderTest { private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2"); @Test - public void it_replaces_all_fields_correctly() throws IOException { + public void it_replaces_all_fields_correctly() { FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment()); Optional<String> config = filebeatConfigProvider.getConfig(getNodeSpec(tenant, application, instance)); @@ -39,7 +38,7 @@ public class FilebeatConfigProviderTest { } @Test - public void it_does_not_generate_config_when_no_logstash_nodes() throws IOException { + public void it_does_not_generate_config_when_no_logstash_nodes() { Environment env = new Environment.Builder() .environment(environment) .region(region) @@ -51,7 +50,7 @@ public class FilebeatConfigProviderTest { } @Test - public void it_does_not_generate_config_for_nodes_wihout_owner() throws IOException { + public void it_does_not_generate_config_for_nodes_wihout_owner() { FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment()); ContainerNodeSpec nodeSpec = new ContainerNodeSpec.Builder() .nodeFlavor("flavor") @@ -64,22 +63,22 @@ public class FilebeatConfigProviderTest { } @Test - public void it_generates_correct_index_source() throws IOException { + public void it_generates_correct_index_source() { assertThat(getConfigString(), containsString("index_source: \"hosted-instance_vespa_music_us-north-1_prod_default\"")); } @Test - public void it_sets_logstash_nodes_properly() throws IOException { + public void it_sets_logstash_nodes_properly() { assertThat(getConfigString(), containsString("hosts: [logstash1,logstash2]")); } @Test - public void it_generates_correct_spool_size() throws IOException { + public void it_generates_correct_spool_size() { // 2 nodes, 3 workers, 2048 buffer size -> 12288 assertThat(getConfigString(), containsString("spool_size: 12288")); } - private String getConfigString() throws IOException { + private String getConfigString() { FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment()); ContainerNodeSpec nodeSpec = getNodeSpec(tenant, application, instance); return filebeatConfigProvider.getConfig(nodeSpec).orElseThrow(() -> new RuntimeException("Failed to get filebeat config")); |