Move filebeat template

3 files changed, 169 insertions, 177 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
index d038614b1ff..786bcce16a1 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java
@@ -3,10 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.logging;
 import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec;
 import com.yahoo.vespa.hosted.node.admin.util.Environment;
 
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
@@ -35,14 +31,14 @@ public class FilebeatConfigProvider {
         this.environment = environment;
     }
 
-    public Optional<String> getConfig(ContainerNodeSpec containerNodeSpec) throws IOException {
+    public Optional<String> getConfig(ContainerNodeSpec containerNodeSpec) {
 
         if (environment.getLogstashNodes().size() == 0 || !containerNodeSpec.owner.isPresent()) {
             return Optional.empty();
         }
         ContainerNodeSpec.Owner owner = containerNodeSpec.owner.get();
         int spoolSize = environment.getLogstashNodes().size() * logstashWorkers * logstashBulkMaxSize;
-        return Optional.of(readTemplate()
+        return Optional.of(getTemplate()
                 .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment())
                 .replaceAll(REGION_FIELD, environment.getRegion())
                 .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize))
@@ -54,9 +50,165 @@ public class FilebeatConfigProvider {
                 .replaceAll(INSTANCE_FIELD, owner.instance));
     }
 
-    private String readTemplate() throws IOException {
-        String file = getClass().getClassLoader().getResource(TEMPLATE).getFile();
-        List<String> lines = Files.readAllLines(Paths.get(file));
-        return lines.stream().collect(Collectors.joining("\n"));
+    private static String getTemplate() {
+        return "################### Filebeat Configuration Example #########################\n" +
+                "\n" +
+                "############################# Filebeat ######################################\n" +
+                "filebeat:\n" +
+                "  # List of prospectors to fetch data.\n" +
+                "  prospectors:\n" +
+                "\n" +
+                "    # vespa\n" +
+                "    - paths:\n" +
+                "        - /home/y/logs/vespa/vespa.log\n" +
+                "      exclude_files: [\".gz$\"]\n" +
+                "      document_type: vespa\n" +
+                "      fields:\n" +
+                "        HV-tenant: %%TENANT%%\n" +
+                "        HV-application: %%APPLICATION%%\n" +
+                "        HV-instance: %%INSTANCE%%\n" +
+                "        HV-region: %%REGION%%\n" +
+                "        HV-environment: %%ENVIRONMENT%%\n" +
+                "        index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" +
+                "      fields_under_root: true\n" +
+                "      close_older: 20m\n" +
+                "      force_close_files: true\n" +
+                "\n" +
+                "    # vespa qrs\n" +
+                "    - paths:\n" +
+                "        - /home/y/logs/vespa/qrs/QueryAccessLog.*.*\n" +
+                "      exclude_files: [\".gz$\"]\n" +
+                "      exclude_lines: [\"reserved-for-internal-use/feedapi\"]\n" +
+                "      document_type: vespa-qrs\n" +
+                "      fields:\n" +
+                "        HV-tenant: %%TENANT%%\n" +
+                "        HV-application: %%APPLICATION%%\n" +
+                "        HV-instance: %%INSTANCE%%\n" +
+                "        HV-region: %%REGION%%\n" +
+                "        HV-environment: %%ENVIRONMENT%%\n" +
+                "        index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" +
+                "      fields_under_root: true\n" +
+                "      close_older: 20m\n" +
+                "      force_close_files: true\n" +
+                "\n" +
+                "  # General filebeat configuration options\n" +
+                "  #\n" +
+                "  # Event count spool threshold - forces network flush if exceeded\n" +
+                "  spool_size: %%FILEBEAT_SPOOL_SIZE%%\n" +
+                "\n" +
+                "  # Defines how often the spooler is flushed. After idle_timeout the spooler is\n" +
+                "  # Flush even though spool_size is not reached.\n" +
+                "  #idle_timeout: 5s\n" +
+                "  publish_async: false\n" +
+                "\n" +
+                "  # Name of the registry file. Per default it is put in the current working\n" +
+                "  # directory. In case the working directory is changed after when running\n" +
+                "  # filebeat again, indexing starts from the beginning again.\n" +
+                "  registry_file: /var/lib/filebeat/registry\n" +
+                "\n" +
+                "  # Full Path to directory with additional prospector configuration files. Each file must end with .yml\n" +
+                "  # These config files must have the full filebeat config part inside, but only\n" +
+                "  # the prospector part is processed. All global options like spool_size are ignored.\n" +
+                "  # The config_dir MUST point to a different directory then where the main filebeat config file is in.\n" +
+                "  #config_dir:\n" +
+                "\n" +
+                "###############################################################################\n" +
+                "############################# Libbeat Config ##################################\n" +
+                "# Base config file used by all other beats for using libbeat features\n" +
+                "\n" +
+                "############################# Output ##########################################\n" +
+                "\n" +
+                "# Configure what outputs to use when sending the data collected by the beat.\n" +
+                "# Multiple outputs may be used.\n" +
+                "output:\n" +
+                "\n" +
+                "  ### Logstash as output\n" +
+                "  logstash:\n" +
+                "    # The Logstash hosts\n" +
+                "    hosts: [%%LOGSTASH_HOSTS%%]\n" +
+                "\n" +
+                "    timeout: 15\n" +
+                "\n" +
+                "    # Number of workers per Logstash host.\n" +
+                "    worker: %%LOGSTASH_WORKERS%%\n" +
+                "\n" +
+                "    # Set gzip compression level.\n" +
+                "    compression_level: 3\n" +
+                "\n" +
+                "    # Optional load balance the events between the Logstash hosts\n" +
+                "    loadbalance: true\n" +
+                "\n" +
+                "    # Optional index name. The default index name depends on the each beat.\n" +
+                "    # For Packetbeat, the default is set to packetbeat, for Topbeat\n" +
+                "    # top topbeat and for Filebeat to filebeat.\n" +
+                "    #index: filebeat\n" +
+                "\n" +
+                "    bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%%\n" +
+                "\n" +
+                "    # Optional TLS. By default is off.\n" +
+                "    #tls:\n" +
+                "      # List of root certificates for HTTPS server verifications\n" +
+                "      #certificate_authorities: [\"/etc/pki/root/ca.pem\"]\n" +
+                "\n" +
+                "      # Certificate for TLS client authentication\n" +
+                "      #certificate: \"/etc/pki/client/cert.pem\"\n" +
+                "\n" +
+                "      # Client Certificate Key\n" +
+                "      #certificate_key: \"/etc/pki/client/cert.key\"\n" +
+                "\n" +
+                "      # Controls whether the client verifies server certificates and host name.\n" +
+                "      # If insecure is set to true, all server host names and certificates will be\n" +
+                "      # accepted. In this mode TLS based connections are susceptible to\n" +
+                "      # man-in-the-middle attacks. Use only for testing.\n" +
+                "      #insecure: true\n" +
+                "\n" +
+                "      # Configure cipher suites to be used for TLS connections\n" +
+                "      #cipher_suites: []\n" +
+                "\n" +
+                "      # Configure curve types for ECDHE based cipher suites\n" +
+                "      #curve_types: []\n" +
+                "\n" +
+                "############################# Shipper #########################################\n" +
+                "\n" +
+                "shipper:\n" +
+                "\n" +
+                "############################# Logging #########################################\n" +
+                "\n" +
+                "# There are three options for the log ouput: syslog, file, stderr.\n" +
+                "# Under Windos systems, the log files are per default sent to the file output,\n" +
+                "# under all other system per default to syslog.\n" +
+                "logging:\n" +
+                "\n" +
+                "  # Send all logging output to syslog. On Windows default is false, otherwise\n" +
+                "  # default is true.\n" +
+                "  to_syslog: false\n" +
+                "\n" +
+                "  # Write all logging output to files. Beats automatically rotate files if rotateeverybytes\n" +
+                "  # limit is reached.\n" +
+                "  to_files: true\n" +
+                "\n" +
+                "  # To enable logging to files, to_files option has to be set to true\n" +
+                "  files:\n" +
+                "    # The directory where the log files will written to.\n" +
+                "    path: /home/y/logs/filebeat\n" +
+                "\n" +
+                "    # The name of the files where the logs are written to.\n" +
+                "    name: filebeat\n" +
+                "\n" +
+                "    # Configure log file size limit. If limit is reached, log file will be\n" +
+                "    # automatically rotated\n" +
+                "    rotateeverybytes: 10485760 # = 10MB\n" +
+                "\n" +
+                "    # Number of rotated log files to keep. Oldest files will be deleted first.\n" +
+                "    keepfiles: 7\n" +
+                "\n" +
+                "  # Enable debug output for selected components. To enable all selectors use [\"*\"]\n" +
+                "  # Other available selectors are beat, publish, service\n" +
+                "  # Multiple selectors can be chained.\n" +
+                "  #selectors: [ ]\n" +
+                "\n" +
+                "  # Sets log level. The default log level is error.\n" +
+                "  # Available log levels are: critical, error, warning, info, debug\n" +
+                "  level: warning\n";
     }
 }
diff --git a/node-admin/src/main/resources/filebeat.yml.template b/node-admin/src/main/resources/filebeat.yml.template
deleted file mode 100644
index 7ab4aa95728..00000000000
--- a/node-admin/src/main/resources/filebeat.yml.template
+++ /dev/null
@@ -1,159 +0,0 @@
-################### Filebeat Configuration Example #########################
-
-############################# Filebeat ######################################
-filebeat:
-  # List of prospectors to fetch data.
-  prospectors:
-
-    # vespa
-    - paths:
-        - /home/y/logs/vespa/vespa.log
-      exclude_files: [".gz$"]
-      document_type: vespa
-      fields:
-        HV-tenant: %%TENANT%%
-        HV-application: %%APPLICATION%%
-        HV-instance: %%INSTANCE%%
-        HV-region: %%REGION%%
-        HV-environment: %%ENVIRONMENT%%
-        index_source: "hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%"
-      fields_under_root: true
-      close_older: 20m
-      force_close_files: true
-
-    # vespa qrs
-    - paths:
-        - /home/y/logs/vespa/qrs/QueryAccessLog.*.*
-      exclude_files: [".gz$"]
-      exclude_lines: ["reserved-for-internal-use/feedapi"]
-      document_type: vespa-qrs
-      fields:
-        HV-tenant: %%TENANT%%
-        HV-application: %%APPLICATION%%
-        HV-instance: %%INSTANCE%%
-        HV-region: %%REGION%%
-        HV-environment: %%ENVIRONMENT%%
-        index_source: "hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%"
-      fields_under_root: true
-      close_older: 20m
-      force_close_files: true
-
-  # General filebeat configuration options
-  #
-  # Event count spool threshold - forces network flush if exceeded
-  spool_size: %%FILEBEAT_SPOOL_SIZE%%
-
-  # Defines how often the spooler is flushed. After idle_timeout the spooler is
-  # Flush even though spool_size is not reached.
-  #idle_timeout: 5s
-  publish_async: false
-
-  # Name of the registry file. Per default it is put in the current working
-  # directory. In case the working directory is changed after when running
-  # filebeat again, indexing starts from the beginning again.
-  registry_file: /var/lib/filebeat/registry
-
-  # Full Path to directory with additional prospector configuration files. Each file must end with .yml
-  # These config files must have the full filebeat config part inside, but only
-  # the prospector part is processed. All global options like spool_size are ignored.
-  # The config_dir MUST point to a different directory then where the main filebeat config file is in.
-  #config_dir:
-
-###############################################################################
-############################# Libbeat Config ##################################
-# Base config file used by all other beats for using libbeat features
-
-############################# Output ##########################################
-
-# Configure what outputs to use when sending the data collected by the beat.
-# Multiple outputs may be used.
-output:
-
-  ### Logstash as output
-  logstash:
-    # The Logstash hosts
-    hosts: [%%LOGSTASH_HOSTS%%]
-
-    timeout: 15
-
-    # Number of workers per Logstash host.
-    worker: %%LOGSTASH_WORKERS%%
-
-    # Set gzip compression level.
-    compression_level: 3
-
-    # Optional load balance the events between the Logstash hosts
-    loadbalance: true
-
-    # Optional index name. The default index name depends on the each beat.
-    # For Packetbeat, the default is set to packetbeat, for Topbeat
-    # top topbeat and for Filebeat to filebeat.
-    #index: filebeat
-
-    bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%%
-
-    # Optional TLS. By default is off.
-    #tls:
-      # List of root certificates for HTTPS server verifications
-      #certificate_authorities: ["/etc/pki/root/ca.pem"]
-
-      # Certificate for TLS client authentication
-      #certificate: "/etc/pki/client/cert.pem"
-
-      # Client Certificate Key
-      #certificate_key: "/etc/pki/client/cert.key"
-
-      # Controls whether the client verifies server certificates and host name.
-      # If insecure is set to true, all server host names and certificates will be
-      # accepted. In this mode TLS based connections are susceptible to
-      # man-in-the-middle attacks. Use only for testing.
-      #insecure: true
-
-      # Configure cipher suites to be used for TLS connections
-      #cipher_suites: []
-
-      # Configure curve types for ECDHE based cipher suites
-      #curve_types: []
-
-############################# Shipper #########################################
-
-shipper:
-
-############################# Logging #########################################
-
-# There are three options for the log ouput: syslog, file, stderr.
-# Under Windos systems, the log files are per default sent to the file output,
-# under all other system per default to syslog.
-logging:
-
-  # Send all logging output to syslog. On Windows default is false, otherwise
-  # default is true.
-  to_syslog: false
-
-  # Write all logging output to files. Beats automatically rotate files if rotateeverybytes
-  # limit is reached.
-  to_files: true
-
-  # To enable logging to files, to_files option has to be set to true
-  files:
-    # The directory where the log files will written to.
-    path: /home/y/logs/filebeat
-
-    # The name of the files where the logs are written to.
-    name: filebeat
-
-    # Configure log file size limit. If limit is reached, log file will be
-    # automatically rotated
-    rotateeverybytes: 10485760 # = 10MB
-
-    # Number of rotated log files to keep. Oldest files will be deleted first.
-    keepfiles: 7
-
-  # Enable debug output for selected components. To enable all selectors use ["*"]
-  # Other available selectors are beat, publish, service
-  # Multiple selectors can be chained.
-  #selectors: [ ]
-
-  # Sets log level. The default log level is error.
-  # Available log levels are: critical, error, warning, info, debug
-  level: warning
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
index ba433051a9f..cc26eb288ae 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java
@@ -6,7 +6,6 @@ import com.yahoo.vespa.hosted.node.admin.util.Environment;
 import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Test;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.Optional;
 
@@ -28,7 +27,7 @@ public class FilebeatConfigProviderTest {
     private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2");
 
     @Test
-    public void it_replaces_all_fields_correctly() throws IOException {
+    public void it_replaces_all_fields_correctly() {
         FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment());
 
         Optional<String> config = filebeatConfigProvider.getConfig(getNodeSpec(tenant, application, instance));
@@ -39,7 +38,7 @@ public class FilebeatConfigProviderTest {
     }
 
     @Test
-    public void it_does_not_generate_config_when_no_logstash_nodes() throws IOException {
+    public void it_does_not_generate_config_when_no_logstash_nodes() {
         Environment env = new Environment.Builder()
                 .environment(environment)
                 .region(region)
@@ -51,7 +50,7 @@ public class FilebeatConfigProviderTest {
     }
 
     @Test
-    public void it_does_not_generate_config_for_nodes_wihout_owner() throws IOException {
+    public void it_does_not_generate_config_for_nodes_wihout_owner() {
         FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment());
         ContainerNodeSpec nodeSpec = new ContainerNodeSpec.Builder()
                 .nodeFlavor("flavor")
@@ -64,22 +63,22 @@ public class FilebeatConfigProviderTest {
     }
 
     @Test
-    public void it_generates_correct_index_source() throws IOException {
+    public void it_generates_correct_index_source() {
         assertThat(getConfigString(), containsString("index_source: \"hosted-instance_vespa_music_us-north-1_prod_default\""));
     }
 
     @Test
-    public void it_sets_logstash_nodes_properly() throws IOException {
+    public void it_sets_logstash_nodes_properly() {
         assertThat(getConfigString(), containsString("hosts: [logstash1,logstash2]"));
     }
 
     @Test
-    public void it_generates_correct_spool_size() throws IOException {
+    public void it_generates_correct_spool_size() {
         // 2 nodes, 3 workers, 2048 buffer size -> 12288
         assertThat(getConfigString(), containsString("spool_size: 12288"));
     }
 
-    private String getConfigString() throws IOException {
+    private String getConfigString() {
         FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment());
         ContainerNodeSpec nodeSpec = getNodeSpec(tenant, application, instance);
         return filebeatConfigProvider.getConfig(nodeSpec).orElseThrow(() -> new RuntimeException("Failed to get filebeat config"));