diff options
6 files changed, 367 insertions, 3 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index ed18b9fe305..99da09ca71b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -57,6 +57,7 @@ public class DockerOperationsImpl implements DockerOperations { private static final Map<String, Boolean> DIRECTORIES_TO_MOUNT = new HashMap<>(); static { DIRECTORIES_TO_MOUNT.put("/etc/yamas-agent", true); + DIRECTORIES_TO_MOUNT.put("/etc/filebeat", true); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/daemontools_y"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/jdisc_core"), false); DIRECTORIES_TO_MOUNT.put(getDefaults().underVespaHome("logs/langdetect/"), false); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java new file mode 100644 index 00000000000..d038614b1ff --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java @@ -0,0 +1,62 @@ +package com.yahoo.vespa.hosted.node.admin.logging; + +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.util.Environment; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author mortent + */ +public class FilebeatConfigProvider { + + private static final String TEMPLATE = "filebeat.yml.template"; + + private static final String TENANT_FIELD = "%%TENANT%%"; + private static final String APPLICATION_FIELD = "%%APPLICATION%%"; + private static final String INSTANCE_FIELD = "%%INSTANCE%%"; + private static final String ENVIRONMENT_FIELD = "%%ENVIRONMENT%%"; + private static final String REGION_FIELD = "%%REGION%%"; + private static final String FILEBEAT_SPOOL_SIZE_FIELD = "%%FILEBEAT_SPOOL_SIZE%%"; + private static final String LOGSTASH_HOSTS_FIELD = "%%LOGSTASH_HOSTS%%"; + private static final String LOGSTASH_WORKERS_FIELD = "%%LOGSTASH_WORKERS%%"; + private static final String LOGSTASH_BULK_MAX_SIZE_FIELD = "%%LOGSTASH_BULK_MAX_SIZE%%"; + + private static final int logstashWorkers = 3; + private static final int logstashBulkMaxSize = 2048; + private final Environment environment; + + public FilebeatConfigProvider(Environment environment) { + this.environment = environment; + } + + public Optional<String> getConfig(ContainerNodeSpec containerNodeSpec) throws IOException { + + if (environment.getLogstashNodes().size() == 0 || !containerNodeSpec.owner.isPresent()) { + return Optional.empty(); + } + ContainerNodeSpec.Owner owner = containerNodeSpec.owner.get(); + int spoolSize = environment.getLogstashNodes().size() * logstashWorkers * logstashBulkMaxSize; + return Optional.of(readTemplate() + .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment()) + .replaceAll(REGION_FIELD, environment.getRegion()) + .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize)) + .replaceAll(LOGSTASH_HOSTS_FIELD, environment.getLogstashNodes().stream().collect(Collectors.joining(","))) + .replaceAll(LOGSTASH_WORKERS_FIELD, Integer.toString(logstashWorkers)) + .replaceAll(LOGSTASH_BULK_MAX_SIZE_FIELD, Integer.toString(logstashBulkMaxSize)) + .replaceAll(TENANT_FIELD, owner.tenant) + .replaceAll(APPLICATION_FIELD, owner.application) + .replaceAll(INSTANCE_FIELD, owner.instance)); + } + + private String readTemplate() throws IOException { + String file = getClass().getClassLoader().getResource(TEMPLATE).getFile(); + List<String> lines = Files.readAllLines(Paths.get(file)); + return lines.stream().collect(Collectors.joining("\n")); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 0dc395913bf..7a88bcad024 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; +import com.yahoo.vespa.hosted.node.admin.logging.FilebeatConfigProvider; import com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.noderepository.NodeRepository; @@ -20,6 +21,7 @@ import com.yahoo.vespa.hosted.node.admin.util.SecretAgentScheduleMaker; import com.yahoo.vespa.hosted.provision.Node; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; @@ -193,10 +195,28 @@ public class NodeAgentImpl implements NodeAgent { } } + private void experimentalWriteFile(final ContainerNodeSpec nodeSpec) { + try { + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); + Optional<String> config = filebeatConfigProvider.getConfig(nodeSpec); + if (! config.isPresent()) { + logger.error("Was not able to generate a config for filebeat, ignoring filebeat file creation." + nodeSpec.toString()); + return; + } + Path filebeatPath = environment.pathInNodeAdminFromPathInNode(containerName, "/etc/filebeat/filebeat.yml"); + Files.write(filebeatPath, config.get().getBytes()); + logger.info("Wrote filebeat config."); + } catch (Throwable t) { + logger.error("Failed writing filebeat config; " + nodeSpec, t); + } + } + private void runLocalResumeScriptIfNeeded(final ContainerNodeSpec nodeSpec) { if (containerState != RUNNING_HOWEVER_RESUME_SCRIPT_NOT_RUN) { return; } + experimentalWriteFile(nodeSpec); + addDebugMessage("Starting optional node program resume command"); logger.info("Starting optional node program resume command"); dockerOperations.resumeNode(containerName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java index 609646221c0..df7fe151cae 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/util/Environment.java @@ -39,6 +39,7 @@ public class Environment { private final String parentHostHostname; private final InetAddressResolver inetAddressResolver; private final PathResolver pathResolver; + private final List<String> logstashNodes; static { filenameFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -51,7 +52,9 @@ public class Environment { getEnvironmentVariable(REGION), HostName.getLocalhost(), new InetAddressResolver(), - new PathResolver()); + new PathResolver(), + Collections.emptyList() + ); } public Environment(Set<String> configServerHosts, @@ -59,13 +62,15 @@ public class Environment { String region, String parentHostHostname, InetAddressResolver inetAddressResolver, - PathResolver pathResolver) { + PathResolver pathResolver, + List<String> logstashNodes) { this.configServerHosts = configServerHosts; this.environment = environment; this.region = region; this.parentHostHostname = parentHostHostname; this.inetAddressResolver = inetAddressResolver; this.pathResolver = pathResolver; + this.logstashNodes = logstashNodes; } public Set<String> getConfigServerHosts() { return configServerHosts; } @@ -160,6 +165,10 @@ public class Environment { .resolve(PathResolver.ROOT.relativize(pathInNode)); } + public List<String> getLogstashNodes() { + return logstashNodes; + } + public static class Builder { private Set<String> configServerHosts = Collections.emptySet(); @@ -168,6 +177,7 @@ public class Environment { private String parentHostHostname; private InetAddressResolver inetAddressResolver; private PathResolver pathResolver; + private List<String> logstashNodes = Collections.emptyList(); public Builder configServerHosts(String... hosts) { configServerHosts = Arrays.stream(hosts).collect(Collectors.toSet()); @@ -199,8 +209,13 @@ public class Environment { return this; } + public Builder logstashNodes(List<String> hosts) { + this.logstashNodes = hosts; + return this; + } + public Environment build() { - return new Environment(configServerHosts, environment, region, parentHostHostname, inetAddressResolver, pathResolver); + return new Environment(configServerHosts, environment, region, parentHostHostname, inetAddressResolver, pathResolver, logstashNodes); } } } diff --git a/node-admin/src/main/resources/filebeat.yml.template b/node-admin/src/main/resources/filebeat.yml.template new file mode 100644 index 00000000000..7ab4aa95728 --- /dev/null +++ b/node-admin/src/main/resources/filebeat.yml.template @@ -0,0 +1,159 @@ +################### Filebeat Configuration Example ######################### + +############################# Filebeat ###################################### +filebeat: + # List of prospectors to fetch data. + prospectors: + + # vespa + - paths: + - /home/y/logs/vespa/vespa.log + exclude_files: [".gz$"] + document_type: vespa + fields: + HV-tenant: %%TENANT%% + HV-application: %%APPLICATION%% + HV-instance: %%INSTANCE%% + HV-region: %%REGION%% + HV-environment: %%ENVIRONMENT%% + index_source: "hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%" + fields_under_root: true + close_older: 20m + force_close_files: true + + # vespa qrs + - paths: + - /home/y/logs/vespa/qrs/QueryAccessLog.*.* + exclude_files: [".gz$"] + exclude_lines: ["reserved-for-internal-use/feedapi"] + document_type: vespa-qrs + fields: + HV-tenant: %%TENANT%% + HV-application: %%APPLICATION%% + HV-instance: %%INSTANCE%% + HV-region: %%REGION%% + HV-environment: %%ENVIRONMENT%% + index_source: "hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%" + fields_under_root: true + close_older: 20m + force_close_files: true + + # General filebeat configuration options + # + # Event count spool threshold - forces network flush if exceeded + spool_size: %%FILEBEAT_SPOOL_SIZE%% + + # Defines how often the spooler is flushed. After idle_timeout the spooler is + # Flush even though spool_size is not reached. + #idle_timeout: 5s + publish_async: false + + # Name of the registry file. Per default it is put in the current working + # directory. In case the working directory is changed after when running + # filebeat again, indexing starts from the beginning again. + registry_file: /var/lib/filebeat/registry + + # Full Path to directory with additional prospector configuration files. Each file must end with .yml + # These config files must have the full filebeat config part inside, but only + # the prospector part is processed. All global options like spool_size are ignored. + # The config_dir MUST point to a different directory then where the main filebeat config file is in. + #config_dir: + +############################################################################### +############################# Libbeat Config ################################## +# Base config file used by all other beats for using libbeat features + +############################# Output ########################################## + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. +output: + + ### Logstash as output + logstash: + # The Logstash hosts + hosts: [%%LOGSTASH_HOSTS%%] + + timeout: 15 + + # Number of workers per Logstash host. + worker: %%LOGSTASH_WORKERS%% + + # Set gzip compression level. + compression_level: 3 + + # Optional load balance the events between the Logstash hosts + loadbalance: true + + # Optional index name. The default index name depends on the each beat. + # For Packetbeat, the default is set to packetbeat, for Topbeat + # top topbeat and for Filebeat to filebeat. + #index: filebeat + + bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%% + + # Optional TLS. By default is off. + #tls: + # List of root certificates for HTTPS server verifications + #certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for TLS client authentication + #certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #certificate_key: "/etc/pki/client/cert.key" + + # Controls whether the client verifies server certificates and host name. + # If insecure is set to true, all server host names and certificates will be + # accepted. In this mode TLS based connections are susceptible to + # man-in-the-middle attacks. Use only for testing. + #insecure: true + + # Configure cipher suites to be used for TLS connections + #cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #curve_types: [] + +############################# Shipper ######################################### + +shipper: + +############################# Logging ######################################### + +# There are three options for the log ouput: syslog, file, stderr. +# Under Windos systems, the log files are per default sent to the file output, +# under all other system per default to syslog. +logging: + + # Send all logging output to syslog. On Windows default is false, otherwise + # default is true. + to_syslog: false + + # Write all logging output to files. Beats automatically rotate files if rotateeverybytes + # limit is reached. + to_files: true + + # To enable logging to files, to_files option has to be set to true + files: + # The directory where the log files will written to. + path: /home/y/logs/filebeat + + # The name of the files where the logs are written to. + name: filebeat + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + keepfiles: 7 + + # Enable debug output for selected components. To enable all selectors use ["*"] + # Other available selectors are beat, publish, service + # Multiple selectors can be chained. + #selectors: [ ] + + # Sets log level. The default log level is error. + # Available log levels are: critical, error, warning, info, debug + level: warning diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java new file mode 100644 index 00000000000..ba433051a9f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java @@ -0,0 +1,107 @@ +package com.yahoo.vespa.hosted.node.admin.logging; + +import com.google.common.collect.ImmutableList; +import com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec; +import com.yahoo.vespa.hosted.node.admin.util.Environment; +import com.yahoo.vespa.hosted.provision.Node; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.*; + +/** + * @author mortent + */ +public class FilebeatConfigProviderTest { + + + private static final String tenant = "vespa"; + private static final String application = "music"; + private static final String instance = "default"; + private static final String environment = "prod"; + private static final String region = "us-north-1"; + private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2"); + + @Test + public void it_replaces_all_fields_correctly() throws IOException { + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment()); + + Optional<String> config = filebeatConfigProvider.getConfig(getNodeSpec(tenant, application, instance)); + + assertTrue(config.isPresent()); + String configString = config.get(); + assertThat(configString, not(containsString("%%"))); + } + + @Test + public void it_does_not_generate_config_when_no_logstash_nodes() throws IOException { + Environment env = new Environment.Builder() + .environment(environment) + .region(region) + .build(); + + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(env); + Optional<String> config = filebeatConfigProvider.getConfig(getNodeSpec(tenant, application, instance)); + assertFalse(config.isPresent()); + } + + @Test + public void it_does_not_generate_config_for_nodes_wihout_owner() throws IOException { + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment()); + ContainerNodeSpec nodeSpec = new ContainerNodeSpec.Builder() + .nodeFlavor("flavor") + .nodeState(Node.State.active) + .nodeType("type") + .hostname("hostname") + .build(); + Optional<String> config = filebeatConfigProvider.getConfig(nodeSpec); + assertFalse(config.isPresent()); + } + + @Test + public void it_generates_correct_index_source() throws IOException { + assertThat(getConfigString(), containsString("index_source: \"hosted-instance_vespa_music_us-north-1_prod_default\"")); + } + + @Test + public void it_sets_logstash_nodes_properly() throws IOException { + assertThat(getConfigString(), containsString("hosts: [logstash1,logstash2]")); + } + + @Test + public void it_generates_correct_spool_size() throws IOException { + // 2 nodes, 3 workers, 2048 buffer size -> 12288 + assertThat(getConfigString(), containsString("spool_size: 12288")); + } + + private String getConfigString() throws IOException { + FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment()); + ContainerNodeSpec nodeSpec = getNodeSpec(tenant, application, instance); + return filebeatConfigProvider.getConfig(nodeSpec).orElseThrow(() -> new RuntimeException("Failed to get filebeat config")); + } + + private Environment getEnvironment() { + return new Environment.Builder() + .environment(environment) + .region(region) + .logstashNodes(logstashNodes) + .build(); + } + + private ContainerNodeSpec getNodeSpec(String tenant, String application, String instance) { + ContainerNodeSpec.Owner owner = new ContainerNodeSpec.Owner(tenant, application, instance); + return new ContainerNodeSpec.Builder() + .owner(owner) + .nodeFlavor("flavor") + .nodeState(Node.State.active) + .nodeType("type") + .hostname("hostname") + .build(); + } + +} |