diff options
author | Valerij Fredriksen <freva@users.noreply.github.com> | 2018-10-23 16:09:31 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-10-23 16:09:31 +0200 |
commit | a4f198e2ea8c34f4a1fcfa532d2d2db203ef65e3 (patch) | |
tree | aa4bb2dd55c2f59ead4241963caeab5af1094417 /node-admin | |
parent | e665645a140cf6702e9317294962a76457f33d9d (diff) | |
parent | 817ea053034b303cb01538e87dbafc822dc85090 (diff) |
Merge pull request #7392 from vespa-engine/revert-7363-revert-7352-freva/cleanup
Revert "Revert "NodeAdmin: Remove unused stuff""
Diffstat (limited to 'node-admin')
12 files changed, 2 insertions, 1306 deletions
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 7b484dfc481..ec911cc5600 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.component; import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; import java.net.URI; import java.util.ArrayList; @@ -24,12 +22,6 @@ public class ConfigServerInfo { private final Map<String, URI> configServerURIs; private final AthenzService configServerIdentity; - // TODO: Remove - public ConfigServerInfo(ConfigServerConfig config) { - this(config.loadBalancerHost(), config.hosts(), config.scheme(), config.port(), - (AthenzService) AthenzIdentities.from(config.configserverAthenzIdentity())); - } - public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames, String scheme, int port, AthenzService configServerAthenzIdentity) { this.configServerHostNames = configServerHostNames; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java deleted file mode 100644 index aaadd3cb24e..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/Environment.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.dockerapi.ContainerName; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddresses; -import com.yahoo.vespa.hosted.node.admin.task.util.network.IPAddressesImpl; - -import java.net.URI; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -/** - * Various utilities for getting values from node-admin's environment. Immutable. - * - * @author Øyvind Bakksjø - * @author hmusum - */ -public class Environment { - private final ConfigServerInfo configServerInfo; - private final String environment; - private final String region; - private final String system; - private final String cloud; - private final String parentHostHostname; - private final IPAddresses ipAddresses; - private final PathResolver pathResolver; - private final List<String> logstashNodes; - private final NodeType nodeType; - private final ContainerEnvironmentResolver containerEnvironmentResolver; - private final String certificateDnsSuffix; - private final URI ztsUri; - private final AthenzService nodeAthenzIdentity; - private final boolean nodeAgentCertEnabled; - private final Path trustStorePath; - private final DockerNetworking dockerNetworking; - - private Environment(ConfigServerInfo configServerInfo, - Path trustStorePath, - String environment, - String region, - String system, - String cloud, - String parentHostHostname, - IPAddresses ipAddresses, - PathResolver pathResolver, - List<String> logstashNodes, - NodeType nodeType, - ContainerEnvironmentResolver containerEnvironmentResolver, - String certificateDnsSuffix, - URI ztsUri, - AthenzService nodeAthenzIdentity, - boolean nodeAgentCertEnabled, - DockerNetworking dockerNetworking) { - this.configServerInfo = Objects.requireNonNull(configServerInfo, "configServerConfig cannot be null"); - this.environment = Objects.requireNonNull(environment, "environment cannot be null");; - this.region = Objects.requireNonNull(region, "region cannot be null");; - this.system = Objects.requireNonNull(system, "system cannot be null");; - this.cloud = Objects.requireNonNull(cloud, "cloud cannot be null"); - this.parentHostHostname = parentHostHostname; - this.ipAddresses = ipAddresses; - this.pathResolver = pathResolver; - this.logstashNodes = logstashNodes; - this.nodeType = nodeType; - this.containerEnvironmentResolver = containerEnvironmentResolver; - this.certificateDnsSuffix = certificateDnsSuffix; - this.ztsUri = ztsUri; - this.nodeAthenzIdentity = nodeAthenzIdentity; - this.nodeAgentCertEnabled = nodeAgentCertEnabled; - this.trustStorePath = trustStorePath; - this.dockerNetworking = Objects.requireNonNull(dockerNetworking, "dockerNetworking cannot be null"); - } - - public List<String> getConfigServerHostNames() { return configServerInfo.getConfigServerHostNames(); } - - public String getEnvironment() { return environment; } - - public String getRegion() { - return region; - } - - public String getSystem() { - return system; - } - - public String getCloud() { return cloud; } - - public String getParentHostHostname() { - return parentHostHostname; - } - - public String getZone() { - return getEnvironment() + "." + getRegion(); - } - - public IPAddresses getIpAddresses() { - return ipAddresses; - } - - public PathResolver getPathResolver() { - return pathResolver; - } - - /** - * Translates an absolute path in node agent container to an absolute path in node admin container. - * @param containerName name of the node agent container - * @param pathInNode absolute path in that container - * @return the absolute path in node admin container pointing at the same inode - */ - public Path pathInNodeAdminFromPathInNode(ContainerName containerName, Path pathInNode) { - if (! pathInNode.isAbsolute()) { - throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode); - } - - return pathResolver.getApplicationStoragePathForNodeAdmin() - .resolve(containerName.asString()) - .resolve(PathResolver.ROOT.relativize(pathInNode)); - } - - /** - * Translates an absolute path in node agent container to an absolute path in host. - * @param containerName name of the node agent container - * @param pathInNode absolute path in that container - * @return the absolute path in host pointing at the same inode - */ - public Path pathInHostFromPathInNode(ContainerName containerName, Path pathInNode) { - if (! pathInNode.isAbsolute()) { - throw new IllegalArgumentException("The specified path in node was not absolute: " + pathInNode); - } - - return pathResolver.getApplicationStoragePathForHost() - .resolve(containerName.asString()) - .resolve(PathResolver.ROOT.relativize(pathInNode)); - } - - public Path pathInNodeUnderVespaHome(String relativePath) { - return pathResolver.getVespaHomePathForContainer() - .resolve(relativePath); - } - - public List<String> getLogstashNodes() { - return logstashNodes; - } - - public NodeType getNodeType() { return nodeType; } - - public ContainerEnvironmentResolver getContainerEnvironmentResolver() { - return containerEnvironmentResolver; - } - - public Path getTrustStorePath() { - return trustStorePath; - } - - public AthenzService getConfigserverAthenzIdentity() { - return configServerInfo.getConfigServerIdentity(); - } - - public AthenzService getNodeAthenzIdentity() { - return nodeAthenzIdentity; - } - - public String getCertificateDnsSuffix() { - return certificateDnsSuffix; - } - - public URI getZtsUri() { - return ztsUri; - } - - public URI getConfigserverLoadBalancerEndpoint() { - return configServerInfo.getLoadBalancerEndpoint(); - } - - public boolean isNodeAgentCertEnabled() { - return nodeAgentCertEnabled; - } - - public DockerNetworking getDockerNetworking() { - return dockerNetworking; - } - - public static class Builder { - private ConfigServerInfo configServerInfo; - private String environment; - private String region; - private String system; - private String cloud; - private String parentHostHostname; - private IPAddresses ipAddresses; - private PathResolver pathResolver; - private List<String> logstashNodes = Collections.emptyList(); - private NodeType nodeType = NodeType.tenant; - private ContainerEnvironmentResolver containerEnvironmentResolver; - private String certificateDnsSuffix; - private URI ztsUri; - private AthenzService nodeAthenzIdentity; - private boolean nodeAgentCertEnabled; - private Path trustStorePath; - private DockerNetworking dockerNetworking; - - public Builder configServerConfig(ConfigServerConfig configServerConfig) { - this.configServerInfo = new ConfigServerInfo(configServerConfig); - return this; - } - - public Builder configServerInfo(ConfigServerInfo configServerInfo) { - this.configServerInfo = configServerInfo; - return this; - } - - public Builder environment(String environment) { - this.environment = environment; - return this; - } - - public Builder region(String region) { - this.region = region; - return this; - } - - public Builder system(String system) { - this.system = system; - return this; - } - - public Builder cloud(String cloud) { - this.cloud = cloud; - return this; - } - - public Builder parentHostHostname(String parentHostHostname) { - this.parentHostHostname = parentHostHostname; - return this; - } - - public Builder ipAddresses(IPAddresses ipAddresses) { - this.ipAddresses = ipAddresses; - return this; - } - - public Builder pathResolver(PathResolver pathResolver) { - this.pathResolver = pathResolver; - return this; - } - - public Builder containerEnvironmentResolver(ContainerEnvironmentResolver containerEnvironmentResolver) { - this.containerEnvironmentResolver = containerEnvironmentResolver; - return this; - } - - public Builder logstashNodes(List<String> hosts) { - this.logstashNodes = hosts; - return this; - } - - public Builder nodeType(NodeType nodeType) { - this.nodeType = nodeType; - return this; - } - - public Builder certificateDnsSuffix(String certificateDnsSuffix) { - this.certificateDnsSuffix = certificateDnsSuffix; - return this; - } - - public Builder ztsUri(URI ztsUri) { - this.ztsUri = ztsUri; - return this; - } - - public Builder nodeAthenzIdentity(AthenzService nodeAthenzIdentity) { - this.nodeAthenzIdentity = nodeAthenzIdentity; - return this; - } - - public Builder enableNodeAgentCert(boolean nodeAgentCertEnabled) { - this.nodeAgentCertEnabled = nodeAgentCertEnabled; - return this; - } - - public Builder trustStorePath(Path trustStorePath) { - this.trustStorePath = trustStorePath; - return this; - } - - public Builder dockerNetworking(DockerNetworking dockerNetworking) { - this.dockerNetworking = dockerNetworking; - return this; - } - - public Environment build() { - return new Environment(configServerInfo, - trustStorePath, - environment, - region, - system, - cloud, - parentHostHostname, - Optional.ofNullable(ipAddresses).orElseGet(IPAddressesImpl::new), - Optional.ofNullable(pathResolver).orElseGet(PathResolver::new), - logstashNodes, - nodeType, - Optional.ofNullable(containerEnvironmentResolver).orElseGet(() -> node -> ""), - certificateDnsSuffix, - ztsUri, - nodeAthenzIdentity, - nodeAgentCertEnabled, - dockerNetworking); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java deleted file mode 100644 index 433fae0e551..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/PathResolver.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.component; - -import com.yahoo.vespa.defaults.Defaults; - -import java.nio.file.Path; -import java.nio.file.Paths; - -/** - * @author freva - */ -public class PathResolver { - public static final Path ROOT = Paths.get("/"); - public static final Path DEFAULT_HOST_ROOT = Paths.get("/host"); - public static final Path RELATIVE_APPLICATION_STORAGE_PATH = Paths.get("home/docker/container-storage"); - - private final Path hostRoot; - private final Path vespaHomePathForContainer; - - private final Path applicationStoragePathForNodeAdmin; - private final Path applicationStoragePathForHost; - - /** - * @param hostRoot the absolute path to the root of the host's file system - * @param vespaHomeForContainer the absolute path of Vespa home in the mount namespace of any - * and all Docker containers managed by Node Admin. - */ - public PathResolver(Path hostRoot, Path vespaHomeForContainer) { - if (!hostRoot.isAbsolute()) { - throw new IllegalArgumentException("Path to root of host file system is not absolute: " + - hostRoot); - } - this.hostRoot = hostRoot; - - if (!vespaHomeForContainer.isAbsolute()) { - throw new IllegalArgumentException("Path to Vespa home is not absolute: " + vespaHomeForContainer); - } - this.vespaHomePathForContainer = vespaHomeForContainer; - - this.applicationStoragePathForNodeAdmin = hostRoot.resolve(RELATIVE_APPLICATION_STORAGE_PATH); - this.applicationStoragePathForHost = ROOT.resolve(RELATIVE_APPLICATION_STORAGE_PATH); - } - - public PathResolver() { - this(DEFAULT_HOST_ROOT, Paths.get(Defaults.getDefaults().vespaHome())); - } - - /** For testing */ - public PathResolver(Path vespaHomePathForContainer, Path applicationStoragePathForNodeAdmin, Path applicationStoragePathForHost) { - this.hostRoot = DEFAULT_HOST_ROOT; - this.vespaHomePathForContainer = vespaHomePathForContainer; - this.applicationStoragePathForNodeAdmin = applicationStoragePathForNodeAdmin; - this.applicationStoragePathForHost = applicationStoragePathForHost; - } - - /** - * Returns the absolute path of the Vespa home directory in any Docker container mount namespace. - * - * It's a limitation of current implementation that all containers MUST have the same Vespa - * home directory path. - */ - public Path getVespaHomePathForContainer() { - return vespaHomePathForContainer; - } - - /** Returns the absolute path to the container storage directory for the node admin (this process). */ - public Path getApplicationStoragePathForNodeAdmin() { - return applicationStoragePathForNodeAdmin; - } - - /** Returns the absolute path to the container storage directory for the host. */ - public Path getApplicationStoragePathForHost() { - return applicationStoragePathForHost; - } - - /** - * Returns the absolute path to the directory which is the root directory of the host - * file system. - */ - public Path getPathToRootOfHost() { - return hostRoot; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java deleted file mode 100644 index 15cfa99c749..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/config/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.hosted.node.admin.config; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java deleted file mode 100644 index ce751548f75..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProvider.java +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.logging; - -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.component.Environment; - -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * @author mortent - */ -public class FilebeatConfigProvider { - - private static final String TENANT_FIELD = "%%TENANT%%"; - private static final String APPLICATION_FIELD = "%%APPLICATION%%"; - private static final String INSTANCE_FIELD = "%%INSTANCE%%"; - private static final String ENVIRONMENT_FIELD = "%%ENVIRONMENT%%"; - private static final String REGION_FIELD = "%%REGION%%"; - private static final String FILEBEAT_SPOOL_SIZE_FIELD = "%%FILEBEAT_SPOOL_SIZE%%"; - private static final String LOGSTASH_HOSTS_FIELD = "%%LOGSTASH_HOSTS%%"; - private static final String LOGSTASH_WORKERS_FIELD = "%%LOGSTASH_WORKERS%%"; - private static final String LOGSTASH_BULK_MAX_SIZE_FIELD = "%%LOGSTASH_BULK_MAX_SIZE%%"; - - private static final int logstashWorkers = 3; - private static final int logstashBulkMaxSize = 2048; - private final Environment environment; - - public FilebeatConfigProvider(Environment environment) { - this.environment = environment; - } - - public Optional<String> getConfig(NodeAgentContext context, NodeSpec node) { - - if (environment.getLogstashNodes().size() == 0 || !node.getOwner().isPresent()) { - return Optional.empty(); - } - NodeSpec.Owner owner = node.getOwner().get(); - int spoolSize = environment.getLogstashNodes().size() * logstashWorkers * logstashBulkMaxSize; - String logstashNodeString = environment.getLogstashNodes().stream() - .map(this::addQuotes) - .collect(Collectors.joining(",")); - return Optional.of(getTemplate(context) - .replaceAll(ENVIRONMENT_FIELD, environment.getEnvironment()) - .replaceAll(REGION_FIELD, environment.getRegion()) - .replaceAll(FILEBEAT_SPOOL_SIZE_FIELD, Integer.toString(spoolSize)) - .replaceAll(LOGSTASH_HOSTS_FIELD, logstashNodeString) - .replaceAll(LOGSTASH_WORKERS_FIELD, Integer.toString(logstashWorkers)) - .replaceAll(LOGSTASH_BULK_MAX_SIZE_FIELD, Integer.toString(logstashBulkMaxSize)) - .replaceAll(TENANT_FIELD, owner.getTenant()) - .replaceAll(APPLICATION_FIELD, owner.getApplication()) - .replaceAll(INSTANCE_FIELD, owner.getInstance())); - } - - private String addQuotes(String logstashNode) { - return logstashNode.startsWith("\"") - ? logstashNode - : String.format("\"%s\"", logstashNode); - } - - private String getTemplate(NodeAgentContext context) { - return "################### Filebeat Configuration Example #########################\n" + - "\n" + - "############################# Filebeat ######################################\n" + - "filebeat:\n" + - " # List of prospectors to fetch data.\n" + - " prospectors:\n" + - "\n" + - " # vespa\n" + - " - paths:\n" + - " - " + context.pathInNodeUnderVespaHome("logs/vespa/vespa.log") + "\n" + - " exclude_files: [\".gz$\"]\n" + - " document_type: vespa\n" + - " fields:\n" + - " HV-tenant: %%TENANT%%\n" + - " HV-application: %%APPLICATION%%\n" + - " HV-instance: %%INSTANCE%%\n" + - " HV-region: %%REGION%%\n" + - " HV-environment: %%ENVIRONMENT%%\n" + - " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + - " fields_under_root: true\n" + - " close_older: 20m\n" + - " force_close_files: true\n" + - "\n" + - " # vespa qrs\n" + - " - paths:\n" + - " - " + context.pathInNodeUnderVespaHome("logs/vespa/qrs/QueryAccessLog.*.*") + "\n" + - " exclude_files: [\".gz$\"]\n" + - " exclude_lines: [\"reserved-for-internal-use/feedapi\"]\n" + - " document_type: vespa-qrs\n" + - " fields:\n" + - " HV-tenant: %%TENANT%%\n" + - " HV-application: %%APPLICATION%%\n" + - " HV-instance: %%INSTANCE%%\n" + - " HV-region: %%REGION%%\n" + - " HV-environment: %%ENVIRONMENT%%\n" + - " index_source: \"hosted-instance_%%TENANT%%_%%APPLICATION%%_%%REGION%%_%%ENVIRONMENT%%_%%INSTANCE%%\"\n" + - " fields_under_root: true\n" + - " close_older: 20m\n" + - " force_close_files: true\n" + - "\n" + - " # General filebeat configuration options\n" + - " #\n" + - " # Event count spool threshold - forces network flush if exceeded\n" + - " spool_size: %%FILEBEAT_SPOOL_SIZE%%\n" + - "\n" + - " # Defines how often the spooler is flushed. After idle_timeout the spooler is\n" + - " # Flush even though spool_size is not reached.\n" + - " #idle_timeout: 5s\n" + - " publish_async: false\n" + - "\n" + - " # Name of the registry file. Per default it is put in the current working\n" + - " # directory. In case the working directory is changed after when running\n" + - " # filebeat again, indexing starts from the beginning again.\n" + - " registry_file: /var/lib/filebeat/registry\n" + - "\n" + - " # Full Path to directory with additional prospector configuration files. Each file must end with .yml\n" + - " # These config files must have the full filebeat config part inside, but only\n" + - " # the prospector part is processed. All global options like spool_size are ignored.\n" + - " # The config_dir MUST point to a different directory then where the main filebeat config file is in.\n" + - " #config_dir:\n" + - "\n" + - "###############################################################################\n" + - "############################# Libbeat Config ##################################\n" + - "# Base config file used by all other beats for using libbeat features\n" + - "\n" + - "############################# Output ##########################################\n" + - "\n" + - "# Configure what outputs to use when sending the data collected by the beat.\n" + - "# Multiple outputs may be used.\n" + - "output:\n" + - "\n" + - " ### Logstash as output\n" + - " logstash:\n" + - " # The Logstash hosts\n" + - " hosts: [%%LOGSTASH_HOSTS%%]\n" + - "\n" + - " timeout: 15\n" + - "\n" + - " # Number of workers per Logstash host.\n" + - " worker: %%LOGSTASH_WORKERS%%\n" + - "\n" + - " # Set gzip compression level.\n" + - " compression_level: 3\n" + - "\n" + - " # Optional load balance the events between the Logstash hosts\n" + - " loadbalance: true\n" + - "\n" + - " # Optional index name. The default index name depends on the each beat.\n" + - " # For Packetbeat, the default is set to packetbeat, for Topbeat\n" + - " # top topbeat and for Filebeat to filebeat.\n" + - " #index: filebeat\n" + - "\n" + - " bulk_max_size: %%LOGSTASH_BULK_MAX_SIZE%%\n" + - "\n" + - " # Optional TLS. By default is off.\n" + - " #tls:\n" + - " # List of root certificates for HTTPS server verifications\n" + - " #certificate_authorities: [\"/etc/pki/root/ca.pem\"]\n" + - "\n" + - " # Certificate for TLS client authentication\n" + - " #certificate: \"/etc/pki/client/cert.pem\"\n" + - "\n" + - " # Client Certificate Key\n" + - " #certificate_key: \"/etc/pki/client/cert.key\"\n" + - "\n" + - " # Controls whether the client verifies server certificates and host name.\n" + - " # If insecure is set to true, all server host names and certificates will be\n" + - " # accepted. In this mode TLS based connections are susceptible to\n" + - " # man-in-the-middle attacks. Use only for testing.\n" + - " #insecure: true\n" + - "\n" + - " # Configure cipher suites to be used for TLS connections\n" + - " #cipher_suites: []\n" + - "\n" + - " # Configure curve types for ECDHE based cipher suites\n" + - " #curve_types: []\n" + - "\n" + - "############################# Shipper #########################################\n" + - "\n" + - "shipper:\n" + - "\n" + - "############################# Logging #########################################\n" + - "\n" + - "# There are three options for the log ouput: syslog, file, stderr.\n" + - "# Under Windos systems, the log files are per default sent to the file output,\n" + - "# under all other system per default to syslog.\n" + - "logging:\n" + - "\n" + - " # Send all logging output to syslog. On Windows default is false, otherwise\n" + - " # default is true.\n" + - " to_syslog: false\n" + - "\n" + - " # Write all logging output to files. Beats automatically rotate files if rotateeverybytes\n" + - " # limit is reached.\n" + - " to_files: true\n" + - "\n" + - " # To enable logging to files, to_files option has to be set to true\n" + - " files:\n" + - " # The directory where the log files will written to.\n" + - " path: " + context.pathInNodeUnderVespaHome("logs/filebeat") + "\n" + - "\n" + - " # The name of the files where the logs are written to.\n" + - " name: filebeat\n" + - "\n" + - " # Configure log file size limit. If limit is reached, log file will be\n" + - " # automatically rotated\n" + - " rotateeverybytes: 10485760 # = 10MB\n" + - "\n" + - " # Number of rotated log files to keep. Oldest files will be deleted first.\n" + - " keepfiles: 7\n" + - "\n" + - " # Enable debug output for selected components. To enable all selectors use [\"*\"]\n" + - " # Other available selectors are beat, publish, service\n" + - " # Multiple selectors can be chained.\n" + - " #selectors: [ ]\n" + - "\n" + - " # Sets log level. The default log level is error.\n" + - " # Available log levels are: critical, error, warning, info, debug\n" + - " level: warning\n"; - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java deleted file mode 100644 index cf010121c2a..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelper.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Duration; -import java.time.Instant; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @author freva - */ -public class FileHelper { - private static final Logger logger = Logger.getLogger(FileHelper.class.getSimpleName()); - - /** - * (Recursively) deletes files if they match all the criteria, also deletes empty directories. - * - * @param basePath Base path from where to start the search - * @param maxAge Delete files older (last modified date) than maxAge - * @param fileNameRegex Delete files where filename matches fileNameRegex - * @param recursive Delete files in sub-directories (with the same criteria) - */ - public static void deleteFiles(Path basePath, Duration maxAge, Optional<String> fileNameRegex, boolean recursive) throws IOException { - Pattern fileNamePattern = fileNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - if (recursive) { - deleteFiles(path, maxAge, fileNameRegex, true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } else if (isPatternMatchingFilename(fileNamePattern, path) && - isTimeSinceLastModifiedMoreThan(path, maxAge)) { - if (! Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes all files in target directory except the n most recent (by modified date) - * - * @param basePath Base path to delete from - * @param nMostRecentToKeep Number of most recent files to keep - */ - static void deleteFilesExceptNMostRecent(Path basePath, int nMostRecentToKeep) throws IOException { - if (nMostRecentToKeep < 1) { - throw new IllegalArgumentException("Number of files to keep must be a positive number"); - } - - List<Path> pathsInDeleteDir = listContentsOfDirectory(basePath).stream() - .filter(Files::isRegularFile) - .sorted(Comparator.comparing(FileHelper::getLastModifiedTime)) - .skip(nMostRecentToKeep) - .collect(Collectors.toList()); - - for (Path path : pathsInDeleteDir) { - if (!Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - - static void deleteFilesLargerThan(Path basePath, long sizeInBytes) throws IOException { - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path)) { - deleteFilesLargerThan(path, sizeInBytes); - } else { - if (Files.size(path) > sizeInBytes && !Files.deleteIfExists(path)) { - logger.warning("Could not delete file: " + path.toAbsolutePath()); - } - } - } - } - - /** - * Deletes directories and their contents if they match all the criteria - * - * @param basePath Base path to delete the directories from - * @param maxAge Delete directories older (last modified date) than maxAge - * @param dirNameRegex Delete directories where directory name matches dirNameRegex - */ - public static void deleteDirectories(Path basePath, Duration maxAge, Optional<String> dirNameRegex) throws IOException { - Pattern dirNamePattern = dirNameRegex.map(Pattern::compile).orElse(null); - - for (Path path : listContentsOfDirectory(basePath)) { - if (Files.isDirectory(path) && isPatternMatchingFilename(dirNamePattern, path)) { - boolean mostRecentFileModifiedBeforeMaxAge = getMostRecentlyModifiedFileIn(path) - .map(mostRecentlyModified -> isTimeSinceLastModifiedMoreThan(mostRecentlyModified, maxAge)) - .orElse(true); - - if (mostRecentFileModifiedBeforeMaxAge) { - deleteFiles(path, Duration.ZERO, Optional.empty(), true); - if (listContentsOfDirectory(path).isEmpty() && !Files.deleteIfExists(path)) { - logger.warning("Could not delete directory: " + path.toAbsolutePath()); - } - } - } - } - } - - /** - * Similar to rm -rf file: - * - It's not an error if file doesn't exist - * - If file is a directory, it and all content is removed - * - For symlinks: Only the symlink is removed, not what the symlink points to - */ - public static void recursiveDelete(Path basePath) throws IOException { - if (Files.isDirectory(basePath)) { - for (Path path : listContentsOfDirectory(basePath)) { - recursiveDelete(path); - } - } - - Files.deleteIfExists(basePath); - } - - public static void moveIfExists(Path from, Path to) throws IOException { - if (Files.exists(from)) { - Files.move(from, to); - } - } - - private static Optional<Path> getMostRecentlyModifiedFileIn(Path basePath) throws IOException { - return Files.walk(basePath).max(Comparator.comparing(FileHelper::getLastModifiedTime)); - } - - private static boolean isTimeSinceLastModifiedMoreThan(Path path, Duration duration) { - Instant nowMinusDuration = Instant.now().minus(duration); - Instant lastModified = getLastModifiedTime(path).toInstant(); - - // Return true also if they are equal for test stability - // (lastModified <= nowMinusDuration) is the same as !(lastModified > nowMinusDuration) - return !lastModified.isAfter(nowMinusDuration); - } - - private static boolean isPatternMatchingFilename(Pattern pattern, Path path) { - return pattern == null || pattern.matcher(path.getFileName().toString()).find(); - } - - /** - * @return list all files in a directory, returns empty list if directory does not exist - */ - public static List<Path> listContentsOfDirectory(Path basePath) { - try (Stream<Path> directoryStream = Files.list(basePath)) { - return directoryStream.collect(Collectors.toList()); - } catch (NoSuchFileException ignored) { - return Collections.emptyList(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to list contents of directory " + basePath.toAbsolutePath(), e); - } - } - - static FileTime getLastModifiedTime(Path path) { - try { - return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS); - } catch (IOException e) { - throw new UncheckedIOException("Failed to get last modified time of " + path.toAbsolutePath(), e); - } - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index bbea505b19d..22093258930 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -70,7 +70,7 @@ public class AthenzCredentialsMaintainer { private final CsrGenerator csrGenerator; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts - private Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); + private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); public AthenzCredentialsMaintainer(URI ztsEndpoint, Path trustStorePath, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 395b4d458a2..ef137c55ffb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -337,7 +337,7 @@ public class NodeAgentImpl implements NodeAgent { } if (node.getWantedDockerImage().isPresent() && !node.getWantedDockerImage().get().equals(existingContainer.image)) { return Optional.of("The node is supposed to run a new Docker image: " - + existingContainer + " -> " + node.getWantedDockerImage().get()); + + existingContainer.image.asString() + " -> " + node.getWantedDockerImage().get().asString()); } if (!existingContainer.state.isRunning()) { return Optional.of("Container no longer running"); diff --git a/node-admin/src/main/resources/configdefinitions/config-server.def b/node-admin/src/main/resources/configdefinitions/config-server.def deleted file mode 100644 index 6a088829bad..00000000000 --- a/node-admin/src/main/resources/configdefinitions/config-server.def +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.hosted.node.admin.config - -hosts[] string -port int default=8080 range=[1,65535] -scheme string default="http" -loadBalancerHost string default="" -configserverAthenzIdentity string default="vespa.configserver"
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java deleted file mode 100644 index 281c7df9a1f..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/component/PathResolverTest.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.vespa.hosted.node.admin.component; - -import org.junit.Test; - -import java.nio.file.Paths; - -import static org.junit.Assert.assertEquals; - -public class PathResolverTest { - @Test - public void testNodeAdminOnHost() { - PathResolver pathResolver = new PathResolver(Paths.get("/"), Paths.get("/home/y")); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost()); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin()); - assertEquals(Paths.get("/"), pathResolver.getPathToRootOfHost()); - assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer()); - } - - @Test - public void testNodeAdminInContainer() { - PathResolver pathResolver = new PathResolver(Paths.get("/host"), Paths.get("/home/y")); - assertEquals(Paths.get("/home/docker/container-storage"), pathResolver.getApplicationStoragePathForHost()); - assertEquals(Paths.get("/host/home/docker/container-storage"), pathResolver.getApplicationStoragePathForNodeAdmin()); - assertEquals(Paths.get("/host"), pathResolver.getPathToRootOfHost()); - assertEquals(Paths.get("/home/y"), pathResolver.getVespaHomePathForContainer()); - } -}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java deleted file mode 100644 index d4fadabe695..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/logging/FilebeatConfigProviderTest.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.logging; - -import com.google.common.collect.ImmutableList; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; -import com.yahoo.vespa.hosted.node.admin.component.Environment; -import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig; -import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; -import com.yahoo.vespa.hosted.provision.Node; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.*; - -/** - * @author mortent - */ -public class FilebeatConfigProviderTest { - - private static final String tenant = "vespa"; - private static final String application = "music"; - private static final String instance = "default"; - private static final String environment = "prod"; - private static final String region = "us-north-1"; - private static final String system = "main"; - private static final List<String> logstashNodes = ImmutableList.of("logstash1", "logstash2"); - private final NodeAgentContext context = new NodeAgentContextImpl.Builder("node-123.hostname.tld").build(); - - @Test - public void it_replaces_all_fields_correctly() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - - assertTrue(config.isPresent()); - String configString = config.get(); - assertThat(configString, not(containsString("%%"))); - } - - @Test - public void it_does_not_generate_config_when_no_logstash_nodes() { - Environment env = getEnvironment(Collections.emptyList()); - - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(env); - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - assertFalse(config.isPresent()); - } - - @Test - public void it_does_not_generate_config_for_nodes_wihout_owner() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - NodeSpec node = new NodeSpec.Builder() - .flavor("flavor") - .state(Node.State.active) - .nodeType(NodeType.tenant) - .hostname("hostname") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - Optional<String> config = filebeatConfigProvider.getConfig(context, node); - assertFalse(config.isPresent()); - } - - @Test - public void it_generates_correct_index_source() { - assertThat(getConfigString(), containsString("index_source: \"hosted-instance_vespa_music_us-north-1_prod_default\"")); - } - - @Test - public void it_sets_logstash_nodes_properly() { - assertThat(getConfigString(), containsString("hosts: [\"logstash1\",\"logstash2\"]")); - } - - @Test - public void it_does_not_add_double_quotes() { - Environment environment = getEnvironment(ImmutableList.of("unquoted", "\"quoted\"")); - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment); - Optional<String> config = filebeatConfigProvider.getConfig(context, createNodeRepositoryNode(tenant, application, instance)); - assertThat(config.get(), containsString("hosts: [\"unquoted\",\"quoted\"]")); - } - - @Test - public void it_generates_correct_spool_size() { - // 2 nodes, 3 workers, 2048 buffer size -> 12288 - assertThat(getConfigString(), containsString("spool_size: 12288")); - } - - private String getConfigString() { - FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(getEnvironment(logstashNodes)); - NodeSpec node = createNodeRepositoryNode(tenant, application, instance); - return filebeatConfigProvider.getConfig(context, node).orElseThrow(() -> new RuntimeException("Failed to get filebeat config")); - } - - private Environment getEnvironment(List<String> logstashNodes) { - return new Environment.Builder() - .configServerConfig(new ConfigServerConfig(new ConfigServerConfig.Builder())) - .environment(environment) - .region(region) - .system(system) - .logstashNodes(logstashNodes) - .cloud("mycloud") - .dockerNetworking(DockerNetworking.HOST_NETWORK) - .build(); - } - - private NodeSpec createNodeRepositoryNode(String tenant, String application, String instance) { - NodeSpec.Owner owner = new NodeSpec.Owner(tenant, application, instance); - return new NodeSpec.Builder() - .owner(owner) - .flavor("flavor") - .state(Node.State.active) - .nodeType(NodeType.tenant) - .hostname("hostname") - .minCpuCores(1) - .minMainMemoryAvailableGb(1) - .minDiskAvailableGb(1) - .build(); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java deleted file mode 100644 index 6b53bc217c4..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/FileHelperTest.java +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.maintenance; - -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author freva - */ -public class FileHelperTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void initFiles() throws IOException { - for (int i=0; i<10; i++) { - File temp = folder.newFile("test_" + i + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(130).toMillis()); - } - - for (int i=0; i<7; i++) { - File temp = folder.newFile("test_" + i + "_file.test"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(250).toMillis()); - } - - for (int i=0; i<5; i++) { - File temp = folder.newFile(i + "-abc" + ".json"); - temp.setLastModified(System.currentTimeMillis() - i*Duration.ofSeconds(80).toMillis()); - } - - File temp = folder.newFile("week_old_file.json"); - temp.setLastModified(System.currentTimeMillis() - Duration.ofDays(8).toMillis()); - } - - @Test - public void testDeleteAll() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.empty(), false); - - assertEquals(0, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(6, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file - } - - @Test - public void testDeleteSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".json$"), false); - - assertEquals(7, getContentsOfDirectory(folder.getRoot()).length); - } - - @Test - public void testDeletePrefixAndSuffix() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 7 test_*_file.test files + week_old_file - } - - @Test - public void testDeleteOld() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(600), Optional.empty(), false); - - assertEquals(13, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 6 (from test_*_.json) - 3 (from test_*_file.test) - 1 week old file - } - - @Test - public void testDeleteWithAllParameters() throws IOException { - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ofSeconds(200), Optional.of("^test_.*\\.json$"), false); - - assertEquals(15, getContentsOfDirectory(folder.getRoot()).length); // All 23 - 8 (from test_*_.json) - } - - @Test - public void testDeleteWithSubDirectoriesNoRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), false); - - // 6 test_*.json from test_folder1/ - // + 9 test_*.json and 4 abc_*.json from test_folder2/ - // + 13 test_*.json from test_folder2/subSubFolder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder1/ and test_folder2/ and test_folder2/subSubFolder2/ themselves - assertEquals(48, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteWithSubDirectoriesRecursive() throws IOException { - initSubDirectories(); - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_.*\\.json$"), true); - - // 4 abc_*.json from test_folder2/ - // + 7 test_*_file.test and 5 *-abc.json and 1 week_old_file from root - // + test_folder2/ itself - assertEquals(18, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteFilesWhereFilenameRegexAlsoMatchesDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteFiles(folder.getRoot().toPath(), Duration.ZERO, Optional.of("^test_"), false); - - assertEquals(8, getContentsOfDirectory(folder.getRoot()).length); // 5 abc files + 1 week_old_file + 2 directories - } - - @Test - public void testGetContentsOfNonExistingDirectory() { - Path fakePath = Paths.get("/some/made/up/dir/"); - assertEquals(Collections.emptyList(), FileHelper.listContentsOfDirectory(fakePath)); - } - - @Test(expected=IllegalArgumentException.class) - public void testDeleteFilesExceptNMostRecentWithNegativeN() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), -5); - } - - @Test - public void testDeleteFilesExceptFiveMostRecent() throws IOException { - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 5); - - assertEquals(5, getContentsOfDirectory(folder.getRoot()).length); - - String[] oldestFiles = {"test_5_file.test", "test_6_file.test", "test_8.json", "test_9.json", "week_old_file.json"}; - String[] remainingFiles = Arrays.stream(getContentsOfDirectory(folder.getRoot())) - .map(File::getName) - .sorted() - .toArray(String[]::new); - - assertArrayEquals(oldestFiles, remainingFiles); - } - - @Test - public void testDeleteFilesExceptNMostRecentWithLargeN() throws IOException { - String[] filesPreDelete = folder.getRoot().list(); - - FileHelper.deleteFilesExceptNMostRecent(folder.getRoot().toPath(), 50); - - assertArrayEquals(filesPreDelete, folder.getRoot().list()); - } - - @Test - public void testDeleteFilesLargerThan10B() throws IOException { - initSubDirectories(); - - File temp1 = new File(folder.getRoot(), "small_file"); - writeNBytesToFile(temp1, 50); - - File temp2 = new File(folder.getRoot(), "some_file"); - writeNBytesToFile(temp2, 20); - - File temp3 = new File(folder.getRoot(), "test_folder1/some_other_file"); - writeNBytesToFile(temp3, 75); - - FileHelper.deleteFilesLargerThan(folder.getRoot().toPath(), 10); - - assertEquals(58, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - assertFalse(temp1.exists() || temp2.exists() || temp3.exists()); - } - - @Test - public void testDeleteDirectories() throws IOException { - initSubDirectories(); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ZERO, Optional.of(".*folder2")); - - //23 files in root - // + 6 in test_folder1 + test_folder1 itself - assertEquals(30, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testDeleteDirectoriesBasedOnAge() throws IOException { - initSubDirectories(); - // Create folder3 which is older than maxAge, inside have a single directory, subSubFolder3, inside it which is - // also older than maxAge inside the sub directory, create some files which are newer than maxAge. - // deleteDirectories() should NOT delete folder3 - File subFolder3 = folder.newFolder("test_folder3"); - File subSubFolder3 = folder.newFolder("test_folder3", "subSubFolder3"); - - for (int j=0; j<11; j++) { - File.createTempFile("test_", ".json", subSubFolder3); - } - - subFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder3.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - - FileHelper.deleteDirectories(folder.getRoot().toPath(), Duration.ofSeconds(50), Optional.of(".*folder.*")); - - //23 files in root - // + 13 in test_folder2 - // + 13 in subSubFolder2 - // + 11 in subSubFolder3 - // + test_folder2 + subSubFolder2 + folder3 + subSubFolder3 itself - assertEquals(64, getNumberOfFilesAndDirectoriesIn(folder.getRoot())); - } - - @Test - public void testRecursivelyDeleteDirectory() throws IOException { - initSubDirectories(); - FileHelper.recursiveDelete(folder.getRoot().toPath()); - assertFalse(folder.getRoot().exists()); - } - - @Test - public void testRecursivelyDeleteRegularFile() throws IOException { - File file = folder.newFile(); - assertTrue(file.exists()); - assertTrue(file.isFile()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testRecursivelyDeleteNonExistingFile() throws IOException { - File file = folder.getRoot().toPath().resolve("non-existing-file.json").toFile(); - assertFalse(file.exists()); - FileHelper.recursiveDelete(file.toPath()); - assertFalse(file.exists()); - } - - @Test - public void testInitSubDirectories() throws IOException { - initSubDirectories(); - assertTrue(folder.getRoot().exists()); - assertTrue(folder.getRoot().isDirectory()); - - Path test_folder1 = folder.getRoot().toPath().resolve("test_folder1"); - assertTrue(test_folder1.toFile().exists()); - assertTrue(test_folder1.toFile().isDirectory()); - - Path test_folder2 = folder.getRoot().toPath().resolve("test_folder2"); - assertTrue(test_folder2.toFile().exists()); - assertTrue(test_folder2.toFile().isDirectory()); - - Path subSubFolder2 = test_folder2.resolve("subSubFolder2"); - assertTrue(subSubFolder2.toFile().exists()); - assertTrue(subSubFolder2.toFile().isDirectory()); - } - - @Test - public void testDoesNotFailOnLastModifiedOnSymLink() throws IOException { - Path symPath = folder.getRoot().toPath().resolve("symlink"); - Path fakePath = Paths.get("/some/not/existant/file"); - - Files.createSymbolicLink(symPath, fakePath); - assertTrue(Files.isSymbolicLink(symPath)); - assertFalse(Files.exists(fakePath)); - - // Not possible to set modified time on symlink in java, so just check that it doesn't crash - FileHelper.getLastModifiedTime(symPath).toInstant(); - } - - private void initSubDirectories() throws IOException { - File subFolder1 = folder.newFolder("test_folder1"); - File subFolder2 = folder.newFolder("test_folder2"); - File subSubFolder2 = folder.newFolder("test_folder2", "subSubFolder2"); - - for (int j=0; j<6; j++) { - File temp = File.createTempFile("test_", ".json", subFolder1); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(60).toMillis()); - } - - for (int j=0; j<9; j++) { - File.createTempFile("test_", ".json", subFolder2); - } - - for (int j=0; j<4; j++) { - File.createTempFile("abc_", ".txt", subFolder2); - } - - for (int j=0; j<13; j++) { - File temp = File.createTempFile("test_", ".json", subSubFolder2); - temp.setLastModified(System.currentTimeMillis() - (j+1)*Duration.ofSeconds(40).toMillis()); - } - - //Must be after all the files have been created - subFolder1.setLastModified(System.currentTimeMillis() - Duration.ofHours(2).toMillis()); - subFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(1).toMillis()); - subSubFolder2.setLastModified(System.currentTimeMillis() - Duration.ofHours(3).toMillis()); - } - - private static int getNumberOfFilesAndDirectoriesIn(File folder) { - int total = 0; - for (File file : getContentsOfDirectory(folder)) { - if (file.isDirectory()) { - total += getNumberOfFilesAndDirectoriesIn(file); - } - total++; - } - - return total; - } - - private static void writeNBytesToFile(File file, int nBytes) throws IOException { - Files.write(file.toPath(), new byte[nBytes]); - } - - private static File[] getContentsOfDirectory(File directory) { - File[] directoryContents = directory.listFiles(); - - return directoryContents == null ? new File[0] : directoryContents; - } -} |